392 lines
8.0 KiB
Go
392 lines
8.0 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"crypto/tls"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"io/ioutil"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
"sort"
|
||
|
|
"strings"
|
||
|
|
"text/tabwriter"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
timeout = time.Minute * 10
|
||
|
|
maxHttpGetErrors = 3
|
||
|
|
maxHttpRespBytes = 8 * 1024 * 1024
|
||
|
|
nodesURL = "http://map.hochstift.freifunk.net/data/nodes.json"
|
||
|
|
addressCache = "knownAddresses.json"
|
||
|
|
marshalIndent = " "
|
||
|
|
sleepBetweenLookup = time.Second
|
||
|
|
maxPrintErrors = 10
|
||
|
|
)
|
||
|
|
|
||
|
|
var start time.Time
|
||
|
|
|
||
|
|
/* ----- Freifunk ----- */
|
||
|
|
|
||
|
|
type Data struct {
|
||
|
|
Nodes []Node
|
||
|
|
}
|
||
|
|
|
||
|
|
type Node struct {
|
||
|
|
Flags Flags
|
||
|
|
Nodeinfo Nodeinfo
|
||
|
|
}
|
||
|
|
|
||
|
|
type Flags struct {
|
||
|
|
Gateway bool
|
||
|
|
Online bool
|
||
|
|
}
|
||
|
|
|
||
|
|
type Nodeinfo struct {
|
||
|
|
Hostname string
|
||
|
|
Location Location
|
||
|
|
}
|
||
|
|
|
||
|
|
type Location struct {
|
||
|
|
Latitude float64
|
||
|
|
Longitude float64
|
||
|
|
}
|
||
|
|
|
||
|
|
func (d *Data) extractLocations() ([]FlatNode, []error) {
|
||
|
|
locs := make([]FlatNode, 0)
|
||
|
|
errs := make([]error, 0)
|
||
|
|
if d == nil {
|
||
|
|
return locs, errs
|
||
|
|
}
|
||
|
|
for _, node := range d.Nodes {
|
||
|
|
if node.isValid() {
|
||
|
|
flatnode, err := node.extractLocation()
|
||
|
|
if !flatnode.isValid() {
|
||
|
|
errMsg := "Could not extract valid address for " + node.Nodeinfo.Hostname
|
||
|
|
if err != nil {
|
||
|
|
errMsg += ": " + err.Error()
|
||
|
|
} else {
|
||
|
|
errMsg += " (" + node.Nodeinfo.Location.String() + ")"
|
||
|
|
}
|
||
|
|
err = errors.New(errMsg)
|
||
|
|
errs = append(errs, err)
|
||
|
|
} else {
|
||
|
|
locs = append(locs, flatnode)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return locs, errs
|
||
|
|
}
|
||
|
|
|
||
|
|
func (n *Node) isValid() bool {
|
||
|
|
if n == nil {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
location := n.Nodeinfo.Location
|
||
|
|
return n.Flags.isActive() && !location.isEmpty()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (n *Node) extractLocation() (FlatNode, error) {
|
||
|
|
if n == nil {
|
||
|
|
return FlatNode{}, nil
|
||
|
|
}
|
||
|
|
address, err := n.Nodeinfo.Location.fetchAddress()
|
||
|
|
if err == nil {
|
||
|
|
lat, lon := n.Nodeinfo.Location.values()
|
||
|
|
return FlatNode{n.Nodeinfo.Hostname, fmt.Sprintf("%v", lat), fmt.Sprintf("%v", lon), address}, nil
|
||
|
|
}
|
||
|
|
return FlatNode{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (f *Flags) isActive() bool {
|
||
|
|
if f == nil {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return !f.Gateway && f.Online
|
||
|
|
}
|
||
|
|
|
||
|
|
func (l *Location) values() (float64, float64) {
|
||
|
|
if l == nil {
|
||
|
|
return 0, 0
|
||
|
|
}
|
||
|
|
return l.Latitude, l.Longitude
|
||
|
|
}
|
||
|
|
|
||
|
|
func (l *Location) isEmpty() bool {
|
||
|
|
if l == nil {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return l.Latitude == 0 || l.Longitude == 0
|
||
|
|
}
|
||
|
|
|
||
|
|
var knownAddresses map[string]Address
|
||
|
|
|
||
|
|
func (l *Location) cachedAddress() (Address, bool) {
|
||
|
|
if l == nil {
|
||
|
|
return Address{}, false
|
||
|
|
}
|
||
|
|
val, ok := knownAddresses[l.String()]
|
||
|
|
return val, ok
|
||
|
|
}
|
||
|
|
|
||
|
|
var shallSleep bool
|
||
|
|
|
||
|
|
func (l *Location) fetchAddress() (Address, error) {
|
||
|
|
if l == nil {
|
||
|
|
return Address{}, nil
|
||
|
|
}
|
||
|
|
if cachedAddress, hasKey := l.cachedAddress(); hasKey {
|
||
|
|
return cachedAddress, nil
|
||
|
|
}
|
||
|
|
if time.Since(start) > timeout {
|
||
|
|
return Address{}, errors.New("Global timeout for location " + l.String())
|
||
|
|
}
|
||
|
|
if shallSleep {
|
||
|
|
time.Sleep(sleepBetweenLookup) // Nominatim allows 1rq/s max
|
||
|
|
}
|
||
|
|
content, err := httpRequest(NominatimRequest(l))
|
||
|
|
shallSleep = true
|
||
|
|
if err == nil && len(content) > 0 {
|
||
|
|
var d Nominatim
|
||
|
|
err = json.Unmarshal(content, &d)
|
||
|
|
if err == nil {
|
||
|
|
address := d.Address.simpleAddress()
|
||
|
|
knownAddresses[l.String()] = address
|
||
|
|
return address, nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return Address{}, errors.New("fetchAddress: " + err.Error())
|
||
|
|
}
|
||
|
|
|
||
|
|
func (l *Location) String() string {
|
||
|
|
if l == nil {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return fmt.Sprintf("%v, %v", l.Latitude, l.Longitude)
|
||
|
|
}
|
||
|
|
|
||
|
|
/* ----- End Freifunk ----- */
|
||
|
|
|
||
|
|
/* ----- Output ----- */
|
||
|
|
|
||
|
|
type FlatNode struct {
|
||
|
|
Hostname string
|
||
|
|
Lat string
|
||
|
|
Lon string
|
||
|
|
Address Address
|
||
|
|
}
|
||
|
|
|
||
|
|
type Address struct {
|
||
|
|
Street string
|
||
|
|
House string
|
||
|
|
Town string
|
||
|
|
}
|
||
|
|
|
||
|
|
func (f *FlatNode) isValid() bool {
|
||
|
|
if f == nil {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return f.Address.isValid()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a Address) fields() []string {
|
||
|
|
return []string{a.Street, a.House, a.Town}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a Address) isValid() bool {
|
||
|
|
for _, field := range a.fields() {
|
||
|
|
if field != "" {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a Address) String() string {
|
||
|
|
s := ""
|
||
|
|
for _, field := range a.fields() {
|
||
|
|
if field != "" {
|
||
|
|
s += " " + field
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return strings.TrimSpace(s)
|
||
|
|
}
|
||
|
|
|
||
|
|
/* ----- End Output ----- */
|
||
|
|
|
||
|
|
/* ----- Nominatim ----- */
|
||
|
|
|
||
|
|
func NominatimRequest(l *Location) string {
|
||
|
|
lat, lon := l.values()
|
||
|
|
return fmt.Sprintf("http://nominatim.openstreetmap.org/reverse?format=jsonv2&addressdetails=1&lat=%v&lon=%v", lat, lon)
|
||
|
|
}
|
||
|
|
|
||
|
|
type Nominatim struct {
|
||
|
|
Address NominatimAddress
|
||
|
|
}
|
||
|
|
|
||
|
|
type NominatimAddress struct {
|
||
|
|
House_Number string
|
||
|
|
Path string
|
||
|
|
Footway string
|
||
|
|
Pedestrian string
|
||
|
|
Road string
|
||
|
|
Suburb string
|
||
|
|
Village string
|
||
|
|
Town string
|
||
|
|
City string
|
||
|
|
}
|
||
|
|
|
||
|
|
func first(values []string) string {
|
||
|
|
for _, v := range values {
|
||
|
|
if v != "" {
|
||
|
|
return v
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a *NominatimAddress) simpleAddress() Address {
|
||
|
|
if a == nil {
|
||
|
|
return Address{}
|
||
|
|
}
|
||
|
|
street := a.getStreet()
|
||
|
|
town := a.getTown()
|
||
|
|
house := a.House_Number
|
||
|
|
return Address{street, house, town}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a NominatimAddress) getStreet() string {
|
||
|
|
prio := []string{a.Path, a.Footway, a.Pedestrian, a.Road}
|
||
|
|
return first(prio)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (a NominatimAddress) getTown() string {
|
||
|
|
prio := []string{a.Village, a.Suburb, a.Town, a.City}
|
||
|
|
return first(prio)
|
||
|
|
}
|
||
|
|
|
||
|
|
/* ----- End Nominatim ----- */
|
||
|
|
|
||
|
|
var httpGetErrors int
|
||
|
|
|
||
|
|
func httpRequest(address string) ([]byte, error) {
|
||
|
|
if httpGetErrors > maxHttpGetErrors {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||
|
|
client := &http.Client{Transport: tr}
|
||
|
|
resp, err := client.Get(address)
|
||
|
|
if err != nil {
|
||
|
|
httpGetErrors += 1
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer resp.Body.Close()
|
||
|
|
content, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxHttpRespBytes))
|
||
|
|
return content, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadNodes() (*Data, error) {
|
||
|
|
d := &Data{}
|
||
|
|
text, err := httpRequest(nodesURL)
|
||
|
|
if err == nil {
|
||
|
|
err = json.Unmarshal(text, d)
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
err = errors.New("loadNodes: " + err.Error())
|
||
|
|
}
|
||
|
|
return d, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadKnownAddresses() {
|
||
|
|
knownAddresses = make(map[string]Address, 0)
|
||
|
|
data, err := ioutil.ReadFile(addressCache)
|
||
|
|
if err == nil {
|
||
|
|
err = json.Unmarshal(data, &knownAddresses)
|
||
|
|
if err != nil {
|
||
|
|
knownAddresses = make(map[string]Address, 0)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(knownAddresses) == 0 {
|
||
|
|
fmt.Fprintln(os.Stderr, "No cached addresses found - this may take a while (timeout: "+timeout.String()+")")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func saveKnownAddresses() error {
|
||
|
|
data, err := json.MarshalIndent(knownAddresses, "", marshalIndent)
|
||
|
|
if err == nil {
|
||
|
|
ioutil.WriteFile(addressCache, []byte(data), 0644)
|
||
|
|
} else {
|
||
|
|
err = errors.New("saveKnownAddresses: " + err.Error())
|
||
|
|
}
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func getLocations() ([]FlatNode, []error) {
|
||
|
|
errs := make([]error, 0)
|
||
|
|
loadKnownAddresses()
|
||
|
|
data, err := loadNodes()
|
||
|
|
if err != nil {
|
||
|
|
errs = append(errs, err)
|
||
|
|
}
|
||
|
|
nodes, errors := data.extractLocations()
|
||
|
|
errs = append(errs, errors...)
|
||
|
|
err = saveKnownAddresses()
|
||
|
|
if err != nil {
|
||
|
|
errs = append(errs, err)
|
||
|
|
}
|
||
|
|
return nodes, errs
|
||
|
|
}
|
||
|
|
|
||
|
|
type ByStreet []FlatNode
|
||
|
|
|
||
|
|
func (m ByStreet) Len() int { return len(m) }
|
||
|
|
func (m ByStreet) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
|
||
|
|
func (m ByStreet) Less(i, j int) bool {
|
||
|
|
towns := strings.Compare(m[i].Address.Town, m[j].Address.Town)
|
||
|
|
streets := strings.Compare(m[i].Address.Street, m[j].Address.Street)
|
||
|
|
houses := strings.Compare(m[i].Address.House, m[j].Address.House) // since they may contain more than digits
|
||
|
|
return towns < 0 || (towns == 0 && streets < 0) || (towns == 0 && streets == 0 && houses < 0)
|
||
|
|
}
|
||
|
|
|
||
|
|
func timestamp() string {
|
||
|
|
return "# " + time.Now().String()
|
||
|
|
}
|
||
|
|
|
||
|
|
func printFormatted(nodes []FlatNode) {
|
||
|
|
if len(nodes) == 0 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
const minWidth = 0
|
||
|
|
const tabWidth = 0
|
||
|
|
const padding = 3
|
||
|
|
const padChar = ' '
|
||
|
|
const flags = 0
|
||
|
|
fmt.Println(timestamp() + "\n")
|
||
|
|
w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
|
||
|
|
sort.Sort(ByStreet(nodes))
|
||
|
|
for _, node := range nodes {
|
||
|
|
fmt.Fprintf(w, "%v\t| %v\t| %v\t| %v\n", node.Hostname, node.Lat, node.Lon, node.Address)
|
||
|
|
}
|
||
|
|
w.Flush()
|
||
|
|
}
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
start = time.Now()
|
||
|
|
nodes, errs := getLocations()
|
||
|
|
printFormatted(nodes)
|
||
|
|
cErrors := len(errs)
|
||
|
|
if cErrors > 0 {
|
||
|
|
fmt.Fprintln(os.Stderr, timestamp())
|
||
|
|
if cErrors > maxPrintErrors {
|
||
|
|
cErrors = maxPrintErrors
|
||
|
|
}
|
||
|
|
for i := 0; i < cErrors; i++ {
|
||
|
|
fmt.Fprintln(os.Stderr, errs[i])
|
||
|
|
}
|
||
|
|
os.Exit(-1)
|
||
|
|
}
|
||
|
|
}
|