package main import ( "crypto/tls" "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/http" "os" "sort" "strings" "text/tabwriter" "time" ) const ( timeout = time.Minute * 10 maxHttpGetErrors = 3 maxHttpRespBytes = 8 * 1024 * 1024 nodesURL = "http://map.hochstift.freifunk.net/data/nodes.json" addressCache = "knownAddresses.json" marshalIndent = " " sleepBetweenLookup = time.Second maxPrintErrors = 10 ) var start time.Time /* ----- Freifunk ----- */ type Data struct { Nodes []Node } type Node struct { Flags Flags Nodeinfo Nodeinfo } type Flags struct { Gateway bool Online bool } type Nodeinfo struct { Hostname string Location Location } type Location struct { Latitude float64 Longitude float64 } func (d *Data) extractLocations() ([]FlatNode, []error) { locs := make([]FlatNode, 0) errs := make([]error, 0) if d == nil { return locs, errs } for _, node := range d.Nodes { if node.isValid() { flatnode, err := node.extractLocation() if !flatnode.isValid() { errMsg := "Could not extract valid address for " + node.Nodeinfo.Hostname if err != nil { errMsg += ": " + err.Error() } else { errMsg += " (" + node.Nodeinfo.Location.String() + ")" } err = errors.New(errMsg) errs = append(errs, err) } else { locs = append(locs, flatnode) } } } return locs, errs } func (n *Node) isValid() bool { if n == nil { return false } location := n.Nodeinfo.Location return n.Flags.isActive() && !location.isEmpty() } func (n *Node) extractLocation() (FlatNode, error) { if n == nil { return FlatNode{}, nil } address, err := n.Nodeinfo.Location.fetchAddress() if err == nil { lat, lon := n.Nodeinfo.Location.values() return FlatNode{n.Nodeinfo.Hostname, fmt.Sprintf("%v", lat), fmt.Sprintf("%v", lon), address}, nil } return FlatNode{}, err } func (f *Flags) isActive() bool { if f == nil { return false } return !f.Gateway && f.Online } func (l *Location) values() (float64, float64) { if l == nil { return 0, 0 } return l.Latitude, l.Longitude } func (l *Location) isEmpty() bool { if l == nil { return true } return l.Latitude == 0 || l.Longitude == 0 } var knownAddresses map[string]Address func (l *Location) cachedAddress() (Address, bool) { if l == nil { return Address{}, false } val, ok := knownAddresses[l.String()] return val, ok } var shallSleep bool func (l *Location) fetchAddress() (Address, error) { if l == nil { return Address{}, nil } if cachedAddress, hasKey := l.cachedAddress(); hasKey { return cachedAddress, nil } if time.Since(start) > timeout { return Address{}, errors.New("Global timeout for location " + l.String()) } if shallSleep { time.Sleep(sleepBetweenLookup) // Nominatim allows 1rq/s max } content, err := httpRequest(NominatimRequest(l)) shallSleep = true if err == nil && len(content) > 0 { var d Nominatim err = json.Unmarshal(content, &d) if err == nil { address := d.Address.simpleAddress() knownAddresses[l.String()] = address return address, nil } } return Address{}, errors.New("fetchAddress: " + err.Error()) } func (l *Location) String() string { if l == nil { return "" } return fmt.Sprintf("%v, %v", l.Latitude, l.Longitude) } /* ----- End Freifunk ----- */ /* ----- Output ----- */ type FlatNode struct { Hostname string Lat string Lon string Address Address } type Address struct { Street string House string Town string } func (f *FlatNode) isValid() bool { if f == nil { return false } return f.Address.isValid() } func (a Address) fields() []string { return []string{a.Street, a.House, a.Town} } func (a Address) isValid() bool { for _, field := range a.fields() { if field != "" { return true } } return false } func (a Address) String() string { s := "" for _, field := range a.fields() { if field != "" { s += " " + field } } return strings.TrimSpace(s) } /* ----- End Output ----- */ /* ----- Nominatim ----- */ func NominatimRequest(l *Location) string { lat, lon := l.values() return fmt.Sprintf("http://nominatim.openstreetmap.org/reverse?format=jsonv2&addressdetails=1&lat=%v&lon=%v", lat, lon) } type Nominatim struct { Address NominatimAddress } type NominatimAddress struct { House_Number string Path string Footway string Pedestrian string Road string Suburb string Village string Town string City string } func first(values []string) string { for _, v := range values { if v != "" { return v } } return "" } func (a *NominatimAddress) simpleAddress() Address { if a == nil { return Address{} } street := a.getStreet() town := a.getTown() house := a.House_Number return Address{street, house, town} } func (a NominatimAddress) getStreet() string { prio := []string{a.Path, a.Footway, a.Pedestrian, a.Road} return first(prio) } func (a NominatimAddress) getTown() string { prio := []string{a.Village, a.Suburb, a.Town, a.City} return first(prio) } /* ----- End Nominatim ----- */ var httpGetErrors int func httpRequest(address string) ([]byte, error) { if httpGetErrors > maxHttpGetErrors { return nil, nil } tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} client := &http.Client{Transport: tr} resp, err := client.Get(address) if err != nil { httpGetErrors += 1 return nil, err } defer resp.Body.Close() content, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxHttpRespBytes)) return content, err } func loadNodes() (*Data, error) { d := &Data{} text, err := httpRequest(nodesURL) if err == nil { err = json.Unmarshal(text, d) } if err != nil { err = errors.New("loadNodes: " + err.Error()) } return d, err } func loadKnownAddresses() { knownAddresses = make(map[string]Address, 0) data, err := ioutil.ReadFile(addressCache) if err == nil { err = json.Unmarshal(data, &knownAddresses) if err != nil { knownAddresses = make(map[string]Address, 0) } } if len(knownAddresses) == 0 { fmt.Fprintln(os.Stderr, "No cached addresses found - this may take a while (timeout: "+timeout.String()+")") } } func saveKnownAddresses() error { data, err := json.MarshalIndent(knownAddresses, "", marshalIndent) if err == nil { ioutil.WriteFile(addressCache, []byte(data), 0644) } else { err = errors.New("saveKnownAddresses: " + err.Error()) } return err } func getLocations() ([]FlatNode, []error) { errs := make([]error, 0) loadKnownAddresses() data, err := loadNodes() if err != nil { errs = append(errs, err) } nodes, errors := data.extractLocations() errs = append(errs, errors...) err = saveKnownAddresses() if err != nil { errs = append(errs, err) } return nodes, errs } type ByStreet []FlatNode func (m ByStreet) Len() int { return len(m) } func (m ByStreet) Swap(i, j int) { m[i], m[j] = m[j], m[i] } func (m ByStreet) Less(i, j int) bool { towns := strings.Compare(m[i].Address.Town, m[j].Address.Town) streets := strings.Compare(m[i].Address.Street, m[j].Address.Street) houses := strings.Compare(m[i].Address.House, m[j].Address.House) // since they may contain more than digits return towns < 0 || (towns == 0 && streets < 0) || (towns == 0 && streets == 0 && houses < 0) } func timestamp() string { return "# " + time.Now().String() } func printFormatted(nodes []FlatNode) { if len(nodes) == 0 { return } const minWidth = 0 const tabWidth = 0 const padding = 3 const padChar = ' ' const flags = 0 fmt.Println(timestamp() + "\n") w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags) sort.Sort(ByStreet(nodes)) for _, node := range nodes { fmt.Fprintf(w, "%v\t| %v\t| %v\t| %v\n", node.Hostname, node.Lat, node.Lon, node.Address) } w.Flush() } func main() { start = time.Now() nodes, errs := getLocations() printFormatted(nodes) cErrors := len(errs) if cErrors > 0 { fmt.Fprintln(os.Stderr, timestamp()) if cErrors > maxPrintErrors { cErrors = maxPrintErrors } for i := 0; i < cErrors; i++ { fmt.Fprintln(os.Stderr, errs[i]) } os.Exit(-1) } }