FFReverseGeo/FFReverseGeo.go

package main

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"text/tabwriter"
	"time"
)

const (
	timeout            = time.Minute * 10
	maxHttpGetErrors   = 3
	maxHttpRespBytes   = 8 * 1024 * 1024
	nodesURL           = "http://map.hochstift.freifunk.net/data/nodes.json"
	addressCache       = "knownAddresses.json"
	marshalIndent      = "  "
	sleepBetweenLookup = time.Second
	maxPrintErrors     = 10
)

var start time.Time

/* ----- Freifunk ----- */

type Data struct {
	Nodes []Node
}

type Node struct {
	Flags    Flags
	Nodeinfo Nodeinfo
}

type Flags struct {
	Gateway bool
	Online  bool
}

type Nodeinfo struct {
	Hostname string
	Location Location
}

type Location struct {
	Latitude  float64
	Longitude float64
}

func (d *Data) extractLocations() ([]FlatNode, []error) {
	locs := make([]FlatNode, 0)
	errs := make([]error, 0)
	if d == nil {
		return locs, errs
	}
	for _, node := range d.Nodes {
		if node.isValid() {
			flatnode, err := node.extractLocation()
			if !flatnode.isValid() {
				errMsg := "Could not extract valid address for " + node.Nodeinfo.Hostname
				if err != nil {
					errMsg += ": " + err.Error()
				} else {
					errMsg += " (" + node.Nodeinfo.Location.String() + ")"
				}
				err = errors.New(errMsg)
				errs = append(errs, err)
			} else {
				locs = append(locs, flatnode)
			}
		}
	}
	return locs, errs
}

func (n *Node) isValid() bool {
	if n == nil {
		return false
	}
	location := n.Nodeinfo.Location
	return n.Flags.isActive() && !location.isEmpty()
}

func (n *Node) extractLocation() (FlatNode, error) {
	if n == nil {
		return FlatNode{}, nil
	}
	address, err := n.Nodeinfo.Location.fetchAddress()
	if err == nil {
		lat, lon := n.Nodeinfo.Location.values()
		return FlatNode{n.Nodeinfo.Hostname, fmt.Sprintf("%v", lat), fmt.Sprintf("%v", lon), address}, nil
	}
	return FlatNode{}, err
}

func (f *Flags) isActive() bool {
	if f == nil {
		return false
	}
	return !f.Gateway && f.Online
}

func (l *Location) values() (float64, float64) {
	if l == nil {
		return 0, 0
	}
	return l.Latitude, l.Longitude
}

func (l *Location) isEmpty() bool {
	if l == nil {
		return true
	}
	return l.Latitude == 0 || l.Longitude == 0
}

var knownAddresses map[string]Address

func (l *Location) cachedAddress() (Address, bool) {
	if l == nil {
		return Address{}, false
	}
	val, ok := knownAddresses[l.String()]
	return val, ok
}

var shallSleep bool

func (l *Location) fetchAddress() (Address, error) {
	if l == nil {
		return Address{}, nil
	}
	if cachedAddress, hasKey := l.cachedAddress(); hasKey {
		return cachedAddress, nil
	}
	if time.Since(start) > timeout {
		return Address{}, errors.New("Global timeout for location " + l.String())
	}
	if shallSleep {
		time.Sleep(sleepBetweenLookup) // Nominatim allows 1rq/s max
	}
	content, err := httpRequest(NominatimRequest(l))
	shallSleep = true
	if err == nil && len(content) > 0 {
		var d Nominatim
		err = json.Unmarshal(content, &d)
		if err == nil {
			address := d.Address.simpleAddress()
			knownAddresses[l.String()] = address
			return address, nil
		}
	}
	return Address{}, errors.New("fetchAddress: " + err.Error())
}

func (l *Location) String() string {
	if l == nil {
		return ""
	}
	return fmt.Sprintf("%v, %v", l.Latitude, l.Longitude)
}

/* ----- End Freifunk ----- */

/* ----- Output ----- */

type FlatNode struct {
	Hostname string
	Lat      string
	Lon      string
	Address  Address
}

type Address struct {
	Street string
	House  string
	Town   string
}

func (f *FlatNode) isValid() bool {
	if f == nil {
		return false
	}
	return f.Address.isValid()
}

func (a Address) fields() []string {
	return []string{a.Street, a.House, a.Town}
}

func (a Address) isValid() bool {
	for _, field := range a.fields() {
		if field != "" {
			return true
		}
	}
	return false
}

func (a Address) String() string {
	s := ""
	for _, field := range a.fields() {
		if field != "" {
			s += " " + field
		}
	}
	return strings.TrimSpace(s)
}

/* ----- End Output ----- */

/* ----- Nominatim ----- */

func NominatimRequest(l *Location) string {
	lat, lon := l.values()
	return fmt.Sprintf("http://nominatim.openstreetmap.org/reverse?format=jsonv2&addressdetails=1&lat=%v&lon=%v", lat, lon)
}

type Nominatim struct {
	Address NominatimAddress
}

type NominatimAddress struct {
	House_Number string
	Path         string
	Footway      string
	Pedestrian   string
	Road         string
	Suburb       string
	Village      string
	Town         string
	City         string
}

func first(values []string) string {
	for _, v := range values {
		if v != "" {
			return v
		}
	}
	return ""
}

func (a *NominatimAddress) simpleAddress() Address {
	if a == nil {
		return Address{}
	}
	street := a.getStreet()
	town := a.getTown()
	house := a.House_Number
	return Address{street, house, town}
}

func (a NominatimAddress) getStreet() string {
	prio := []string{a.Path, a.Footway, a.Pedestrian, a.Road}
	return first(prio)
}

func (a NominatimAddress) getTown() string {
	prio := []string{a.Village, a.Suburb, a.Town, a.City}
	return first(prio)
}

/* ----- End Nominatim ----- */

var httpGetErrors int

func httpRequest(address string) ([]byte, error) {
	if httpGetErrors > maxHttpGetErrors {
		return nil, nil
	}
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	resp, err := client.Get(address)
	if err != nil {
		httpGetErrors += 1
		return nil, err
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxHttpRespBytes))
	return content, err
}

func loadNodes() (*Data, error) {
	d := &Data{}
	text, err := httpRequest(nodesURL)
	if err == nil {
		err = json.Unmarshal(text, d)
	}
	if err != nil {
		err = errors.New("loadNodes: " + err.Error())
	}
	return d, err
}

func loadKnownAddresses() {
	knownAddresses = make(map[string]Address, 0)
	data, err := ioutil.ReadFile(addressCache)
	if err == nil {
		err = json.Unmarshal(data, &knownAddresses)
		if err != nil {
			knownAddresses = make(map[string]Address, 0)
		}
	}
	if len(knownAddresses) == 0 {
		fmt.Fprintln(os.Stderr, "No cached addresses found - this may take a while (timeout: "+timeout.String()+")")
	}
}

func saveKnownAddresses() error {
	data, err := json.MarshalIndent(knownAddresses, "", marshalIndent)
	if err == nil {
		ioutil.WriteFile(addressCache, []byte(data), 0644)
	} else {
		err = errors.New("saveKnownAddresses: " + err.Error())
	}
	return err
}

func getLocations() ([]FlatNode, []error) {
	errs := make([]error, 0)
	loadKnownAddresses()
	data, err := loadNodes()
	if err != nil {
		errs = append(errs, err)
	}
	nodes, errors := data.extractLocations()
	errs = append(errs, errors...)
	err = saveKnownAddresses()
	if err != nil {
		errs = append(errs, err)
	}
	return nodes, errs
}

type ByStreet []FlatNode

func (m ByStreet) Len() int      { return len(m) }
func (m ByStreet) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m ByStreet) Less(i, j int) bool {
	towns := strings.Compare(m[i].Address.Town, m[j].Address.Town)
	streets := strings.Compare(m[i].Address.Street, m[j].Address.Street)
	houses := strings.Compare(m[i].Address.House, m[j].Address.House) // since they may contain more than digits
	return towns < 0 || (towns == 0 && streets < 0) || (towns == 0 && streets == 0 && houses < 0)
}

func timestamp() string {
	return "# " + time.Now().String()
}

func printFormatted(nodes []FlatNode) {
	if len(nodes) == 0 {
		return
	}
	const minWidth = 0
	const tabWidth = 0
	const padding = 3
	const padChar = ' '
	const flags = 0
	fmt.Println(timestamp() + "\n")
	w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
	sort.Sort(ByStreet(nodes))
	for _, node := range nodes {
		fmt.Fprintf(w, "%v\t| %v\t| %v\t| %v\n", node.Hostname, node.Lat, node.Lon, node.Address)
	}
	w.Flush()
}

func main() {
	start = time.Now()
	nodes, errs := getLocations()
	printFormatted(nodes)
	cErrors := len(errs)
	if cErrors > 0 {
		fmt.Fprintln(os.Stderr, timestamp())
		if cErrors > maxPrintErrors {
			cErrors = maxPrintErrors
		}
		for i := 0; i < cErrors; i++ {
			fmt.Fprintln(os.Stderr, errs[i])
		}
		os.Exit(-1)
	}
}