initial commit

FFReverseGeo.go

@@ -0,0 +1,391 @@
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+)
+
+const (
+	timeout            = time.Minute * 10
+	maxHttpGetErrors   = 3
+	maxHttpRespBytes   = 8 * 1024 * 1024
+	nodesURL           = "http://map.hochstift.freifunk.net/data/nodes.json"
+	addressCache       = "knownAddresses.json"
+	marshalIndent      = "  "
+	sleepBetweenLookup = time.Second
+	maxPrintErrors     = 10
+)
+
+var start time.Time
+
+/* ----- Freifunk ----- */
+
+type Data struct {
+	Nodes []Node
+}
+
+type Node struct {
+	Flags    Flags
+	Nodeinfo Nodeinfo
+}
+
+type Flags struct {
+	Gateway bool
+	Online  bool
+}
+
+type Nodeinfo struct {
+	Hostname string
+	Location Location
+}
+
+type Location struct {
+	Latitude  float64
+	Longitude float64
+}
+
+func (d *Data) extractLocations() ([]FlatNode, []error) {
+	locs := make([]FlatNode, 0)
+	errs := make([]error, 0)
+	if d == nil {
+		return locs, errs
+	}
+	for _, node := range d.Nodes {
+		if node.isValid() {
+			flatnode, err := node.extractLocation()
+			if !flatnode.isValid() {
+				errMsg := "Could not extract valid address for " + node.Nodeinfo.Hostname
+				if err != nil {
+					errMsg += ": " + err.Error()
+				} else {
+					errMsg += " (" + node.Nodeinfo.Location.String() + ")"
+				}
+				err = errors.New(errMsg)
+				errs = append(errs, err)
+			} else {
+				locs = append(locs, flatnode)
+			}
+		}
+	}
+	return locs, errs
+}
+
+func (n *Node) isValid() bool {
+	if n == nil {
+		return false
+	}
+	location := n.Nodeinfo.Location
+	return n.Flags.isActive() && !location.isEmpty()
+}
+
+func (n *Node) extractLocation() (FlatNode, error) {
+	if n == nil {
+		return FlatNode{}, nil
+	}
+	address, err := n.Nodeinfo.Location.fetchAddress()
+	if err == nil {
+		lat, lon := n.Nodeinfo.Location.values()
+		return FlatNode{n.Nodeinfo.Hostname, fmt.Sprintf("%v", lat), fmt.Sprintf("%v", lon), address}, nil
+	}
+	return FlatNode{}, err
+}
+
+func (f *Flags) isActive() bool {
+	if f == nil {
+		return false
+	}
+	return !f.Gateway && f.Online
+}
+
+func (l *Location) values() (float64, float64) {
+	if l == nil {
+		return 0, 0
+	}
+	return l.Latitude, l.Longitude
+}
+
+func (l *Location) isEmpty() bool {
+	if l == nil {
+		return true
+	}
+	return l.Latitude == 0 || l.Longitude == 0
+}
+
+var knownAddresses map[string]Address
+
+func (l *Location) cachedAddress() (Address, bool) {
+	if l == nil {
+		return Address{}, false
+	}
+	val, ok := knownAddresses[l.String()]
+	return val, ok
+}
+
+var shallSleep bool
+
+func (l *Location) fetchAddress() (Address, error) {
+	if l == nil {
+		return Address{}, nil
+	}
+	if cachedAddress, hasKey := l.cachedAddress(); hasKey {
+		return cachedAddress, nil
+	}
+	if time.Since(start) > timeout {
+		return Address{}, errors.New("Global timeout for location " + l.String())
+	}
+	if shallSleep {
+		time.Sleep(sleepBetweenLookup) // Nominatim allows 1rq/s max
+	}
+	content, err := httpRequest(NominatimRequest(l))
+	shallSleep = true
+	if err == nil && len(content) > 0 {
+		var d Nominatim
+		err = json.Unmarshal(content, &d)
+		if err == nil {
+			address := d.Address.simpleAddress()
+			knownAddresses[l.String()] = address
+			return address, nil
+		}
+	}
+	return Address{}, errors.New("fetchAddress: " + err.Error())
+}
+
+func (l *Location) String() string {
+	if l == nil {
+		return ""
+	}
+	return fmt.Sprintf("%v, %v", l.Latitude, l.Longitude)
+}
+
+/* ----- End Freifunk ----- */
+
+/* ----- Output ----- */
+
+type FlatNode struct {
+	Hostname string
+	Lat      string
+	Lon      string
+	Address  Address
+}
+
+type Address struct {
+	Street string
+	House  string
+	Town   string
+}
+
+func (f *FlatNode) isValid() bool {
+	if f == nil {
+		return false
+	}
+	return f.Address.isValid()
+}
+
+func (a Address) fields() []string {
+	return []string{a.Street, a.House, a.Town}
+}
+
+func (a Address) isValid() bool {
+	for _, field := range a.fields() {
+		if field != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func (a Address) String() string {
+	s := ""
+	for _, field := range a.fields() {
+		if field != "" {
+			s += " " + field
+		}
+	}
+	return strings.TrimSpace(s)
+}
+
+/* ----- End Output ----- */
+
+/* ----- Nominatim ----- */
+
+func NominatimRequest(l *Location) string {
+	lat, lon := l.values()
+	return fmt.Sprintf("http://nominatim.openstreetmap.org/reverse?format=jsonv2&addressdetails=1&lat=%v&lon=%v", lat, lon)
+}
+
+type Nominatim struct {
+	Address NominatimAddress
+}
+
+type NominatimAddress struct {
+	House_Number string
+	Path         string
+	Footway      string
+	Pedestrian   string
+	Road         string
+	Suburb       string
+	Village      string
+	Town         string
+	City         string
+}
+
+func first(values []string) string {
+	for _, v := range values {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func (a *NominatimAddress) simpleAddress() Address {
+	if a == nil {
+		return Address{}
+	}
+	street := a.getStreet()
+	town := a.getTown()
+	house := a.House_Number
+	return Address{street, house, town}
+}
+
+func (a NominatimAddress) getStreet() string {
+	prio := []string{a.Path, a.Footway, a.Pedestrian, a.Road}
+	return first(prio)
+}
+
+func (a NominatimAddress) getTown() string {
+	prio := []string{a.Village, a.Suburb, a.Town, a.City}
+	return first(prio)
+}
+
+/* ----- End Nominatim ----- */
+
+var httpGetErrors int
+
+func httpRequest(address string) ([]byte, error) {
+	if httpGetErrors > maxHttpGetErrors {
+		return nil, nil
+	}
+	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(address)
+	if err != nil {
+		httpGetErrors += 1
+		return nil, err
+	}
+	defer resp.Body.Close()
+	content, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxHttpRespBytes))
+	return content, err
+}
+
+func loadNodes() (*Data, error) {
+	d := &Data{}
+	text, err := httpRequest(nodesURL)
+	if err == nil {
+		err = json.Unmarshal(text, d)
+	}
+	if err != nil {
+		err = errors.New("loadNodes: " + err.Error())
+	}
+	return d, err
+}
+
+func loadKnownAddresses() {
+	knownAddresses = make(map[string]Address, 0)
+	data, err := ioutil.ReadFile(addressCache)
+	if err == nil {
+		err = json.Unmarshal(data, &knownAddresses)
+		if err != nil {
+			knownAddresses = make(map[string]Address, 0)
+		}
+	}
+	if len(knownAddresses) == 0 {
+		fmt.Fprintln(os.Stderr, "No cached addresses found - this may take a while (timeout: "+timeout.String()+")")
+	}
+}
+
+func saveKnownAddresses() error {
+	data, err := json.MarshalIndent(knownAddresses, "", marshalIndent)
+	if err == nil {
+		ioutil.WriteFile(addressCache, []byte(data), 0644)
+	} else {
+		err = errors.New("saveKnownAddresses: " + err.Error())
+	}
+	return err
+}
+
+func getLocations() ([]FlatNode, []error) {
+	errs := make([]error, 0)
+	loadKnownAddresses()
+	data, err := loadNodes()
+	if err != nil {
+		errs = append(errs, err)
+	}
+	nodes, errors := data.extractLocations()
+	errs = append(errs, errors...)
+	err = saveKnownAddresses()
+	if err != nil {
+		errs = append(errs, err)
+	}
+	return nodes, errs
+}
+
+type ByStreet []FlatNode
+
+func (m ByStreet) Len() int      { return len(m) }
+func (m ByStreet) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
+func (m ByStreet) Less(i, j int) bool {
+	towns := strings.Compare(m[i].Address.Town, m[j].Address.Town)
+	streets := strings.Compare(m[i].Address.Street, m[j].Address.Street)
+	houses := strings.Compare(m[i].Address.House, m[j].Address.House) // since they may contain more than digits
+	return towns < 0 || (towns == 0 && streets < 0) || (towns == 0 && streets == 0 && houses < 0)
+}
+
+func timestamp() string {
+	return "# " + time.Now().String()
+}
+
+func printFormatted(nodes []FlatNode) {
+	if len(nodes) == 0 {
+		return
+	}
+	const minWidth = 0
+	const tabWidth = 0
+	const padding = 3
+	const padChar = ' '
+	const flags = 0
+	fmt.Println(timestamp() + "\n")
+	w := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	sort.Sort(ByStreet(nodes))
+	for _, node := range nodes {
+		fmt.Fprintf(w, "%v\t| %v\t| %v\t| %v\n", node.Hostname, node.Lat, node.Lon, node.Address)
+	}
+	w.Flush()
+}
+
+func main() {
+	start = time.Now()
+	nodes, errs := getLocations()
+	printFormatted(nodes)
+	cErrors := len(errs)
+	if cErrors > 0 {
+		fmt.Fprintln(os.Stderr, timestamp())
+		if cErrors > maxPrintErrors {
+			cErrors = maxPrintErrors
+		}
+		for i := 0; i < cErrors; i++ {
+			fmt.Fprintln(os.Stderr, errs[i])
+		}
+		os.Exit(-1)
+	}
+}

FFReverseGeo.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+minSize=50 #lines
+dest=/usr/jails/webserver/usr/local/www/gutmet.org/freifunk/
+destErrs=$dest
+
+addrFile=Adressen.txt
+errFile=Errors.txt
+
+dir=`dirname $0`
+cd $dir
+stdbuf -oL -eL ./FFReverseGeo 2>> $errFile > $addrFile
+
+size=`wc -l < $addrFile`
+if [ "$size" -gt "$minSize" ]; then
+    cp $addrFile $dest
+else
+    date >> $errFile
+    echo "Got weird size: $size" >> $errFile
+fi
+
+cp $errFile $destErrs

Makefile

@@ -0,0 +1,12 @@
+EXE=FFReverseGeo
+SRC=$(EXE).go
+GOOS=linux
+GOARCH=amd64
+
+all: $(EXE)
+
+$(EXE):
+	GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 go build -ldflags '-s -w' $(SRC)
+
+clean:
+	-rm $(EXE)