Fix deprecated batctl option

maps only from nodelist.json

.gitignore

@@ -1,3 +1,6 @@
+# backups
+*~
+
 # script-generated
 aliases*.json
 nodedb/

README.md

@@ -18,6 +18,13 @@ For the script's regular execution add the following to the crontab:
 * * * * * backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a d2:d0:93:63:f7:da
 </pre>
 
+# Dependencies
+
+- Python 3
+- Python 3 Package [Networkx](https://networkx.github.io/)
+- [alfred-json](https://github.com/tcatm/alfred-json)
+- rrdtool (if run with `--with-rrd`)
+
 # Running as unprivileged user
 
 Some information collected by ffmap-backend requires access to specific system resources.
@@ -72,6 +79,21 @@ will prefix `sudo` where necessary.
 - online
 - gateway
 
+## Old data format
+
+If you want to still use the old [ffmap-d3](https://github.com/ffnord/ffmap-d3)
+front end, you can use the file `ffmap-d3.jq` to convert the new output to the
+old one:
+
+```
+jq -n -f ffmap-d3.jq \
+    --argfile nodes nodedb/nodes.json \
+    --argfile graph nodedb/graph.json \
+    > nodedb/ffmap-d3.json
+```
+
+Then point your ffmap-d3 instance to the `ffmap-d3.json` file.
+
 # Removing owner information
 
 If you'd like to redact information about the node owner from `nodes.json`,

aliases.json_sample

@@ -7,18 +7,30 @@
       "latitude": 53.86
     },
     "network": {
-      "mesh_interfaces": [
+      "mesh": {
+        "bat0": {
+          "interfaces": {
+            "tunnel": [
               "00:25:86:e6:f1:bf"
             ]
           }
+        }
+      }
+    }
   },
   {
     "node_id": "gw1",
     "hostname": "burgtor",
     "network": {
-      "mesh_interfaces": [
+      "mesh": {
+        "bat0": {
+          "interfaces": {
+            "tunnel": [
               "52:54:00:f3:62:d9"
             ]
           }
         }
+      }
+    }
+  }
 ]

backend.py

@@ -2,11 +2,20 @@
 """
 backend.py - ffmap-backend runner
 https://github.com/ffnord/ffmap-backend
+
+Erweiterte Version von Freifunk Pinneberg
+  - Graphiken aus RRD-Daten nur auf Anforderung erzeugen
+  - Verzeichnis für die RRD-Nodedb als Kommandozeilenparameter
+  - Statistikerzeugung korrigiert: Initialisierung und Befüllung
+    zu passenden Zeitpunkten
+
 """
 import argparse
+import configparser
 import json
 import os
 import sys
+import logging, logging.handlers
 from datetime import datetime
 
 import networkx as nx
@@ -15,11 +24,25 @@ from networkx.readwrite import json_graph
 from lib import graph, nodes
 from lib.alfred import Alfred
 from lib.batman import Batman
+from lib.rrddb import RRD
 from lib.nodelist import export_nodelist
+from lib.validate import validate_nodeinfos
 
 NODES_VERSION = 1
 GRAPH_VERSION = 1
 
+cfg = {
+    'cfgfile': '/etc/ffmap/ffmap-test.cfg',
+    'logfile': '/var/log/ffmap.log',
+    'loglevel': 5,
+    'dest_dir': '/var/lib/ffmap/mapdata',
+    'aliases': [],
+    'prune': 0,
+    'nodedb': '/var/lib/ffmap/nodedb',
+    'rrd_data': False,
+    'rrd_graphs': False,
+    'redis': False
+}
 
 def main(params):
     os.makedirs(params['dest_dir'], exist_ok=True)
@@ -54,11 +77,11 @@ def main(params):
                     'Unparseable value "{0}" in --mesh parameter.'.
                     format(value))
 
-    # read nodedb state from node.json
+    # read nodedb state from nodes.json
     try:
         with open(nodes_fn, 'r') as nodedb_handle:
             nodedb = json.load(nodedb_handle)
-    except IOError:
+    except (IOError, ValueError):
         nodedb = {'nodes': dict()}
 
     # flush nodedb if it uses the old format
@@ -75,18 +98,19 @@ def main(params):
 
     # integrate alfred nodeinfo
     for alfred in alfred_instances:
-        nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(),
+        nodeinfo = validate_nodeinfos(alfred.nodeinfo())
+        nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
                               now, assume_online=True)
 
     # integrate static aliases data
     for aliases in params['aliases']:
         with open(aliases, 'r') as f:
-            nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
+            nodeinfo = validate_nodeinfos(json.load(f))
+            nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
                                   now, assume_online=False)
 
+    # prepare statistics collection
     nodes.reset_statistics(nodedb['nodes'])
-    for alfred in alfred_instances:
-        nodes.import_statistics(nodedb['nodes'], alfred.statistics())
 
     # acquire gwl and visdata for each batman instance
     mesh_info = []
@@ -103,6 +127,10 @@ def main(params):
         nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
         nodes.mark_gateways(nodedb['nodes'], gwl)
 
+    # get alfred statistics
+    for alfred in alfred_instances:
+        nodes.import_statistics(nodedb['nodes'], alfred.statistics())
+
     # clear the nodedb from nodes that have not been online in $prune days
     if params['prune']:
         nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
@@ -116,6 +144,19 @@ def main(params):
     if params['vpn']:
         graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
 
+    def extract_tunnel(nodes):
+        macs = set()
+        for id, node in nodes.items():
+            try:
+                for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
+                    macs.add(mac)
+            except KeyError:
+                pass
+
+        return macs
+
+    graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
+
     batadv_graph = graph.merge_nodes(batadv_graph)
     batadv_graph = graph.to_undirected(batadv_graph)
 
@@ -132,26 +173,97 @@ def main(params):
     with open(nodelist_fn, 'w') as f:
         json.dump(export_nodelist(now, nodedb), f)
 
+    # optional rrd graphs (trigger with --rrd)
+    if params['rrd']:
+        if params['nodedb']:
+            rrd = RRD(params['nodedb'], os.path.join(params['dest_dir'], 'nodes'))
+        else:
+            script_directory = os.path.dirname(os.path.realpath(__file__))
+            rrd = RRD(os.path.join(script_directory, 'nodedb'),
+                      os.path.join(params['dest_dir'], 'nodes'))
+        rrd.update_database(nodedb['nodes'])
+        if params['img']:
+            rrd.update_images()
 
+def set_loglevel(nr):
+    """
+    Umsetzen der Nummer auf einen für "logging" passenden Wert
+    Die Nummer kann ein Wert zwischen 0 - kein Logging und 5 - Debug sein
+    """
+    level = (None, logging.CRITICAL, logging.ERROR, logging.WARNING,
+             logging.INFO, logging.DEBUG)
+    if nr > 5:
+        nr = 5
+    elif nr < 0:
+        nr = 0
+    return level[nr]
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
+
+    # get options from command line
+    parser = argparse.ArgumentParser(
+        description = "Collect data for ffmap: creates json files and "
+                      "optional rrd data and graphs")
 
     parser.add_argument('-a', '--aliases',
                         help='Read aliases from FILE',
                         nargs='+', default=[], metavar='FILE')
     parser.add_argument('-m', '--mesh',
                         default=['bat0'], nargs='+',
-                        help='Use given batman-adv mesh interface(s) (defaults'
+                        help='Use given batman-adv mesh interface(s) (defaults '
                              'to bat0); specify alfred unix socket like '
                              'bat0:/run/alfred0.sock.')
     parser.add_argument('-d', '--dest-dir', action='store',
                         help='Write output to destination directory',
-                        required=True)
+                        required=False)
+    parser.add_argument('-c', '--config', action='store', metavar='FILE',
+                        help='read configuration from FILE')
     parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
                         help='Assume MAC addresses are part of vpn')
     parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
-                        help='forget nodes offline for at least DAYS')
+                        help='Forget nodes offline for at least DAYS')
+    parser.add_argument('-r', '--with-rrd', dest='rrd', action='store_true',
+                        default=False,
+                        help='Enable the collection of RRD data')
+    parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
+                        help='Directory for node RRD data files')
+    parser.add_argument('-i', '--with-img', dest='img', action='store_true',
+                        default=False,
+                        help='Enable the rendering of RRD graphs (cpu '
+                             'intensive)')
 
     options = vars(parser.parse_args())
+    if options['config']:
+        cfg['cfgfile'] = options['config']
+
+    config = configparser.ConfigParser(cfg)
+    if config.read(cfg['cfgfile']):
+        if not options['nodedb']:
+            options['nodedb'] = config.get('rrd', 'nodedb')
+        if not options['dest_dir']:
+            options['dest_dir'] = config.get('global', 'dest_dir')
+        if not options['rrd']:
+            options['rrd'] = config.getboolean('rrd', 'enabled')
+        if not options['img']:
+            options['img'] = config.getboolean('rrd', 'graphs')
+        cfg['logfile'] = config.get('global', 'logfile')
+        cfg['loglevel'] = config.getint('global', 'loglevel')
+
+    # At this point global configuration is available. Time to enable logging
+    # Logging is handled by the operating system, so use WatchedFileHandler
+    handler = logging.handlers.WatchedFileHandler(cfg['logfile'])
+    handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s',
+                                           datefmt='%Y-%m-%d %H:%M:%S'))
+    log = logging.getLogger()
+    log.addHandler(handler)
+    loglevel = set_loglevel(cfg['loglevel'])
+    if loglevel:
+        log.setLevel(loglevel)
+    else:
+        log.disabled = True
+
+    log.info("%s started" % sys.argv[0])
+    if os.path.isfile(cfg['cfgfile']):
+        log.info("using configuration from '%s'" % cfg['cfgfile'])
     main(options)
+    log.info("%s finished" % sys.argv[0])

ffmap-d3.jq

@@ -0,0 +1,52 @@
+{
+    "meta": {
+        "timestamp": $nodes.timestamp
+    },
+    "nodes": (
+        $graph.batadv.nodes
+        | map(
+            if has("node_id") and .node_id
+            then (
+                $nodes.nodes[.node_id] as $node
+                | {
+                    "id": .id,
+                    "uptime": $node.statistics.uptime,
+                    "flags": ($node.flags + {"client": false}),
+                    "name": $node.nodeinfo.hostname,
+                    "clientcount": (if $node.statistics.clients >= 0 then $node.statistics.clients else 0 end),
+                    "hardware": $node.nodeinfo.hardware.model,
+                    "firmware": $node.nodeinfo.software.firmware.release,
+                    "geo": (if $node.nodeinfo.location then [$node.nodeinfo.location.latitude, $node.nodeinfo.location.longitude] else null end),
+                    #"lastseen": $node.lastseen,
+                    "network": $node.nodeinfo.network
+                }
+            )
+            else
+                {
+                    "flags": {},
+                    "id": .id,
+                    "geo": null,
+                    "clientcount": 0
+                }
+            end
+        )
+    ),
+    "links": (
+        $graph.batadv.links
+        | map(
+            $graph.batadv.nodes[.source].node_id as $source_id
+            | $graph.batadv.nodes[.target].node_id as $target_id
+            | select(
+                $source_id and $target_id and
+                ($nodes.nodes | (has($source_id) and has($target_id)))
+            )
+            | {
+                "target": .target,
+                "source": .source,
+                "quality": "\(.tq), \(.tq)",
+                "id": ($source_id + "-" + $target_id),
+                "type": (if .vpn then "vpn" else null end)
+            }
+        )
+    )
+}

ffmap-modify.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Knotendaten manuell ändern
+Das ist z.B. für ausgeschaltete Knoten interessant, die nur 
+temporär nicht zur Verfügung stehen. Die können ausgeblendet
+werden.
+Das ist besser als löschen, weil so die Statistik nicht
+verschwindet
+
+Änderungsprotokoll
+==================
+
+Version  Datum       Änderung(en)                                           von 
+-------- ----------- ------------------------------------------------------ ----
+1.0      2017-08-03  Programm in das ffmap-backend Projekt integriert       tho
+
+"""
+
+import argparse
+import configparser
+import json
+import os
+import sys
+import glob
+
+# Einstellungen werden in folgender Reihenfolge verarbeitet
+# später gesetzte Werte überschreiben frühere
+#  1. im Programm hart codiert
+#  2. aus der zentralen Konfigurationsdatei gelesen  
+#  3. als Kommandozeilenoptionen angegeben
+cfg = {
+    'cfgfile': '/etc/ffmap/ffmap.cfg',
+    'logfile': '/var/log/ffmap.log',
+    'loglevel': 2,
+    'dest_dir': '/var/lib/ffmap/mapdata',
+    'nodedb': '/var/lib/ffmap/nodedb',
+    'imgpath': '/var/www/meshviewer/stats/img'
+}
+
+roles_defined = ('node', 'temp', 'mobile', 'offloader', 'service', 'test', 'gate', 'plan', 'hidden')
+
+def main(cfg):
+
+    # Pfade zu den beteiligten Dateien
+    nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
+    nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
+
+    # 1. Knotendaten (NodeDB)
+    # 1.1 Daten laden
+    try:
+        with open(nodes_fn, 'r') as nodedb_handle:
+            nodedb = json.load(nodedb_handle)
+    except IOError:
+        print("Error reading nodedb file %s" % nodes_fn)
+        nodedb = {'nodes': dict()}
+    # 1.2 Knoten bearbeiten
+    changed = False
+    for n in cfg['nodeid']:
+        if n in nodedb['nodes']:
+            print("Modify %s in nodedb" % n)
+            if 'role' in cfg and cfg['role'] in roles_defined:
+                try:
+                    oldrole = nodedb['nodes'][n]['nodeinfo']['system']['role']
+                except KeyError:
+                    oldrole = '<unset>'
+                print("  - change role from '%s' to '%s'" % (oldrole, cfg['role']))
+                nodedb['nodes'][n]['nodeinfo']['system']['role'] = cfg['role']
+                changed = True
+            if 'location' in cfg:
+                print("  - remove location")
+                # del nodedb['nodes'][n]['nodeinfo']['location']
+                changed = True
+        else:
+            print("Node %s not found in nodedb" % n)        
+    # 1.3 Geänderte Daten zurückschreiben
+    if changed:
+        try:
+            with open(nodes_fn, 'w') as nodedb_handle:
+                json.dump(nodedb, nodedb_handle)
+        except IOError:
+            print("Error writing nodedb file %s" % nodes_fn)
+
+    # 2. Knotenliste (NodeList)
+    try:
+        with open(nodelist_fn, 'r') as nodelist_handle:
+            nodelist = json.load(nodelist_handle)
+    except IOError:
+        print("Error reading nodelist file %s" % nodelist_fn)
+        nodelist = {'nodelist': dict()}
+    # 2.1 Knoten bearbeiten
+    changed = False
+    ixlist = []
+    for nodeid in cfg['nodeid']:
+        found = False
+        for ix, node in enumerate(nodelist['nodes']):
+            if node['id'] == nodeid:
+                found = True
+                break
+        if found:
+            print("Modify %s in nodelist" % nodeid)
+            if 'role' in cfg and cfg['role'] in roles_defined:
+                try:
+                    oldrole = nodelist['nodes'][ix]['role']
+                except KeyError:
+                    oldrole = '<unset>'
+                print("  - change role from '%s' to '%s'" % (oldrole, cfg['role']))
+                nodelist['nodes'][ix]['role'] = cfg['role']
+            if 'location' in cfg:
+                print("  - remove location")
+                try:
+                    #del nodelist['nodes'][ix]['position']
+                    pass
+                except KeyError:
+                    pass
+                changed = True
+        else:
+            print ("Node %s not found in nodelist" % nodeid)
+    # 2.3 Geänderte Daten zurückschreiben
+    if changed:
+        try:
+            with open(nodelist_fn, 'w') as nodelist_handle:
+                json.dump(nodelist, nodelist_handle)
+        except IOError:
+            print("Error writing nodelist file %s" % nodelist_fn)
+
+if __name__ == "__main__":
+
+    # Optionen von der Kommandozeile lesen
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-c', '--config', action='store',
+                        help='Configuration file')
+
+    parser.add_argument('-d', '--dest-dir', action='store',
+                        help='Directory with JSON data files',
+                        required=False)
+
+    parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
+                        nargs='+', required=True,
+                        help='Node id to modify')
+
+    parser.add_argument('-l', '--location', action='store_true',
+                        help='Clear location information (hides node)',
+                        required=False)
+
+    parser.add_argument('-r', '--role', action='store',
+                        help='Set new role',
+                        required=False)
+
+    # TODO
+    # Optionen was genau gemacht werden soll
+    # -p Position entfernen, Knoten wird nicht mehr angezeigt
+    # -r <rolle> Rolle einstellen
+
+    options = vars(parser.parse_args())
+
+
+    # Konfigurationsdatei einlesen
+    if options['config']:
+        cfg['cfgfile'] = options['config']
+    config = configparser.ConfigParser(cfg)
+    # config.read liefert eine Liste der geparsten Dateien
+    # zurück. Wenn sie leer ist, war z.B. die Datei nicht
+    # vorhanden
+    if config.read(cfg['cfgfile']):
+        if 'global' in config:
+            cfg['logfile'] = config['global']['logfile']
+            cfg['loglevel'] = config['global']['loglevel']
+            cfg['dest_dir'] = config['global']['dest_dir']
+    else:
+        print('Config file %s not parsed' % cfg['cfgfile'])
+
+    # Optionen von der Kommandozeile haben höchste Priorität
+    cfg['nodeid'] = options['nodeid']
+    if options['dest_dir']:
+        cfg['dest_dir'] = options['dest_dir']
+    if options['location']:
+        cfg['location'] = True
+    if options['role']:
+        cfg['role'] = options['role']
+
+    # Alles initialisiert, auf geht's
+    main(cfg)

ffmap-remove.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Lösche einen Knoten manuell aus dem Backend:
+- JSON
+  - NodeDB
+  - NodeList
+  - Graph
+- RRD-Dateien
+- Bilder vom Webserver
+
+Änderungsprotokoll
+==================
+
+Version  Datum       Änderung(en)                                           von 
+-------- ----------- ------------------------------------------------------ ----
+1.0      2017-01-06  Programm in das ffmap-backend Projekt integriert       tho
+
+"""
+
+import argparse
+import configparser
+import json
+import os
+import sys
+import glob
+
+# Einstellungen werden in folgender Reihenfolge verarbeitet
+# später gesetzte Werte überschreiben frühere
+#  1. im Programm hart codiert
+#  2. aus der zentralen Konfigurationsdatei gelesen  
+#  3. als Kommandozeilenoptionen angegeben
+cfg = {
+    'cfgfile': '/etc/ffmap/ffmap.cfg',
+    'logfile': '/var/log/ffmap.log',
+    'loglevel': 2,
+    'dest_dir': '/var/lib/ffmap/mapdata',
+    'nodedb': '/var/lib/ffmap/nodedb',
+    'imgpath': '/var/www/meshviewer/stats/img'
+}
+
+def main(cfg):
+
+    # Pfade zu den beteiligten Dateien
+    nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
+    graph_fn = os.path.join(cfg['dest_dir'], 'graph.json')
+    nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
+
+    # 1. Knotendaten (NodeDB) bereinigen
+    # 1.1 Daten laden
+    try:
+        with open(nodes_fn, 'r') as nodedb_handle:
+            nodedb = json.load(nodedb_handle)
+    except IOError:
+        print("Error reading nodedb file %s" % nodes_fn)
+        nodedb = {'nodes': dict()}
+    # 1.2 Knoten entfernen
+    changed = False
+    for n in cfg['nodeid']:
+        if n in nodedb['nodes']:
+            print("Remove %s from nodedb" % n)
+            del nodedb['nodes'][n]
+            changed = True
+        else:
+            print("Node %s not found in nodedb" % n)        
+    # 1.3 Geänderte Daten zurückschreiben
+    if changed:
+        try:
+            with open(nodes_fn, 'w') as nodedb_handle:
+                json.dump(nodedb, nodedb_handle)
+        except IOError:
+            print("Error writing nodedb file %s" % nodes_fn)
+
+    # 2. Knotenliste (NodeList) bereinigen
+    try:
+        with open(nodelist_fn, 'r') as nodelist_handle:
+            nodelist = json.load(nodelist_handle)
+    except IOError:
+        print("Error reading nodelist file %s" % nodelist_fn)
+        nodelist = {'nodelist': dict()}
+    # 2.1 Knoten entfernen
+    changed = False
+    ixlist = []
+    for nodeid in cfg['nodeid']:
+        found = False
+        for ix, node in enumerate(nodelist['nodes']):
+            if node['id'] == nodeid:
+                found = True
+                break
+        if found:
+            print("Remove %s from nodelist" % nodeid)
+            del nodelist['nodes'][ix]
+            changed = True
+        else:
+            print ("Node %s not found in nodelist" % nodeid)
+    # 2.3 Geänderte Daten zurückschreiben
+    if changed:
+        try:
+            with open(nodelist_fn, 'w') as nodelist_handle:
+                json.dump(nodelist, nodelist_handle)
+        except IOError:
+            print("Error writing nodelist file %s" % nodelist_fn)
+
+    # 3. Graph (NodeGraph) bereinigen
+    # 3.1 Graph laden
+    try:
+        with open(graph_fn, 'r') as graph_handle:
+            graph = json.load(graph_handle)
+    except IOError:
+        print("Error reading graph file %s" % graph_fn)
+        graph = {'graph': dict()}
+    # 3.2 Finde Knoten und Links
+    # Nodes und Links gehören zusammen   
+    changed = False
+    for nodeid in cfg['nodeid']:
+        found = False
+        for ixn, node in enumerate(graph["batadv"]["nodes"]):
+            # Es kann nodes ohne "node_id" geben
+            try:
+                if node["node_id"] == nodeid:
+                    found = True
+                    break
+            except KeyError:
+                pass
+        if found:
+            print("Found %s in graph nodes at index %d" % (nodeid, ixn))
+            del graph["batadv"]["nodes"][ixn]
+            # Suche Link source oder target dem gefundenen Index entsprechen
+            ixlist = []
+            for ixg, link in enumerate(graph["batadv"]["links"]):
+                if link["source"] == ixn:
+                    print("Found source link at index %d" % ixg)
+                    print("  -> %s" % graph["batadv"]["nodes"][link["target"]])
+                    ixlist.append(ixg)
+                if link["target"] == ixn:
+                    print("Found target link at index %d" % ixg)
+                    print("  -> %s" % graph["batadv"]["nodes"][link["source"]])
+                    ixlist.append(ixg)
+            for ix in ixlist:
+                del graph["batadv"]["nodes"][ix]
+            changed = True
+        else:
+            print("Node %s not found in graph nodes" % nodeid)
+    # 3.3 Zurückschreiben der geänderten Daten
+    if changed:
+        try:
+            with open(graph_fn, 'w') as graph_handle:
+                json.dump(graph, graph_handle)
+        except IOError:
+            print("Error writing graph file %s" % graph_fn)
+
+    # 4. Entferne RRD-Dateien
+    for nodeid in cfg['nodeid']:
+        rrdfile = os.path.join(cfg['nodedb'], nodeid+'.rrd')
+        if os.path.isfile(rrdfile):
+            print("Removing RRD database file %s" % os.path.basename(rrdfile))
+        else:
+            print("RRD database file %s not found" % os.path.basename(rrdfile))
+        try:
+            os.remove(rrdfile)
+        except OSError:
+            pass
+
+    # 5. Entferne Bilder vom Webserver
+    count_deleted = 0
+    for nodeid in cfg['nodeid']:
+        for imagefile in glob.glob(os.path.join(cfg['imgpath'], nodeid+'_*.png')):
+            print("Removing stats image %s" % os.path.basename(imagefile))
+            try:
+                os.remove(imagefile)
+                count_deleted += 1
+            except OSError:
+                pass
+    if count_deleted == 0:
+        print("No stats images found in %s" % cfg['imgpath'])
+
+if __name__ == "__main__":
+
+    # Optionen von der Kommandozeile lesen
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-c', '--config', action='store',
+                        help='Configuration file')
+
+    parser.add_argument('-d', '--dest-dir', action='store',
+                        help='Directory with JSON data files',
+                        required=False)
+
+    parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
+                        nargs='+', required=True,
+                        help='Node id  to remove')
+
+    parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
+                        help='Directory for node RRD data files')
+
+    options = vars(parser.parse_args())
+
+
+    # Konfigurationsdatei einlesen
+    if options['config']:
+        cfg['cfgfile'] = options['config']
+    config = configparser.ConfigParser(cfg)
+    # config.read liefert eine Liste der geparsten Dateien
+    # zurück. Wenn sie leer ist, war z.B. die Datei nicht
+    # vorhanden
+    if config.read(cfg['cfgfile']):
+        if 'global' in config:
+            cfg['logfile'] = config['global']['logfile']
+            cfg['loglevel'] = config['global']['loglevel']
+            cfg['dest_dir'] = config['global']['dest_dir']
+        if 'rrd' in config:
+            cfg['nodedb'] = config['rrd']['nodedb']
+    else:
+        print('Config file %s not parsed' % cfg['cfgfile'])
+
+    # Optionen von der Kommandozeile haben höchste Priorität
+    cfg['nodeid'] = options['nodeid']
+    if options['dest_dir']:
+        cfg['dest_dir'] = options['dest_dir']
+    if options['nodedb']:
+        cfg['nodedb'] = options['nodedb']
+
+    # Alles initialisiert, auf geht's
+    main(cfg)

lib/GateRRD.py

@@ -0,0 +1,74 @@
+"""
+RRD for gateways
+"""
+
+import os
+import subprocess
+
+from lib.RRD import DS, RRA, RRD
+
+
+class GateRRD(RRD):
+    ds_list = [
+        DS('upstate', 'GAUGE', 120, 0, 1),
+        DS('clients', 'GAUGE', 120, 0, float('NaN')),
+        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
+        DS('leases', 'GAUGE', 120, 0, float('NaN')),       
+    ]
+    rra_list = [
+        RRA('AVERAGE', 0.5, 1, 120),    #  2 hours of  1 minute samples
+        RRA('AVERAGE', 0.5, 5, 1440),   #  5 days  of  5 minute samples
+        RRA('AVERAGE', 0.5, 15, 672),   #  7 days  of 15 minute samples
+        RRA('AVERAGE', 0.5, 60, 720),   # 30 days  of  1 hour   samples
+        RRA('AVERAGE', 0.5, 720, 730),  #  1 year  of 12 hour   samples
+    ]
+
+    def __init__(self, filename, node=None):
+        """
+        Create a new RRD for a given node.
+
+        If the RRD isn't supposed to be updated, the node can be omitted.
+        """
+        self.node = node
+        super().__init__(filename)
+        self.ensure_sanity(self.ds_list, self.rra_list, step=60)
+
+    @property
+    def imagename(self):
+        return "{basename}.png".format(
+            basename=os.path.basename(self.filename).rsplit('.', 2)[0])
+
+    # TODO: fix this, python does not support function overloading
+    def update(self):
+        values  = {
+            'upstate': int(self.node['flags']['online']),
+            'clients': float(self.node['statistics']['clients']),
+        }
+        if 'loadavg' in self.node['statistics']:
+            values['loadavg'] = float(self.node['statistics'].get('loadavg', 0))
+        # Gateways can send the peer count. We use the clients field to store data 
+        if 'peers' in self.node['statistics']:
+            values['clients'] = self.node['statistics']['peers']
+        if 'leases' in self.node['statistics']:
+            values['leases'] = self.node['statistics']['leases']
+        super().update(values)
+
+    def graph(self, directory, timeframe):
+        """
+        Create a graph in the given directory. The file will be named
+        basename.png if the RRD file is named basename.rrd
+        """
+        args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
+                '-s', '-' + timeframe,
+                '-w', '800',
+                '-h', '400',
+                '-l', '0',
+                '-y', '1:1',
+                'DEF:clients=' + self.filename + ':clients:AVERAGE',
+                'VDEF:maxc=clients,MAXIMUM',
+                'CDEF:c=0,clients,ADDNAN',
+                'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
+                'AREA:c#0F0:up\\l',
+                'AREA:d#F00:down\\l',
+                'LINE1:c#00F:clients connected\\l']
+        subprocess.check_output(args)

lib/GlobalRRD.py

@@ -0,0 +1,40 @@
+import os
+import subprocess
+
+from lib.RRD import DS, RRA, RRD
+
+
+class GlobalRRD(RRD):
+    ds_list = [
+        # Number of nodes available
+        DS('nodes', 'GAUGE', 120, 0, float('NaN')),
+        # Number of client available
+        DS('clients', 'GAUGE', 120, 0, float('NaN')),
+    ]
+    rra_list = [
+        # 2 hours of 1 minute samples
+        RRA('AVERAGE', 0.5, 1, 120),
+        # 31 days  of 1 hour samples
+        RRA('AVERAGE', 0.5, 60, 744),
+        # ~5 years of 1 day samples
+        RRA('AVERAGE', 0.5, 1440, 1780),
+    ]
+
+    def __init__(self, directory):
+        super().__init__(os.path.join(directory, "nodes.rrd"))
+        self.ensure_sanity(self.ds_list, self.rra_list, step=60)
+
+    # TODO: fix this, python does not support function overloading
+    def update(self, node_count, client_count):
+        super().update({'nodes': node_count, 'clients': client_count})
+
+    def graph(self, filename, timeframe):
+        args = ["rrdtool", 'graph', filename,
+                '-s', '-' + timeframe,
+                '-w', '800',
+                '-h' '400',
+                'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
+                'LINE1:nodes#F00:nodes\\l',
+                'DEF:clients=' + self.filename + ':clients:AVERAGE',
+                'LINE2:clients#00F:clients']
+        subprocess.check_output(args)

lib/NodeRRD.py

@@ -0,0 +1,67 @@
+"""
+RRD for nodes
+"""
+
+import os
+import subprocess
+
+from lib.RRD import DS, RRA, RRD
+
+
+class NodeRRD(RRD):
+    ds_list = [
+        DS('upstate', 'GAUGE', 120, 0, 1),
+        DS('clients', 'GAUGE', 120, 0, float('NaN')),
+        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
+    ]
+    rra_list = [
+        RRA('AVERAGE', 0.5, 1, 120),    #  2 hours of  1 minute samples
+        RRA('AVERAGE', 0.5, 5, 1440),   #  5 days  of  5 minute samples
+        RRA('AVERAGE', 0.5, 60, 720),   # 30 days  of  1 hour   samples
+        RRA('AVERAGE', 0.5, 720, 730),  #  1 year  of 12 hour   samples
+    ]
+
+    def __init__(self, filename, node=None):
+        """
+        Create a new RRD for a given node.
+
+        If the RRD isn't supposed to be updated, the node can be omitted.
+        """
+        self.node = node
+        super().__init__(filename)
+        self.ensure_sanity(self.ds_list, self.rra_list, step=60)
+
+    @property
+    def imagename(self):
+        return "{basename}.png".format(
+            basename=os.path.basename(self.filename).rsplit('.', 2)[0])
+
+    # TODO: fix this, python does not support function overloading
+    def update(self):
+        values  = {
+            'upstate': int(self.node['flags']['online']),
+            'clients': self.node['statistics']['clients']
+        }
+        if 'loadavg' in self.node['statistics']:
+            values['loadavg'] = float(self.node['statistics']['loadavg'])
+        super().update(values)
+
+    def graph(self, directory, timeframe):
+        """
+        Create a graph in the given directory. The file will be named
+        basename.png if the RRD file is named basename.rrd
+        """
+        args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
+                '-s', '-' + timeframe,
+                '-w', '800',
+                '-h', '400',
+                '-l', '0',
+                '-y', '1:1',
+                'DEF:clients=' + self.filename + ':clients:AVERAGE',
+                'VDEF:maxc=clients,MAXIMUM',
+                'CDEF:c=0,clients,ADDNAN',
+                'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
+                'AREA:c#0F0:up\\l',
+                'AREA:d#F00:down\\l',
+                'LINE1:c#00F:clients connected\\l']
+        subprocess.check_output(args)

lib/RRD.py

@@ -0,0 +1,346 @@
+import subprocess
+import re
+import os
+from operator import xor, eq
+from functools import reduce
+from itertools import starmap
+import math
+
+
+class RRDIncompatibleException(Exception):
+    """
+    Is raised when an RRD doesn't have the desired definition and cannot be
+    upgraded to it.
+    """
+    pass
+
+
+class RRDOutdatedException(Exception):
+    """
+    Is raised when an RRD doesn't have the desired definition, but can be
+    upgraded to it.
+    """
+    pass
+
+if not hasattr(__builtins__, "FileNotFoundError"):
+    class FileNotFoundError(Exception):
+        pass
+
+
+class RRD(object):
+    """
+    An RRD is a Round Robin Database, a database which forgets old data and
+    aggregates multiple records into new ones.
+
+    It contains multiple Data Sources (DS) which can be thought of as columns,
+    and Round Robin Archives (RRA) which can be thought of as tables with the
+    DS as columns and time-dependant rows.
+    """
+
+    # rra[2].cdp_prep[0].value = 1,8583033333e+03
+    _info_regex = re.compile("""
+            (?P<section>[a-z_]+)
+            \[ (?P<key>[a-zA-Z0-9_]+) \]
+            \.
+        |
+            (?P<name>[a-z_]+)
+            \s*=\s*
+            "? (?P<value>.*?) "?
+        $""", re.X)
+    _cached_info = None
+
+    def _exec_rrdtool(self, cmd, *args, **kwargs):
+        pargs = ["rrdtool", cmd, self.filename]
+        for k, v in kwargs.items():
+            pargs.extend(["--" + k, str(v)])
+        pargs.extend(args)
+        subprocess.check_output(pargs)
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def ensure_sanity(self, ds_list, rra_list, **kwargs):
+        """
+        Create or upgrade the RRD file if necessary to contain all DS in
+        ds_list. If it needs to be created, the RRAs in rra_list and any kwargs
+        will be used for creation. Note that RRAs and options of an existing
+        database are NOT modified!
+        """
+        try:
+            self.check_sanity(ds_list)
+        except FileNotFoundError:
+            self.create(ds_list, rra_list, **kwargs)
+        except RRDOutdatedException:
+            self.upgrade(ds_list)
+
+    def check_sanity(self, ds_list=()):
+        """
+        Check if the RRD file exists and contains (at least) the DS listed in
+        ds_list.
+        """
+        if not os.path.exists(self.filename):
+            raise FileNotFoundError(self.filename)
+        info = self.info()
+        if set(ds_list) - set(info['ds'].values()) != set():
+            for ds in ds_list:
+                if ds.name in info['ds'] and\
+                   ds.type != info['ds'][ds.name].type:
+                    raise RRDIncompatibleException(
+                        "{} is {} but should be {}".format(
+                            ds.name, ds.type, info['ds'][ds.name].type))
+            else:
+                raise RRDOutdatedException()
+
+    def upgrade(self, dss):
+        """
+        Upgrade the DS definitions (!) of this RRD.
+        (To update its values, use update())
+
+        The list dss contains DSS objects to be updated or added. The
+        parameters of a DS can be changed, but not its type. New DS are always
+        added at the end in the order of their appearance in the list.
+
+        This is done internally via an rrdtool dump -> rrdtool restore and
+        modifying the dump on the fly.
+        """
+        info = self.info()
+        new_ds = list(info['ds'].values())
+        new_ds.sort(key=lambda ds: ds.index)
+        for ds in dss:
+            if ds.name in info['ds']:
+                old_ds = info['ds'][ds.name]
+                if info['ds'][ds.name].type != ds.type:
+                    raise RuntimeError(
+                        "Cannot convert existing DS '{}'"
+                        "from type '{}' to '{}'".format(
+                            ds.name, old_ds.type, ds.type))
+                ds.index = old_ds.index
+                new_ds[ds.index] = ds
+            else:
+                ds.index = len(new_ds)
+                new_ds.append(ds)
+        added_ds_num = len(new_ds) - len(info['ds'])
+
+        dump = subprocess.Popen(
+            ["rrdtool", "dump", self.filename],
+            stdout=subprocess.PIPE)
+
+        restore = subprocess.Popen(
+            ["rrdtool", "restore", "-", self.filename + ".new"],
+            stdin=subprocess.PIPE)
+        echo = True
+        ds_definitions = True
+        for line in dump.stdout:
+            if ds_definitions and b'<ds>' in line:
+                echo = False
+            if b'<!-- Round Robin Archives -->' in line:
+                ds_definitions = False
+                for ds in new_ds:
+                    restore.stdin.write(bytes("""
+                        <ds>
+                           <name> %s </name>
+                           <type> %s </type>
+                           <minimal_heartbeat>%i</minimal_heartbeat>
+                           <min>%s</min>
+                           <max>%s</max>
+
+                           <!-- PDP Status -->
+                           <last_ds>%s</last_ds>
+                           <value>%s</value>
+                           <unknown_sec> %i </unknown_sec>
+                        </ds>
+                        """ % (ds.name,
+                               ds.type,
+                               ds.args[0],
+                               ds.args[1],
+                               ds.args[2],
+                               ds.last_ds,
+                               ds.value,
+                               ds.unknown_sec), "utf-8"))
+
+            if b'</cdp_prep>' in line:
+                restore.stdin.write(added_ds_num * b"""
+                        <ds>
+                        <primary_value> NaN </primary_value>
+                        <secondary_value> NaN </secondary_value>
+                        <value> NaN </value>
+                        <unknown_datapoints> 0 </unknown_datapoints>
+                        </ds>
+                """)
+
+            # echoing of input line
+            if echo:
+                restore.stdin.write(
+                    line.replace(
+                        b'</row>',
+                        (added_ds_num * b'<v>NaN</v>') + b'</row>'
+                    )
+                )
+
+            if ds_definitions and b'</ds>' in line:
+                echo = True
+        dump.stdout.close()
+        restore.stdin.close()
+        dump.wait()
+        restore.wait()
+
+        os.rename(self.filename + ".new", self.filename)
+        self._cached_info = None
+
+    def create(self, ds_list, rra_list, **kwargs):
+        """
+        Create a new RRD file with the specified list of RRAs and DSs.
+
+        Any kwargs are passed as --key=value to rrdtool create.
+        """
+        self._exec_rrdtool(
+            "create",
+            *map(str, rra_list + ds_list),
+            **kwargs
+        )
+        self._cached_info = None
+
+    def update(self, V):
+        """
+        Update the RRD with new values V.
+
+        V can be either list or dict:
+        * If it is a dict, its keys must be DS names in the RRD and it is
+          ensured that the correct DS are updated with the correct values, by
+          passing a "template" to rrdtool update (see man rrdupdate).
+        * If it is a list, no template is generated and the order of the
+          values in V must be the same as that of the DS in the RRD.
+        """
+        try:
+            args = ['N:' + ':'.join(map(str, V.values()))]
+            kwargs = {'template': ':'.join(V.keys())}
+        except AttributeError:
+            args = ['N:' + ':'.join(map(str, V))]
+            kwargs = {}
+        self._exec_rrdtool("update", *args, **kwargs)
+        self._cached_info = None
+
+    def info(self):
+        """
+        Return a dictionary with information about the RRD.
+
+        See `man rrdinfo` for more details.
+        """
+        if self._cached_info:
+            return self._cached_info
+        env = os.environ.copy()
+        env["LC_ALL"] = "C"
+        proc = subprocess.Popen(
+            ["rrdtool", "info", self.filename],
+            stdout=subprocess.PIPE,
+            env=env
+        )
+        out, err = proc.communicate()
+        out = out.decode()
+        info = {}
+        for line in out.splitlines():
+            base = info
+            for match in self._info_regex.finditer(line):
+                section, key, name, value = match.group(
+                    "section", "key", "name", "value")
+                if section and key:
+                    try:
+                        key = int(key)
+                    except ValueError:
+                        pass
+                    if section not in base:
+                        base[section] = {}
+                    if key not in base[section]:
+                        base[section][key] = {}
+                    base = base[section][key]
+                if name and value:
+                    try:
+                        base[name] = int(value)
+                    except ValueError:
+                        try:
+                            base[name] = float(value)
+                        except:
+                            base[name] = value
+        dss = {}
+        for name, ds in info['ds'].items():
+            ds_obj = DS(name, ds['type'], ds['minimal_heartbeat'],
+                        ds['min'], ds['max'])
+            ds_obj.index = ds['index']
+            ds_obj.last_ds = ds['last_ds']
+            ds_obj.value = ds['value']
+            ds_obj.unknown_sec = ds['unknown_sec']
+            dss[name] = ds_obj
+        info['ds'] = dss
+        rras = []
+        for rra in info['rra'].values():
+            rras.append(RRA(rra['cf'], rra['xff'],
+                            rra['pdp_per_row'], rra['rows']))
+        info['rra'] = rras
+        self._cached_info = info
+        return info
+
+
+class DS(object):
+    """
+    DS stands for Data Source and represents one line of data points in a Round
+    Robin Database (RRD).
+    """
+    name = None
+    type = None
+    args = []
+    index = -1
+    last_ds = 'U'
+    value = 0
+    unknown_sec = 0
+
+    def __init__(self, name, dst, *args):
+        self.name = name
+        self.type = dst
+        self.args = args
+
+    def __str__(self):
+        return "DS:%s:%s:%s" % (
+            self.name,
+            self.type,
+            ":".join(map(str, self._nan_to_u_args()))
+        )
+
+    def __repr__(self):
+        return "%s(%r, %r, %s)" % (
+            self.__class__.__name__,
+            self.name,
+            self.type,
+            ", ".join(map(repr, self.args))
+        )
+
+    def __eq__(self, other):
+        return all(starmap(eq, zip(self.compare_keys(), other.compare_keys())))
+
+    def __hash__(self):
+        return reduce(xor, map(hash, self.compare_keys()))
+
+    def _nan_to_u_args(self):
+        return tuple(
+            'U' if type(arg) is float and math.isnan(arg)
+            else arg
+            for arg in self.args
+        )
+
+    def compare_keys(self):
+        return self.name, self.type, self._nan_to_u_args()
+
+
+class RRA(object):
+    def __init__(self, cf, *args):
+        self.cf = cf
+        self.args = args
+
+    def __str__(self):
+        return "RRA:%s:%s" % (self.cf, ":".join(map(str, self.args)))
+
+    def __repr__(self):
+        return "%s(%r, %s)" % (
+            self.__class__.__name__,
+            self.cf,
+            ", ".join(map(repr, self.args))
+        )

lib/alfred.py

@@ -20,7 +20,11 @@ class Alfred(object):
         if self.unix_sock:
             cmd.extend(['-s', self.unix_sock])
 
-        output = subprocess.check_output(cmd)
+        # There should not be any warnings which would be sent by cron
+        # every minute. Therefore suppress error output of called program
+        FNULL = open(os.devnull, 'w')
+        output = subprocess.check_output(cmd, stderr=FNULL)
+        FNULL.close()
         return json.loads(output.decode("utf-8")).values()
 
     def nodeinfo(self):

lib/batman.py

@@ -3,7 +3,6 @@ import json
 import os
 import re
 
-
 class Batman(object):
     """
     Bindings for B.A.T.M.A.N. Advanced
@@ -22,7 +21,7 @@ class Batman(object):
         self.environ = env
 
         # compile regular expressions only once on startup
-        self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})')
+        self.mac_addr_pattern = re.compile(r'(([a-f0-9]{2}:){5}[a-f0-9]{2})')
 
     def vis_data(self):
         return self.vis_data_batadv_vis()
@@ -52,10 +51,10 @@ class Batman(object):
 
     def gateway_list(self):
         """
-        Parse "batctl -m <mesh_interface> gwl -n"
+        Parse "batctl meshif <mesh_interface> gwl -n"
         into an array of dictionaries.
         """
-        cmd = ['batctl', '-m', self.mesh_interface, 'gwl', '-n']
+        cmd = ['batctl', 'meshif', self.mesh_interface, 'gwl', '-n']
         if os.geteuid() > 0:
             cmd.insert(0, 'sudo')
         output = subprocess.check_output(cmd, env=self.environ)
@@ -81,10 +80,10 @@ class Batman(object):
 
     def gateway_mode(self):
         """
-        Parse "batctl -m <mesh_interface> gw"
+        Parse "batctl meshif <mesh_interface> gw"
         return: tuple mode, bandwidth, if mode != server then bandwidth is None
         """
-        cmd = ['batctl', '-m', self.mesh_interface, 'gw']
+        cmd = ['batctl', 'meshif', self.mesh_interface, 'gw']
         if os.geteuid() > 0:
             cmd.insert(0, 'sudo')
         output = subprocess.check_output(cmd, env=self.environ)

lib/nodelist.py

@@ -13,12 +13,20 @@ def export_nodelist(now, nodedb):
         node_out["status"] = dict()
         node_out["status"]["online"] = node["flags"]["online"]
 
+        if "firstseen" in node:
+            node_out["status"]["firstcontact"] = node["firstseen"]
+
         if "lastseen" in node:
             node_out["status"]["lastcontact"] = node["lastseen"]
 
         if "clients" in node["statistics"]:
             node_out["status"]["clients"] = node["statistics"]["clients"]
 
+        if "role" in node["nodeinfo"]["system"]:
+            node_out["role"] = node["nodeinfo"]["system"]["role"]
+        else:
+            node_out["role"] = "node"
+
         nodelist.append(node_out)
 
     return {"version": "1.0.1", "nodes": nodelist, "updated_at": now.isoformat()}

lib/nodes.py

@@ -11,6 +11,15 @@ def build_mac_table(nodes):
                 macs[mac] = node_id
         except KeyError:
             pass
+
+        try:
+            for upper_if in node['nodeinfo']['network']['mesh'].values():
+                for lower_if in upper_if['interfaces'].values():
+                    for mac in lower_if:
+                        macs[mac] = node_id
+        except KeyError:
+            pass
+
     return macs
 
 
@@ -59,7 +68,7 @@ def import_statistics(nodes, stats):
             node['statistics'][target] = f(reduce(dict.__getitem__,
                                                   source,
                                                   statistics))
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, ZeroDivisionError):
             pass
 
     macs = build_mac_table(nodes)
@@ -73,6 +82,7 @@ def import_statistics(nodes, stats):
         add(node, stats, 'memory_usage', ['memory'],
             lambda d: 1 - d['free'] / d['total'])
         add(node, stats, 'rootfs_usage', ['rootfs_usage'])
+        add(node, stats, 'traffic', ['traffic'])
 
 
 def import_mesh_ifs_vis_data(nodes, vis_data):
@@ -97,12 +107,29 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
     for v in mesh_nodes:
         node = v[0]
 
-        try:
-            mesh_ifs = set(node['nodeinfo']['network']['mesh_interfaces'])
-        except KeyError:
-            mesh_ifs = set()
+        ifs = set()
 
-        node['nodeinfo']['network']['mesh_interfaces'] = list(mesh_ifs | v[1])
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh_interfaces']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
+        except KeyError:
+            pass
+
+        node['nodeinfo']['network']['mesh_interfaces'] = list(ifs | v[1])
 
 
 def import_vis_clientcount(nodes, vis_data):

lib/rrddb.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+import time
+import os
+
+from lib.GlobalRRD import GlobalRRD
+from lib.NodeRRD import NodeRRD
+from lib.GateRRD import GateRRD
+
+class RRD(object):
+    def __init__(self,
+                 database_directory,
+                 image_path,
+                 display_time_global="7d",
+                 display_time_node="1d"):
+
+        self.dbPath = database_directory
+        self.globalDb = GlobalRRD(self.dbPath)
+        self.imagePath = image_path
+        self.displayTimeGlobal = display_time_global
+        self.displayTimeNode = display_time_node
+
+        self.currentTimeInt = (int(time.time()) / 60) * 60
+        self.currentTime = str(self.currentTimeInt)
+
+    def update_database(self, nodes):
+        online_nodes = dict(filter(
+            lambda d: d[1]['flags']['online'], nodes.items()))
+        client_count = sum(map(
+            lambda d: d['statistics']['clients'], online_nodes.values()))
+
+        # Refresh global database
+        self.globalDb.update(len(online_nodes), client_count)
+
+        # Refresh databases for all single nodes
+        for node_id, node in online_nodes.items():
+            if node['flags']['gateway']:
+                rrd = GateRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
+            else:
+                rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
+            rrd.update()
+
+    def update_images(self):
+        # Create image path if it does not exist
+        try:
+            os.stat(self.imagePath)
+        except OSError:
+            os.mkdir(self.imagePath)
+
+        self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"),
+                            self.displayTimeGlobal)
+
+        nodedb_files = os.listdir(self.dbPath)
+
+        for file_name in nodedb_files:
+            if not os.path.isfile(os.path.join(self.dbPath, file_name)):
+                continue
+
+            node_name = os.path.basename(file_name).split('.')
+            if node_name[1] == 'rrd' and not node_name[0] == "nodes":
+                rrd = NodeRRD(os.path.join(self.dbPath, file_name))
+                rrd.graph(self.imagePath, self.displayTimeNode)

lib/validate.py

@@ -0,0 +1,19 @@
+import json
+
+
+def validate_nodeinfos(nodeinfos):
+    result = []
+
+    for nodeinfo in nodeinfos:
+        if validate_nodeinfo(nodeinfo):
+            result.append(nodeinfo)
+
+    return result
+
+
+def validate_nodeinfo(nodeinfo):
+    if 'location' in nodeinfo:
+        if 'latitude' not in nodeinfo['location'] or 'longitude' not in nodeinfo['location']:
+            return False
+
+    return True