early respondc support

add firstseen to nodelist.json

.gitignore

@@ -1,6 +1,3 @@
-# backups
-*~
-
 # script-generated
 aliases*.json
 nodedb/

README.md

@@ -22,6 +22,7 @@ For the script's regular execution add the following to the crontab:
 
 - Python 3
 - Python 3 Package [Networkx](https://networkx.github.io/)
+- Python 3 Package dateutil
 - [alfred-json](https://github.com/tcatm/alfred-json)
 - rrdtool (if run with `--with-rrd`)
 
@@ -115,3 +116,10 @@ This will remove owner information from nodes.json before copying the data
 to your webserver.
 
 [jq]: https://stedolan.github.io/jq/
+
+
+# Convert from nodes.json version 1 to version 2
+
+    jq '.nodes = (.nodes | to_entries | map(.value)) | .version = 2' \
+    < nodes.json > nodes.json.new
+    mv nodes.json.new nodes.json

backend.py

@@ -2,20 +2,12 @@
 """
 backend.py - ffmap-backend runner
 https://github.com/ffnord/ffmap-backend
-
-Erweiterte Version von Freifunk Pinneberg
-  - Graphiken aus RRD-Daten nur auf Anforderung erzeugen
-  - Verzeichnis für die RRD-Nodedb als Kommandozeilenparameter
-  - Statistikerzeugung korrigiert: Initialisierung und Befüllung
-    zu passenden Zeitpunkten
-
 """
 import argparse
-import configparser
 import json
 import os
 import sys
-import logging, logging.handlers
+import dateutil.parser
 from datetime import datetime
 
 import networkx as nx
@@ -27,24 +19,29 @@ from lib.batman import Batman
 from lib.rrddb import RRD
 from lib.nodelist import export_nodelist
 from lib.validate import validate_nodeinfos
+from lib.respondc import request
 
-NODES_VERSION = 1
+NODES_VERSION = 2
 GRAPH_VERSION = 1
 
-cfg = {
-    'cfgfile': '/etc/ffmap/ffmap-test.cfg',
-    'logfile': '/var/log/ffmap.log',
-    'loglevel': 5,
-    'dest_dir': '/var/lib/ffmap/mapdata',
-    'aliases': [],
-    'prune': 0,
-    'nodedb': '/var/lib/ffmap/nodedb',
-    'rrd_data': False,
-    'rrd_graphs': False,
-    'redis': False
-}
+def recently_lost_nodes(now, nodesdict, maxage=600):
+  nodes = []
+  for node in nodesdict.values():
+    lastseen = dateutil.parser.parse(node['lastseen'])
+    age = (now - lastseen).total_seconds()
+
+    if age < maxage and age != 0:
+      nodes.append(node)
+
+  return nodes
 
 def main(params):
+    def node_to_ips(node):
+      try:
+        return node['nodeinfo']['network']['addresses']
+      except KeyError:
+        return []
+
     os.makedirs(params['dest_dir'], exist_ok=True)
 
     nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
@@ -77,76 +74,110 @@ def main(params):
                     'Unparseable value "{0}" in --mesh parameter.'.
                     format(value))
 
-    # read nodedb state from nodes.json
+    # read nodedb state from node.json
     try:
-        with open(nodes_fn, 'r') as nodedb_handle:
+        with open(nodes_fn, 'r', encoding=('UTF-8')) as nodedb_handle:
             nodedb = json.load(nodedb_handle)
-    except (IOError, ValueError):
-        nodedb = {'nodes': dict()}
-
-    # flush nodedb if it uses the old format
-    if 'links' in nodedb:
-        nodedb = {'nodes': dict()}
+    except IOError:
+        nodedb = {'nodes': []}
 
     # set version we're going to output
     nodedb['version'] = NODES_VERSION
 
     # update timestamp and assume all nodes are offline
     nodedb['timestamp'] = now.isoformat()
-    for node_id, node in nodedb['nodes'].items():
-        node['flags']['online'] = False
+
+    nodesdict = {}
+
+    nodesdict, graph = DO(params, nodesdict, graph)
+
+    for node in nodedb['nodes']:
+        nodesdict[node['nodeinfo']['node_id']] = node
 
     # integrate alfred nodeinfo
     for alfred in alfred_instances:
         nodeinfo = validate_nodeinfos(alfred.nodeinfo())
-        nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
-                              now, assume_online=True)
+        nodes.import_nodeinfo(nodesdict, nodeinfo, now, assume_online=True)
+
+
+    # acquire data from respondd
+    responses = list(request('nodeinfo statistics', ['ff02::2:1001'], interface=params['interface']))
+
+    nodeinfos = list(map(lambda x: x['nodeinfo'], filter(lambda x: 'nodeinfo' in x, responses)))
+    nodes.import_nodeinfo(nodesdict, validate_nodeinfos(nodeinfos), now, assume_online=True)
+
+    ips = [i[0] for i in map(node_to_ips, recently_lost_nodes(now, nodesdict))]
+    a = request('nodeinfo statistics', ips, interface=params['interface'], timeout=2)
+    nodeinfos = list(map(lambda x: x['nodeinfo'], filter(lambda x: 'nodeinfo' in x, a)))
+    nodes.import_nodeinfo(nodesdict, validate_nodeinfos(nodeinfos), now, assume_online=True)
+    responses += a
+
+    ips = [i[0] for i in map(node_to_ips, recently_lost_nodes(now, nodesdict))]
+    a = request('nodeinfo statistics', ips, interface=params['interface'], timeout=2)
+    nodeinfos = list(map(lambda x: x['nodeinfo'], filter(lambda x: 'nodeinfo' in x, a)))
+    nodes.import_nodeinfo(nodesdict, validate_nodeinfos(nodeinfos), now, assume_online=True)
+    responses += a
+
+    ips = [i[0] for i in map(node_to_ips, recently_lost_nodes(now, nodesdict))]
+    a = request('nodeinfo statistics', ips, interface=params['interface'], timeout=2)
+    nodeinfos = list(map(lambda x: x['nodeinfo'], filter(lambda x: 'nodeinfo' in x, a)))
+    nodes.import_nodeinfo(nodesdict, validate_nodeinfos(nodeinfos), now, assume_online=True)
+    responses += a
+
+    for node in nodesdict.values():
+      lastseen = dateutil.parser.parse(node['lastseen'])
+      age = (now - lastseen).total_seconds()
+
+      online = age < params['hysteresis']
+
+      node['flags']['online'] = online
+
+      if not online:
+        nodes.reset_statistics(node)
+
 
     # integrate static aliases data
     for aliases in params['aliases']:
         with open(aliases, 'r') as f:
             nodeinfo = validate_nodeinfos(json.load(f))
-            nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
+            nodes.import_nodeinfo(nodesdict, nodeinfo,
                                   now, assume_online=False)
 
-    # prepare statistics collection
-    nodes.reset_statistics(nodedb['nodes'])
+    for alfred in alfred_instances:
+        nodes.import_statistics(nodesdict, alfred.statistics())
 
-    # acquire gwl and visdata for each batman instance
+    nodes.import_statistics(nodesdict, list(map(lambda x: x['statistics'], filter(lambda x: 'statistics' in x, responses))))
+
+    # acquire visdata for each batman instance
     mesh_info = []
     for batman in batman_instances:
         vd = batman.vis_data()
-        gwl = batman.gateway_list()
 
-        mesh_info.append((vd, gwl))
+        mesh_info.append(vd)
 
     # update nodedb from batman-adv data
-    for vd, gwl in mesh_info:
-        nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vd)
-        nodes.import_vis_clientcount(nodedb['nodes'], vd)
-        nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
-        nodes.mark_gateways(nodedb['nodes'], gwl)
-
-    # get alfred statistics
-    for alfred in alfred_instances:
-        nodes.import_statistics(nodedb['nodes'], alfred.statistics())
+    for vd in mesh_info:
+        nodes.import_mesh_ifs_vis_data(nodesdict, vd)
+        nodes.import_vis_clientcount(nodesdict, vd)
 
     # clear the nodedb from nodes that have not been online in $prune days
     if params['prune']:
-        nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
+        nodes.prune_nodes(nodesdict, now, params['prune'])
 
     # build nxnetworks graph from nodedb and visdata
     batadv_graph = nx.DiGraph()
-    for vd, gwl in mesh_info:
-        graph.import_vis_data(batadv_graph, nodedb['nodes'], vd)
+    for vd in mesh_info:
+        graph.import_vis_data(batadv_graph, nodesdict, vd)
 
     # force mac addresses to be vpn-link only (like gateways for example)
     if params['vpn']:
         graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
 
+    nodedb['nodes'] = list(nodesdict.values())
+
     def extract_tunnel(nodes):
         macs = set()
-        for id, node in nodes.items():
+        for node in nodes:
             try:
                 for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
                     macs.add(mac)
@@ -175,95 +206,40 @@ def main(params):
 
     # optional rrd graphs (trigger with --rrd)
     if params['rrd']:
-        if params['nodedb']:
-            rrd = RRD(params['nodedb'], os.path.join(params['dest_dir'], 'nodes'))
-        else:
-            script_directory = os.path.dirname(os.path.realpath(__file__))
-            rrd = RRD(os.path.join(script_directory, 'nodedb'),
-                      os.path.join(params['dest_dir'], 'nodes'))
+        script_directory = os.path.dirname(os.path.realpath(__file__))
+        rrd = RRD(os.path.join(script_directory, 'nodedb'),
+                  os.path.join(params['dest_dir'], 'nodes'))
         rrd.update_database(nodedb['nodes'])
-        if params['img']:
-            rrd.update_images()
+        rrd.update_images()
 
-def set_loglevel(nr):
-    """
-    Umsetzen der Nummer auf einen für "logging" passenden Wert
-    Die Nummer kann ein Wert zwischen 0 - kein Logging und 5 - Debug sein
-    """
-    level = (None, logging.CRITICAL, logging.ERROR, logging.WARNING,
-             logging.INFO, logging.DEBUG)
-    if nr > 5:
-        nr = 5
-    elif nr < 0:
-        nr = 0
-    return level[nr]
 
 if __name__ == '__main__':
-
-    # get options from command line
-    parser = argparse.ArgumentParser(
-        description = "Collect data for ffmap: creates json files and "
-                      "optional rrd data and graphs")
+    parser = argparse.ArgumentParser()
 
     parser.add_argument('-a', '--aliases',
                         help='Read aliases from FILE',
                         nargs='+', default=[], metavar='FILE')
     parser.add_argument('-m', '--mesh',
-                        default=['bat0'], nargs='+',
-                        help='Use given batman-adv mesh interface(s) (defaults '
+                        default=[], nargs='+',
+                        help='Use given batman-adv mesh interface(s) (defaults'
                              'to bat0); specify alfred unix socket like '
                              'bat0:/run/alfred0.sock.')
+    parser.add_argument('--hysteresis', default=300,
+                        help='Duration (seconds) after which a node is considered to be offline')
+    parser.add_argument('-i', '--interface',
+                        help='Interface for contacting respondd',
+                        required=True)
     parser.add_argument('-d', '--dest-dir', action='store',
                         help='Write output to destination directory',
-                        required=False)
-    parser.add_argument('-c', '--config', action='store', metavar='FILE',
-                        help='read configuration from FILE')
+                        required=True)
     parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
                         help='Assume MAC addresses are part of vpn')
     parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
-                        help='Forget nodes offline for at least DAYS')
-    parser.add_argument('-r', '--with-rrd', dest='rrd', action='store_true',
+                        help='forget nodes offline for at least DAYS')
+    parser.add_argument('--with-rrd', dest='rrd', action='store_true',
                         default=False,
-                        help='Enable the collection of RRD data')
-    parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
-                        help='Directory for node RRD data files')
-    parser.add_argument('-i', '--with-img', dest='img', action='store_true',
-                        default=False,
-                        help='Enable the rendering of RRD graphs (cpu '
+                        help='enable the rendering of RRD graphs (cpu '
                              'intensive)')
 
     options = vars(parser.parse_args())
-    if options['config']:
-        cfg['cfgfile'] = options['config']
-
-    config = configparser.ConfigParser(cfg)
-    if config.read(cfg['cfgfile']):
-        if not options['nodedb']:
-            options['nodedb'] = config.get('rrd', 'nodedb')
-        if not options['dest_dir']:
-            options['dest_dir'] = config.get('global', 'dest_dir')
-        if not options['rrd']:
-            options['rrd'] = config.getboolean('rrd', 'enabled')
-        if not options['img']:
-            options['img'] = config.getboolean('rrd', 'graphs')
-        cfg['logfile'] = config.get('global', 'logfile')
-        cfg['loglevel'] = config.getint('global', 'loglevel')
-
-    # At this point global configuration is available. Time to enable logging
-    # Logging is handled by the operating system, so use WatchedFileHandler
-    handler = logging.handlers.WatchedFileHandler(cfg['logfile'])
-    handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s',
-                                           datefmt='%Y-%m-%d %H:%M:%S'))
-    log = logging.getLogger()
-    log.addHandler(handler)
-    loglevel = set_loglevel(cfg['loglevel'])
-    if loglevel:
-        log.setLevel(loglevel)
-    else:
-        log.disabled = True
-
-    log.info("%s started" % sys.argv[0])
-    if os.path.isfile(cfg['cfgfile']):
-        log.info("using configuration from '%s'" % cfg['cfgfile'])
     main(options)
-    log.info("%s finished" % sys.argv[0])

ffmap-modify.py

@@ -1,185 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-Knotendaten manuell ändern
-Das ist z.B. für ausgeschaltete Knoten interessant, die nur 
-temporär nicht zur Verfügung stehen. Die können ausgeblendet
-werden.
-Das ist besser als löschen, weil so die Statistik nicht
-verschwindet
-
-Änderungsprotokoll
-==================
-
-Version  Datum       Änderung(en)                                           von 
--------- ----------- ------------------------------------------------------ ----
-1.0      2017-08-03  Programm in das ffmap-backend Projekt integriert       tho
-
-"""
-
-import argparse
-import configparser
-import json
-import os
-import sys
-import glob
-
-# Einstellungen werden in folgender Reihenfolge verarbeitet
-# später gesetzte Werte überschreiben frühere
-#  1. im Programm hart codiert
-#  2. aus der zentralen Konfigurationsdatei gelesen  
-#  3. als Kommandozeilenoptionen angegeben
-cfg = {
-    'cfgfile': '/etc/ffmap/ffmap.cfg',
-    'logfile': '/var/log/ffmap.log',
-    'loglevel': 2,
-    'dest_dir': '/var/lib/ffmap/mapdata',
-    'nodedb': '/var/lib/ffmap/nodedb',
-    'imgpath': '/var/www/meshviewer/stats/img'
-}
-
-roles_defined = ('node', 'temp', 'mobile', 'offloader', 'service', 'test', 'gate', 'plan', 'hidden')
-
-def main(cfg):
-
-    # Pfade zu den beteiligten Dateien
-    nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
-    nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
-
-    # 1. Knotendaten (NodeDB)
-    # 1.1 Daten laden
-    try:
-        with open(nodes_fn, 'r') as nodedb_handle:
-            nodedb = json.load(nodedb_handle)
-    except IOError:
-        print("Error reading nodedb file %s" % nodes_fn)
-        nodedb = {'nodes': dict()}
-    # 1.2 Knoten bearbeiten
-    changed = False
-    for n in cfg['nodeid']:
-        if n in nodedb['nodes']:
-            print("Modify %s in nodedb" % n)
-            if 'role' in cfg and cfg['role'] in roles_defined:
-                try:
-                    oldrole = nodedb['nodes'][n]['nodeinfo']['system']['role']
-                except KeyError:
-                    oldrole = '<unset>'
-                print("  - change role from '%s' to '%s'" % (oldrole, cfg['role']))
-                nodedb['nodes'][n]['nodeinfo']['system']['role'] = cfg['role']
-                changed = True
-            if 'location' in cfg:
-                print("  - remove location")
-                # del nodedb['nodes'][n]['nodeinfo']['location']
-                changed = True
-        else:
-            print("Node %s not found in nodedb" % n)        
-    # 1.3 Geänderte Daten zurückschreiben
-    if changed:
-        try:
-            with open(nodes_fn, 'w') as nodedb_handle:
-                json.dump(nodedb, nodedb_handle)
-        except IOError:
-            print("Error writing nodedb file %s" % nodes_fn)
-
-    # 2. Knotenliste (NodeList)
-    try:
-        with open(nodelist_fn, 'r') as nodelist_handle:
-            nodelist = json.load(nodelist_handle)
-    except IOError:
-        print("Error reading nodelist file %s" % nodelist_fn)
-        nodelist = {'nodelist': dict()}
-    # 2.1 Knoten bearbeiten
-    changed = False
-    ixlist = []
-    for nodeid in cfg['nodeid']:
-        found = False
-        for ix, node in enumerate(nodelist['nodes']):
-            if node['id'] == nodeid:
-                found = True
-                break
-        if found:
-            print("Modify %s in nodelist" % nodeid)
-            if 'role' in cfg and cfg['role'] in roles_defined:
-                try:
-                    oldrole = nodelist['nodes'][ix]['role']
-                except KeyError:
-                    oldrole = '<unset>'
-                print("  - change role from '%s' to '%s'" % (oldrole, cfg['role']))
-                nodelist['nodes'][ix]['role'] = cfg['role']
-            if 'location' in cfg:
-                print("  - remove location")
-                try:
-                    #del nodelist['nodes'][ix]['position']
-                    pass
-                except KeyError:
-                    pass
-                changed = True
-        else:
-            print ("Node %s not found in nodelist" % nodeid)
-    # 2.3 Geänderte Daten zurückschreiben
-    if changed:
-        try:
-            with open(nodelist_fn, 'w') as nodelist_handle:
-                json.dump(nodelist, nodelist_handle)
-        except IOError:
-            print("Error writing nodelist file %s" % nodelist_fn)
-
-if __name__ == "__main__":
-
-    # Optionen von der Kommandozeile lesen
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('-c', '--config', action='store',
-                        help='Configuration file')
-
-    parser.add_argument('-d', '--dest-dir', action='store',
-                        help='Directory with JSON data files',
-                        required=False)
-
-    parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
-                        nargs='+', required=True,
-                        help='Node id to modify')
-
-    parser.add_argument('-l', '--location', action='store_true',
-                        help='Clear location information (hides node)',
-                        required=False)
-
-    parser.add_argument('-r', '--role', action='store',
-                        help='Set new role',
-                        required=False)
-
-    # TODO
-    # Optionen was genau gemacht werden soll
-    # -p Position entfernen, Knoten wird nicht mehr angezeigt
-    # -r <rolle> Rolle einstellen
-
-    options = vars(parser.parse_args())
-
-
-    # Konfigurationsdatei einlesen
-    if options['config']:
-        cfg['cfgfile'] = options['config']
-    config = configparser.ConfigParser(cfg)
-    # config.read liefert eine Liste der geparsten Dateien
-    # zurück. Wenn sie leer ist, war z.B. die Datei nicht
-    # vorhanden
-    if config.read(cfg['cfgfile']):
-        if 'global' in config:
-            cfg['logfile'] = config['global']['logfile']
-            cfg['loglevel'] = config['global']['loglevel']
-            cfg['dest_dir'] = config['global']['dest_dir']
-    else:
-        print('Config file %s not parsed' % cfg['cfgfile'])
-
-    # Optionen von der Kommandozeile haben höchste Priorität
-    cfg['nodeid'] = options['nodeid']
-    if options['dest_dir']:
-        cfg['dest_dir'] = options['dest_dir']
-    if options['location']:
-        cfg['location'] = True
-    if options['role']:
-        cfg['role'] = options['role']
-
-    # Alles initialisiert, auf geht's
-    main(cfg)

ffmap-remove.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-Lösche einen Knoten manuell aus dem Backend:
-- JSON
-  - NodeDB
-  - NodeList
-  - Graph
-- RRD-Dateien
-- Bilder vom Webserver
-
-Änderungsprotokoll
-==================
-
-Version  Datum       Änderung(en)                                           von 
--------- ----------- ------------------------------------------------------ ----
-1.0      2017-01-06  Programm in das ffmap-backend Projekt integriert       tho
-
-"""
-
-import argparse
-import configparser
-import json
-import os
-import sys
-import glob
-
-# Einstellungen werden in folgender Reihenfolge verarbeitet
-# später gesetzte Werte überschreiben frühere
-#  1. im Programm hart codiert
-#  2. aus der zentralen Konfigurationsdatei gelesen  
-#  3. als Kommandozeilenoptionen angegeben
-cfg = {
-    'cfgfile': '/etc/ffmap/ffmap.cfg',
-    'logfile': '/var/log/ffmap.log',
-    'loglevel': 2,
-    'dest_dir': '/var/lib/ffmap/mapdata',
-    'nodedb': '/var/lib/ffmap/nodedb',
-    'imgpath': '/var/www/meshviewer/stats/img'
-}
-
-def main(cfg):
-
-    # Pfade zu den beteiligten Dateien
-    nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
-    graph_fn = os.path.join(cfg['dest_dir'], 'graph.json')
-    nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
-
-    # 1. Knotendaten (NodeDB) bereinigen
-    # 1.1 Daten laden
-    try:
-        with open(nodes_fn, 'r') as nodedb_handle:
-            nodedb = json.load(nodedb_handle)
-    except IOError:
-        print("Error reading nodedb file %s" % nodes_fn)
-        nodedb = {'nodes': dict()}
-    # 1.2 Knoten entfernen
-    changed = False
-    for n in cfg['nodeid']:
-        if n in nodedb['nodes']:
-            print("Remove %s from nodedb" % n)
-            del nodedb['nodes'][n]
-            changed = True
-        else:
-            print("Node %s not found in nodedb" % n)        
-    # 1.3 Geänderte Daten zurückschreiben
-    if changed:
-        try:
-            with open(nodes_fn, 'w') as nodedb_handle:
-                json.dump(nodedb, nodedb_handle)
-        except IOError:
-            print("Error writing nodedb file %s" % nodes_fn)
-
-    # 2. Knotenliste (NodeList) bereinigen
-    try:
-        with open(nodelist_fn, 'r') as nodelist_handle:
-            nodelist = json.load(nodelist_handle)
-    except IOError:
-        print("Error reading nodelist file %s" % nodelist_fn)
-        nodelist = {'nodelist': dict()}
-    # 2.1 Knoten entfernen
-    changed = False
-    ixlist = []
-    for nodeid in cfg['nodeid']:
-        found = False
-        for ix, node in enumerate(nodelist['nodes']):
-            if node['id'] == nodeid:
-                found = True
-                break
-        if found:
-            print("Remove %s from nodelist" % nodeid)
-            del nodelist['nodes'][ix]
-            changed = True
-        else:
-            print ("Node %s not found in nodelist" % nodeid)
-    # 2.3 Geänderte Daten zurückschreiben
-    if changed:
-        try:
-            with open(nodelist_fn, 'w') as nodelist_handle:
-                json.dump(nodelist, nodelist_handle)
-        except IOError:
-            print("Error writing nodelist file %s" % nodelist_fn)
-
-    # 3. Graph (NodeGraph) bereinigen
-    # 3.1 Graph laden
-    try:
-        with open(graph_fn, 'r') as graph_handle:
-            graph = json.load(graph_handle)
-    except IOError:
-        print("Error reading graph file %s" % graph_fn)
-        graph = {'graph': dict()}
-    # 3.2 Finde Knoten und Links
-    # Nodes und Links gehören zusammen   
-    changed = False
-    for nodeid in cfg['nodeid']:
-        found = False
-        for ixn, node in enumerate(graph["batadv"]["nodes"]):
-            # Es kann nodes ohne "node_id" geben
-            try:
-                if node["node_id"] == nodeid:
-                    found = True
-                    break
-            except KeyError:
-                pass
-        if found:
-            print("Found %s in graph nodes at index %d" % (nodeid, ixn))
-            del graph["batadv"]["nodes"][ixn]
-            # Suche Link source oder target dem gefundenen Index entsprechen
-            ixlist = []
-            for ixg, link in enumerate(graph["batadv"]["links"]):
-                if link["source"] == ixn:
-                    print("Found source link at index %d" % ixg)
-                    print("  -> %s" % graph["batadv"]["nodes"][link["target"]])
-                    ixlist.append(ixg)
-                if link["target"] == ixn:
-                    print("Found target link at index %d" % ixg)
-                    print("  -> %s" % graph["batadv"]["nodes"][link["source"]])
-                    ixlist.append(ixg)
-            for ix in ixlist:
-                del graph["batadv"]["nodes"][ix]
-            changed = True
-        else:
-            print("Node %s not found in graph nodes" % nodeid)
-    # 3.3 Zurückschreiben der geänderten Daten
-    if changed:
-        try:
-            with open(graph_fn, 'w') as graph_handle:
-                json.dump(graph, graph_handle)
-        except IOError:
-            print("Error writing graph file %s" % graph_fn)
-
-    # 4. Entferne RRD-Dateien
-    for nodeid in cfg['nodeid']:
-        rrdfile = os.path.join(cfg['nodedb'], nodeid+'.rrd')
-        if os.path.isfile(rrdfile):
-            print("Removing RRD database file %s" % os.path.basename(rrdfile))
-        else:
-            print("RRD database file %s not found" % os.path.basename(rrdfile))
-        try:
-            os.remove(rrdfile)
-        except OSError:
-            pass
-
-    # 5. Entferne Bilder vom Webserver
-    count_deleted = 0
-    for nodeid in cfg['nodeid']:
-        for imagefile in glob.glob(os.path.join(cfg['imgpath'], nodeid+'_*.png')):
-            print("Removing stats image %s" % os.path.basename(imagefile))
-            try:
-                os.remove(imagefile)
-                count_deleted += 1
-            except OSError:
-                pass
-    if count_deleted == 0:
-        print("No stats images found in %s" % cfg['imgpath'])
-
-if __name__ == "__main__":
-
-    # Optionen von der Kommandozeile lesen
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('-c', '--config', action='store',
-                        help='Configuration file')
-
-    parser.add_argument('-d', '--dest-dir', action='store',
-                        help='Directory with JSON data files',
-                        required=False)
-
-    parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
-                        nargs='+', required=True,
-                        help='Node id  to remove')
-
-    parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
-                        help='Directory for node RRD data files')
-
-    options = vars(parser.parse_args())
-
-
-    # Konfigurationsdatei einlesen
-    if options['config']:
-        cfg['cfgfile'] = options['config']
-    config = configparser.ConfigParser(cfg)
-    # config.read liefert eine Liste der geparsten Dateien
-    # zurück. Wenn sie leer ist, war z.B. die Datei nicht
-    # vorhanden
-    if config.read(cfg['cfgfile']):
-        if 'global' in config:
-            cfg['logfile'] = config['global']['logfile']
-            cfg['loglevel'] = config['global']['loglevel']
-            cfg['dest_dir'] = config['global']['dest_dir']
-        if 'rrd' in config:
-            cfg['nodedb'] = config['rrd']['nodedb']
-    else:
-        print('Config file %s not parsed' % cfg['cfgfile'])
-
-    # Optionen von der Kommandozeile haben höchste Priorität
-    cfg['nodeid'] = options['nodeid']
-    if options['dest_dir']:
-        cfg['dest_dir'] = options['dest_dir']
-    if options['nodedb']:
-        cfg['nodedb'] = options['nodedb']
-
-    # Alles initialisiert, auf geht's
-    main(cfg)

lib/GateRRD.py

@@ -1,74 +0,0 @@
-"""
-RRD for gateways
-"""
-
-import os
-import subprocess
-
-from lib.RRD import DS, RRA, RRD
-
-
-class GateRRD(RRD):
-    ds_list = [
-        DS('upstate', 'GAUGE', 120, 0, 1),
-        DS('clients', 'GAUGE', 120, 0, float('NaN')),
-        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
-        DS('leases', 'GAUGE', 120, 0, float('NaN')),       
-    ]
-    rra_list = [
-        RRA('AVERAGE', 0.5, 1, 120),    #  2 hours of  1 minute samples
-        RRA('AVERAGE', 0.5, 5, 1440),   #  5 days  of  5 minute samples
-        RRA('AVERAGE', 0.5, 15, 672),   #  7 days  of 15 minute samples
-        RRA('AVERAGE', 0.5, 60, 720),   # 30 days  of  1 hour   samples
-        RRA('AVERAGE', 0.5, 720, 730),  #  1 year  of 12 hour   samples
-    ]
-
-    def __init__(self, filename, node=None):
-        """
-        Create a new RRD for a given node.
-
-        If the RRD isn't supposed to be updated, the node can be omitted.
-        """
-        self.node = node
-        super().__init__(filename)
-        self.ensure_sanity(self.ds_list, self.rra_list, step=60)
-
-    @property
-    def imagename(self):
-        return "{basename}.png".format(
-            basename=os.path.basename(self.filename).rsplit('.', 2)[0])
-
-    # TODO: fix this, python does not support function overloading
-    def update(self):
-        values  = {
-            'upstate': int(self.node['flags']['online']),
-            'clients': float(self.node['statistics']['clients']),
-        }
-        if 'loadavg' in self.node['statistics']:
-            values['loadavg'] = float(self.node['statistics'].get('loadavg', 0))
-        # Gateways can send the peer count. We use the clients field to store data 
-        if 'peers' in self.node['statistics']:
-            values['clients'] = self.node['statistics']['peers']
-        if 'leases' in self.node['statistics']:
-            values['leases'] = self.node['statistics']['leases']
-        super().update(values)
-
-    def graph(self, directory, timeframe):
-        """
-        Create a graph in the given directory. The file will be named
-        basename.png if the RRD file is named basename.rrd
-        """
-        args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
-                '-s', '-' + timeframe,
-                '-w', '800',
-                '-h', '400',
-                '-l', '0',
-                '-y', '1:1',
-                'DEF:clients=' + self.filename + ':clients:AVERAGE',
-                'VDEF:maxc=clients,MAXIMUM',
-                'CDEF:c=0,clients,ADDNAN',
-                'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
-                'AREA:c#0F0:up\\l',
-                'AREA:d#F00:down\\l',
-                'LINE1:c#00F:clients connected\\l']
-        subprocess.check_output(args)

lib/NodeRRD.py

@@ -1,7 +1,3 @@
-"""
-RRD for nodes
-"""
-
 import os
 import subprocess
 
@@ -12,13 +8,16 @@ class NodeRRD(RRD):
     ds_list = [
         DS('upstate', 'GAUGE', 120, 0, 1),
         DS('clients', 'GAUGE', 120, 0, float('NaN')),
-        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
     ]
     rra_list = [
-        RRA('AVERAGE', 0.5, 1, 120),    #  2 hours of  1 minute samples
-        RRA('AVERAGE', 0.5, 5, 1440),   #  5 days  of  5 minute samples
-        RRA('AVERAGE', 0.5, 60, 720),   # 30 days  of  1 hour   samples
-        RRA('AVERAGE', 0.5, 720, 730),  #  1 year  of 12 hour   samples
+        # 2 hours of  1 minute samples
+        RRA('AVERAGE', 0.5, 1, 120),
+        #  5 days  of  5 minute samples
+        RRA('AVERAGE', 0.5, 5, 1440),
+        # 30 days  of  1 hour   samples
+        RRA('AVERAGE', 0.5, 60, 720),
+        #  1 year  of 12 hour   samples
+        RRA('AVERAGE', 0.5, 720, 730),
     ]
 
     def __init__(self, filename, node=None):
@@ -38,13 +37,8 @@ class NodeRRD(RRD):
 
     # TODO: fix this, python does not support function overloading
     def update(self):
-        values  = {
-            'upstate': int(self.node['flags']['online']),
-            'clients': self.node['statistics']['clients']
-        }
-        if 'loadavg' in self.node['statistics']:
-            values['loadavg'] = float(self.node['statistics']['loadavg'])
-        super().update(values)
+        super().update({'upstate': int(self.node['flags']['online']),
+                        'clients': self.node['statistics']['clients']})
 
     def graph(self, directory, timeframe):
         """

lib/alfred.py

@@ -20,11 +20,7 @@ class Alfred(object):
         if self.unix_sock:
             cmd.extend(['-s', self.unix_sock])
 
-        # There should not be any warnings which would be sent by cron
-        # every minute. Therefore suppress error output of called program
-        FNULL = open(os.devnull, 'w')
-        output = subprocess.check_output(cmd, stderr=FNULL)
-        FNULL.close()
+        output = subprocess.check_output(cmd)
         return json.loads(output.decode("utf-8")).values()
 
     def nodeinfo(self):

lib/batman.py

@@ -3,6 +3,7 @@ import json
 import os
 import re
 
+
 class Batman(object):
     """
     Bindings for B.A.T.M.A.N. Advanced
@@ -21,7 +22,7 @@ class Batman(object):
         self.environ = env
 
         # compile regular expressions only once on startup
-        self.mac_addr_pattern = re.compile(r'(([a-f0-9]{2}:){5}[a-f0-9]{2})')
+        self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})')
 
     def vis_data(self):
         return self.vis_data_batadv_vis()
@@ -51,10 +52,10 @@ class Batman(object):
 
     def gateway_list(self):
         """
-        Parse "batctl meshif <mesh_interface> gwl -n"
+        Parse "batctl -m <mesh_interface> gwl -n"
         into an array of dictionaries.
         """
-        cmd = ['batctl', 'meshif', self.mesh_interface, 'gwl', '-n']
+        cmd = ['batctl', '-m', self.mesh_interface, 'gwl', '-n']
         if os.geteuid() > 0:
             cmd.insert(0, 'sudo')
         output = subprocess.check_output(cmd, env=self.environ)
@@ -80,10 +81,10 @@ class Batman(object):
 
     def gateway_mode(self):
         """
-        Parse "batctl meshif <mesh_interface> gw"
+        Parse "batctl -m <mesh_interface> gw"
         return: tuple mode, bandwidth, if mode != server then bandwidth is None
         """
-        cmd = ['batctl', 'meshif', self.mesh_interface, 'gw']
+        cmd = ['batctl', '-m', self.mesh_interface, 'gw']
         if os.geteuid() > 0:
             cmd.insert(0, 'sudo')
         output = subprocess.check_output(cmd, env=self.environ)

lib/nodelist.py

@@ -1,9 +1,9 @@
 def export_nodelist(now, nodedb):
     nodelist = list()
 
-    for node_id, node in nodedb["nodes"].items():
+    for node in nodedb["nodes"]:
         node_out = dict()
-        node_out["id"] = node_id
+        node_out["id"] = node["nodeinfo"]["node_id"]
         node_out["name"] = node["nodeinfo"]["hostname"]
 
         if "location" in node["nodeinfo"]:
@@ -22,11 +22,6 @@ def export_nodelist(now, nodedb):
         if "clients" in node["statistics"]:
             node_out["status"]["clients"] = node["statistics"]["clients"]
 
-        if "role" in node["nodeinfo"]["system"]:
-            node_out["role"] = node["nodeinfo"]["system"]["role"]
-        else:
-            node_out["role"] = "node"
-
         nodelist.append(node_out)
 
     return {"version": "1.0.1", "nodes": nodelist, "updated_at": now.isoformat()}

lib/nodes.py

@@ -13,10 +13,20 @@ def build_mac_table(nodes):
             pass
 
         try:
-            for upper_if in node['nodeinfo']['network']['mesh'].values():
-                for lower_if in upper_if['interfaces'].values():
-                    for mac in lower_if:
-                        macs[mac] = node_id
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
+                macs[mac] = node_id
         except KeyError:
             pass
 
@@ -43,7 +53,6 @@ def prune_nodes(nodes, now, days):
 def mark_online(node, now):
     node['lastseen'] = now.isoformat()
     node.setdefault('firstseen', now.isoformat())
-    node['flags']['online'] = True
 
 
 def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
@@ -51,15 +60,13 @@ def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
         node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
         node['nodeinfo'] = nodeinfo
         node['flags']['online'] = False
-        node['flags']['gateway'] = False
 
         if assume_online:
             mark_online(node, now)
 
 
-def reset_statistics(nodes):
-    for node in nodes.values():
-        node['statistics'] = {'clients': 0}
+def reset_statistics(node):
+    node['statistics'] = {'clients': 0}
 
 
 def import_statistics(nodes, stats):
@@ -76,11 +83,10 @@ def import_statistics(nodes, stats):
     stats = filter(lambda d: d['node_id'] in nodes, stats)
     for node, stats in map(lambda d: (nodes[d['node_id']], d), stats):
         add(node, stats, 'clients', ['clients', 'total'])
-        add(node, stats, 'gateway', ['gateway'], lambda d: macs.get(d, d))
         add(node, stats, 'uptime', ['uptime'])
         add(node, stats, 'loadavg', ['loadavg'])
         add(node, stats, 'memory_usage', ['memory'],
-            lambda d: 1 - d['free'] / d['total'])
+            lambda d: 1 - (d['free'] + d['buffers'] + d['cached']) / d['total'])
         add(node, stats, 'rootfs_usage', ['rootfs_usage'])
         add(node, stats, 'traffic', ['traffic'])
 
@@ -142,14 +148,6 @@ def import_vis_clientcount(nodes, vis_data):
         nodes[node_id]['statistics'].setdefault('clients', clientcount)
 
 
-def mark_gateways(nodes, gateways):
-    macs = build_mac_table(nodes)
-    gateways = filter(lambda d: d in macs, gateways)
-
-    for node in map(lambda d: nodes[macs[d]], gateways):
-        node['flags']['gateway'] = True
-
-
 def mark_vis_data_online(nodes, vis_data, now):
     macs = build_mac_table(nodes)
 

lib/respondc.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import socket
+import zlib
+import json
+import sys
+
+
+def request(request, targets, interface, timeout=0.5, singleshot=False):
+  try:
+    if_id = socket.if_nametoindex(interface)
+  except OSError:
+    print('interface \'{}\' not found'.format(ifname), file=sys.stderr)
+    return []
+
+  sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+
+  # request
+  message = bytes('GET {}'.format(request), 'utf-8')
+
+  for target in targets:
+    sock.sendto(message, (target, 1001, 0, if_id))
+
+  print('+ {:s}'.format(str(message)), file=sys.stderr)
+
+  sock.settimeout(timeout)
+
+  # receive
+  responses = {}
+  rsp, err = 0, 0
+  while True:
+    print('\r+ {rsp} responses, {err} errors'.format(**locals()), end='', file=sys.stderr)
+
+    try:
+      buffer, address = sock.recvfrom(2048)
+    except socket.timeout:
+      print('\n+ no replies for %f seconds, continuing...' % timeout, file=sys.stderr)
+      break
+
+    try:
+      source = address[0].split('%')[0]
+      data = zlib.decompress(buffer, -15)
+      nodeinfo = json.loads(data.decode('utf-8'))
+      responses[source] = nodeinfo
+      rsp += 1
+    except (zlib.error, UnicodeDecodeError, ValueError):
+      err += 1
+      print('- unreadable answer from {addr}'.format(addr=source), file=sys.stderr)
+
+    if singleshot:
+      break
+
+  return responses.values()
+

lib/rrddb.py

@@ -4,7 +4,7 @@ import os
 
 from lib.GlobalRRD import GlobalRRD
 from lib.NodeRRD import NodeRRD
-from lib.GateRRD import GateRRD
+
 
 class RRD(object):
     def __init__(self,
@@ -22,30 +22,23 @@ class RRD(object):
         self.currentTimeInt = (int(time.time()) / 60) * 60
         self.currentTime = str(self.currentTimeInt)
 
+        try:
+            os.stat(self.imagePath)
+        except OSError:
+            os.mkdir(self.imagePath)
+
     def update_database(self, nodes):
         online_nodes = dict(filter(
             lambda d: d[1]['flags']['online'], nodes.items()))
         client_count = sum(map(
             lambda d: d['statistics']['clients'], online_nodes.values()))
 
-        # Refresh global database
         self.globalDb.update(len(online_nodes), client_count)
-
-        # Refresh databases for all single nodes
         for node_id, node in online_nodes.items():
-            if node['flags']['gateway']:
-                rrd = GateRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
-            else:
-                rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
+            rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
             rrd.update()
 
     def update_images(self):
-        # Create image path if it does not exist
-        try:
-            os.stat(self.imagePath)
-        except OSError:
-            os.mkdir(self.imagePath)
-
         self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"),
                             self.displayTimeGlobal)