ffmap-backend/backend.py

#!/usr/bin/env python3
"""
backend.py - ffmap-backend runner
https://github.com/ffnord/ffmap-backend

Erweiterte Version von Freifunk Pinneberg
  - Graphiken aus RRD-Daten nur auf Anforderung erzeugen
  - Verzeichnis für die RRD-Nodedb als Kommandozeilenparameter
  - Statistikerzeugung korrigiert: Initialisierung und Befüllung
    zu passenden Zeitpunkten

"""
import argparse
import configparser
import json
import os
import sys
import logging, logging.handlers
from datetime import datetime

import networkx as nx
from networkx.readwrite import json_graph

from lib import graph, nodes
from lib.alfred import Alfred
from lib.batman import Batman
from lib.rrddb import RRD
from lib.nodelist import export_nodelist
from lib.validate import validate_nodeinfos

NODES_VERSION = 1
GRAPH_VERSION = 1

cfg = {
    'cfgfile': '/etc/ffmap/ffmap-test.cfg',
    'logfile': '/var/log/ffmap.log',
    'loglevel': 5,
    'dest_dir': '/var/lib/ffmap/mapdata',
    'aliases': [],
    'prune': 0,
    'nodedb': '/var/lib/ffmap/nodedb',
    'rrd_data': False,
    'rrd_graphs': False,
    'redis': False
}

def main(params):
    os.makedirs(params['dest_dir'], exist_ok=True)

    nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
    graph_fn = os.path.join(params['dest_dir'], 'graph.json')
    nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')

    now = datetime.utcnow().replace(microsecond=0)

    # parse mesh param and instantiate Alfred/Batman instances
    alfred_instances = []
    batman_instances = []
    for value in params['mesh']:
        # (1) only batman-adv if, no alfred sock
        if ':' not in value:
            if len(params['mesh']) > 1:
                raise ValueError(
                    'Multiple mesh interfaces require the use of '
                    'alfred socket paths.')
            alfred_instances.append(Alfred(unix_sockpath=None))
            batman_instances.append(Batman(mesh_interface=value))
        else:
            # (2) batman-adv if + alfred socket
            try:
                batif, alfredsock = value.split(':')
                alfred_instances.append(Alfred(unix_sockpath=alfredsock))
                batman_instances.append(Batman(mesh_interface=batif,
                                               alfred_sockpath=alfredsock))
            except ValueError:
                raise ValueError(
                    'Unparseable value "{0}" in --mesh parameter.'.
                    format(value))

    # read nodedb state from nodes.json
    try:
        with open(nodes_fn, 'r') as nodedb_handle:
            nodedb = json.load(nodedb_handle)
    except (IOError, ValueError):
        nodedb = {'nodes': dict()}

    # flush nodedb if it uses the old format
    if 'links' in nodedb:
        nodedb = {'nodes': dict()}

    # set version we're going to output
    nodedb['version'] = NODES_VERSION

    # update timestamp and assume all nodes are offline
    nodedb['timestamp'] = now.isoformat()
    for node_id, node in nodedb['nodes'].items():
        node['flags']['online'] = False

    # integrate alfred nodeinfo
    for alfred in alfred_instances:
        nodeinfo = validate_nodeinfos(alfred.nodeinfo())
        nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
                              now, assume_online=True)

    # integrate static aliases data
    for aliases in params['aliases']:
        with open(aliases, 'r') as f:
            nodeinfo = validate_nodeinfos(json.load(f))
            nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
                                  now, assume_online=False)

    # prepare statistics collection
    nodes.reset_statistics(nodedb['nodes'])

    # acquire gwl and visdata for each batman instance
    mesh_info = []
    for batman in batman_instances:
        vd = batman.vis_data()
        gwl = batman.gateway_list()

        mesh_info.append((vd, gwl))

    # update nodedb from batman-adv data
    for vd, gwl in mesh_info:
        nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vd)
        nodes.import_vis_clientcount(nodedb['nodes'], vd)
        nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
        nodes.mark_gateways(nodedb['nodes'], gwl)

    # get alfred statistics
    for alfred in alfred_instances:
        nodes.import_statistics(nodedb['nodes'], alfred.statistics())

    # clear the nodedb from nodes that have not been online in $prune days
    if params['prune']:
        nodes.prune_nodes(nodedb['nodes'], now, params['prune'])

    # build nxnetworks graph from nodedb and visdata
    batadv_graph = nx.DiGraph()
    for vd, gwl in mesh_info:
        graph.import_vis_data(batadv_graph, nodedb['nodes'], vd)

    # force mac addresses to be vpn-link only (like gateways for example)
    if params['vpn']:
        graph.mark_vpn(batadv_graph, frozenset(params['vpn']))

    def extract_tunnel(nodes):
        macs = set()
        for id, node in nodes.items():
            try:
                for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
                    macs.add(mac)
            except KeyError:
                pass

        return macs

    graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))

    batadv_graph = graph.merge_nodes(batadv_graph)
    batadv_graph = graph.to_undirected(batadv_graph)

    # write processed data to dest dir
    with open(nodes_fn, 'w') as f:
        json.dump(nodedb, f)

    graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
                 'version': GRAPH_VERSION}

    with open(graph_fn, 'w') as f:
        json.dump(graph_out, f)

    with open(nodelist_fn, 'w') as f:
        json.dump(export_nodelist(now, nodedb), f)

    # optional rrd graphs (trigger with --rrd)
    if params['rrd']:
        if params['nodedb']:
            rrd = RRD(params['nodedb'], os.path.join(params['dest_dir'], 'nodes'))
        else:
            script_directory = os.path.dirname(os.path.realpath(__file__))
            rrd = RRD(os.path.join(script_directory, 'nodedb'),
                      os.path.join(params['dest_dir'], 'nodes'))
        rrd.update_database(nodedb['nodes'])
        if params['img']:
            rrd.update_images()

def set_loglevel(nr):
    """
    Umsetzen der Nummer auf einen für "logging" passenden Wert
    Die Nummer kann ein Wert zwischen 0 - kein Logging und 5 - Debug sein
    """
    level = (None, logging.CRITICAL, logging.ERROR, logging.WARNING,
             logging.INFO, logging.DEBUG)
    if nr > 5:
        nr = 5
    elif nr < 0:
        nr = 0
    return level[nr]

if __name__ == '__main__':

    # get options from command line
    parser = argparse.ArgumentParser(
        description = "Collect data for ffmap: creates json files and "
                      "optional rrd data and graphs")

    parser.add_argument('-a', '--aliases',
                        help='Read aliases from FILE',
                        nargs='+', default=[], metavar='FILE')
    parser.add_argument('-m', '--mesh',
                        default=['bat0'], nargs='+',
                        help='Use given batman-adv mesh interface(s) (defaults '
                             'to bat0); specify alfred unix socket like '
                             'bat0:/run/alfred0.sock.')
    parser.add_argument('-d', '--dest-dir', action='store',
                        help='Write output to destination directory',
                        required=False)
    parser.add_argument('-c', '--config', action='store', metavar='FILE',
                        help='read configuration from FILE')
    parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
                        help='Assume MAC addresses are part of vpn')
    parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
                        help='Forget nodes offline for at least DAYS')
    parser.add_argument('-r', '--with-rrd', dest='rrd', action='store_true',
                        default=False,
                        help='Enable the collection of RRD data')
    parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
                        help='Directory for node RRD data files')
    parser.add_argument('-i', '--with-img', dest='img', action='store_true',
                        default=False,
                        help='Enable the rendering of RRD graphs (cpu '
                             'intensive)')

    options = vars(parser.parse_args())
    if options['config']:
        cfg['cfgfile'] = options['config']

    config = configparser.ConfigParser(cfg)
    if config.read(cfg['cfgfile']):
        if not options['nodedb']:
            options['nodedb'] = config.get('rrd', 'nodedb')
        if not options['dest_dir']:
            options['dest_dir'] = config.get('global', 'dest_dir')
        if not options['rrd']:
            options['rrd'] = config.getboolean('rrd', 'enabled')
        if not options['img']:
            options['img'] = config.getboolean('rrd', 'graphs')
        cfg['logfile'] = config.get('global', 'logfile')
        cfg['loglevel'] = config.getint('global', 'loglevel')

    # At this point global configuration is available. Time to enable logging
    # Logging is handled by the operating system, so use WatchedFileHandler
    handler = logging.handlers.WatchedFileHandler(cfg['logfile'])
    handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s',
                                           datefmt='%Y-%m-%d %H:%M:%S'))
    log = logging.getLogger()
    log.addHandler(handler)
    loglevel = set_loglevel(cfg['loglevel'])
    if loglevel:
        log.setLevel(loglevel)
    else:
        log.disabled = True

    log.info("%s started" % sys.argv[0])
    if os.path.isfile(cfg['cfgfile']):
        log.info("using configuration from '%s'" % cfg['cfgfile'])
    main(options)
    log.info("%s finished" % sys.argv[0])