270 lines
9.6 KiB
Python
Executable File
270 lines
9.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
backend.py - ffmap-backend runner
|
|
https://github.com/ffnord/ffmap-backend
|
|
|
|
Erweiterte Version von Freifunk Pinneberg
|
|
- Graphiken aus RRD-Daten nur auf Anforderung erzeugen
|
|
- Verzeichnis für die RRD-Nodedb als Kommandozeilenparameter
|
|
- Statistikerzeugung korrigiert: Initialisierung und Befüllung
|
|
zu passenden Zeitpunkten
|
|
|
|
"""
|
|
import argparse
|
|
import configparser
|
|
import json
|
|
import os
|
|
import sys
|
|
import logging, logging.handlers
|
|
from datetime import datetime
|
|
|
|
import networkx as nx
|
|
from networkx.readwrite import json_graph
|
|
|
|
from lib import graph, nodes
|
|
from lib.alfred import Alfred
|
|
from lib.batman import Batman
|
|
from lib.rrddb import RRD
|
|
from lib.nodelist import export_nodelist
|
|
from lib.validate import validate_nodeinfos
|
|
|
|
NODES_VERSION = 1
|
|
GRAPH_VERSION = 1
|
|
|
|
cfg = {
|
|
'cfgfile': '/etc/ffmap/ffmap-test.cfg',
|
|
'logfile': '/var/log/ffmap.log',
|
|
'loglevel': 5,
|
|
'dest_dir': '/var/lib/ffmap/mapdata',
|
|
'aliases': [],
|
|
'prune': 0,
|
|
'nodedb': '/var/lib/ffmap/nodedb',
|
|
'rrd_data': False,
|
|
'rrd_graphs': False,
|
|
'redis': False
|
|
}
|
|
|
|
def main(params):
|
|
os.makedirs(params['dest_dir'], exist_ok=True)
|
|
|
|
nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
|
|
graph_fn = os.path.join(params['dest_dir'], 'graph.json')
|
|
nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
|
|
|
|
now = datetime.utcnow().replace(microsecond=0)
|
|
|
|
# parse mesh param and instantiate Alfred/Batman instances
|
|
alfred_instances = []
|
|
batman_instances = []
|
|
for value in params['mesh']:
|
|
# (1) only batman-adv if, no alfred sock
|
|
if ':' not in value:
|
|
if len(params['mesh']) > 1:
|
|
raise ValueError(
|
|
'Multiple mesh interfaces require the use of '
|
|
'alfred socket paths.')
|
|
alfred_instances.append(Alfred(unix_sockpath=None))
|
|
batman_instances.append(Batman(mesh_interface=value))
|
|
else:
|
|
# (2) batman-adv if + alfred socket
|
|
try:
|
|
batif, alfredsock = value.split(':')
|
|
alfred_instances.append(Alfred(unix_sockpath=alfredsock))
|
|
batman_instances.append(Batman(mesh_interface=batif,
|
|
alfred_sockpath=alfredsock))
|
|
except ValueError:
|
|
raise ValueError(
|
|
'Unparseable value "{0}" in --mesh parameter.'.
|
|
format(value))
|
|
|
|
# read nodedb state from nodes.json
|
|
try:
|
|
with open(nodes_fn, 'r') as nodedb_handle:
|
|
nodedb = json.load(nodedb_handle)
|
|
except (IOError, ValueError):
|
|
nodedb = {'nodes': dict()}
|
|
|
|
# flush nodedb if it uses the old format
|
|
if 'links' in nodedb:
|
|
nodedb = {'nodes': dict()}
|
|
|
|
# set version we're going to output
|
|
nodedb['version'] = NODES_VERSION
|
|
|
|
# update timestamp and assume all nodes are offline
|
|
nodedb['timestamp'] = now.isoformat()
|
|
for node_id, node in nodedb['nodes'].items():
|
|
node['flags']['online'] = False
|
|
|
|
# integrate alfred nodeinfo
|
|
for alfred in alfred_instances:
|
|
nodeinfo = validate_nodeinfos(alfred.nodeinfo())
|
|
nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
|
|
now, assume_online=True)
|
|
|
|
# integrate static aliases data
|
|
for aliases in params['aliases']:
|
|
with open(aliases, 'r') as f:
|
|
nodeinfo = validate_nodeinfos(json.load(f))
|
|
nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
|
|
now, assume_online=False)
|
|
|
|
# prepare statistics collection
|
|
nodes.reset_statistics(nodedb['nodes'])
|
|
|
|
# acquire gwl and visdata for each batman instance
|
|
mesh_info = []
|
|
for batman in batman_instances:
|
|
vd = batman.vis_data()
|
|
gwl = batman.gateway_list()
|
|
|
|
mesh_info.append((vd, gwl))
|
|
|
|
# update nodedb from batman-adv data
|
|
for vd, gwl in mesh_info:
|
|
nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vd)
|
|
nodes.import_vis_clientcount(nodedb['nodes'], vd)
|
|
nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
|
|
nodes.mark_gateways(nodedb['nodes'], gwl)
|
|
|
|
# get alfred statistics
|
|
for alfred in alfred_instances:
|
|
nodes.import_statistics(nodedb['nodes'], alfred.statistics())
|
|
|
|
# clear the nodedb from nodes that have not been online in $prune days
|
|
if params['prune']:
|
|
nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
|
|
|
|
# build nxnetworks graph from nodedb and visdata
|
|
batadv_graph = nx.DiGraph()
|
|
for vd, gwl in mesh_info:
|
|
graph.import_vis_data(batadv_graph, nodedb['nodes'], vd)
|
|
|
|
# force mac addresses to be vpn-link only (like gateways for example)
|
|
if params['vpn']:
|
|
graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
|
|
|
|
def extract_tunnel(nodes):
|
|
macs = set()
|
|
for id, node in nodes.items():
|
|
try:
|
|
for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
|
|
macs.add(mac)
|
|
except KeyError:
|
|
pass
|
|
|
|
return macs
|
|
|
|
graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
|
|
|
|
batadv_graph = graph.merge_nodes(batadv_graph)
|
|
batadv_graph = graph.to_undirected(batadv_graph)
|
|
|
|
# write processed data to dest dir
|
|
with open(nodes_fn, 'w') as f:
|
|
json.dump(nodedb, f)
|
|
|
|
graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
|
|
'version': GRAPH_VERSION}
|
|
|
|
with open(graph_fn, 'w') as f:
|
|
json.dump(graph_out, f)
|
|
|
|
with open(nodelist_fn, 'w') as f:
|
|
json.dump(export_nodelist(now, nodedb), f)
|
|
|
|
# optional rrd graphs (trigger with --rrd)
|
|
if params['rrd']:
|
|
if params['nodedb']:
|
|
rrd = RRD(params['nodedb'], os.path.join(params['dest_dir'], 'nodes'))
|
|
else:
|
|
script_directory = os.path.dirname(os.path.realpath(__file__))
|
|
rrd = RRD(os.path.join(script_directory, 'nodedb'),
|
|
os.path.join(params['dest_dir'], 'nodes'))
|
|
rrd.update_database(nodedb['nodes'])
|
|
if params['img']:
|
|
rrd.update_images()
|
|
|
|
def set_loglevel(nr):
|
|
"""
|
|
Umsetzen der Nummer auf einen für "logging" passenden Wert
|
|
Die Nummer kann ein Wert zwischen 0 - kein Logging und 5 - Debug sein
|
|
"""
|
|
level = (None, logging.CRITICAL, logging.ERROR, logging.WARNING,
|
|
logging.INFO, logging.DEBUG)
|
|
if nr > 5:
|
|
nr = 5
|
|
elif nr < 0:
|
|
nr = 0
|
|
return level[nr]
|
|
|
|
if __name__ == '__main__':
|
|
|
|
# get options from command line
|
|
parser = argparse.ArgumentParser(
|
|
description = "Collect data for ffmap: creates json files and "
|
|
"optional rrd data and graphs")
|
|
|
|
parser.add_argument('-a', '--aliases',
|
|
help='Read aliases from FILE',
|
|
nargs='+', default=[], metavar='FILE')
|
|
parser.add_argument('-m', '--mesh',
|
|
default=['bat0'], nargs='+',
|
|
help='Use given batman-adv mesh interface(s) (defaults '
|
|
'to bat0); specify alfred unix socket like '
|
|
'bat0:/run/alfred0.sock.')
|
|
parser.add_argument('-d', '--dest-dir', action='store',
|
|
help='Write output to destination directory',
|
|
required=False)
|
|
parser.add_argument('-c', '--config', action='store', metavar='FILE',
|
|
help='read configuration from FILE')
|
|
parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
|
|
help='Assume MAC addresses are part of vpn')
|
|
parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
|
|
help='Forget nodes offline for at least DAYS')
|
|
parser.add_argument('-r', '--with-rrd', dest='rrd', action='store_true',
|
|
default=False,
|
|
help='Enable the collection of RRD data')
|
|
parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
|
|
help='Directory for node RRD data files')
|
|
parser.add_argument('-i', '--with-img', dest='img', action='store_true',
|
|
default=False,
|
|
help='Enable the rendering of RRD graphs (cpu '
|
|
'intensive)')
|
|
|
|
options = vars(parser.parse_args())
|
|
if options['config']:
|
|
cfg['cfgfile'] = options['config']
|
|
|
|
config = configparser.ConfigParser(cfg)
|
|
if config.read(cfg['cfgfile']):
|
|
if not options['nodedb']:
|
|
options['nodedb'] = config.get('rrd', 'nodedb')
|
|
if not options['dest_dir']:
|
|
options['dest_dir'] = config.get('global', 'dest_dir')
|
|
if not options['rrd']:
|
|
options['rrd'] = config.getboolean('rrd', 'enabled')
|
|
if not options['img']:
|
|
options['img'] = config.getboolean('rrd', 'graphs')
|
|
cfg['logfile'] = config.get('global', 'logfile')
|
|
cfg['loglevel'] = config.getint('global', 'loglevel')
|
|
|
|
# At this point global configuration is available. Time to enable logging
|
|
# Logging is handled by the operating system, so use WatchedFileHandler
|
|
handler = logging.handlers.WatchedFileHandler(cfg['logfile'])
|
|
handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S'))
|
|
log = logging.getLogger()
|
|
log.addHandler(handler)
|
|
loglevel = set_loglevel(cfg['loglevel'])
|
|
if loglevel:
|
|
log.setLevel(loglevel)
|
|
else:
|
|
log.disabled = True
|
|
|
|
log.info("%s started" % sys.argv[0])
|
|
if os.path.isfile(cfg['cfgfile']):
|
|
log.info("using configuration from '%s'" % cfg['cfgfile'])
|
|
main(options)
|
|
log.info("%s finished" % sys.argv[0])
|