diff --git a/GlobalRRD.py b/GlobalRRD.py
index f3f3960..b3cf31a 100644
--- a/GlobalRRD.py
+++ b/GlobalRRD.py
@@ -2,6 +2,7 @@ import os
import subprocess
from RRD import RRD, DS, RRA
+
class GlobalRRD(RRD):
ds_list = [
# Number of nodes available
@@ -10,14 +11,17 @@ class GlobalRRD(RRD):
DS('clients', 'GAUGE', 120, 0, float('NaN')),
]
rra_list = [
- RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 60, 744), # 31 days of 1 hour samples
- RRA('AVERAGE', 0.5, 1440, 1780),# ~5 years of 1 day samples
+ # 2 hours of 1 minute samples
+ RRA('AVERAGE', 0.5, 1, 120),
+ # 31 days of 1 hour samples
+ RRA('AVERAGE', 0.5, 60, 744),
+ # ~5 years of 1 day samples
+ RRA('AVERAGE', 0.5, 1440, 1780),
]
def __init__(self, directory):
super().__init__(os.path.join(directory, "nodes.rrd"))
- self.ensureSanity(self.ds_list, self.rra_list, step=60)
+ self.ensure_sanity(self.ds_list, self.rra_list, step=60)
def update(self, nodeCount, clientCount):
super().update({'nodes': nodeCount, 'clients': clientCount})
@@ -30,6 +34,5 @@ class GlobalRRD(RRD):
'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
'LINE1:nodes#F00:nodes\\l',
'DEF:clients=' + self.filename + ':clients:AVERAGE',
- 'LINE2:clients#00F:clients',
- ]
+ 'LINE2:clients#00F:clients']
subprocess.check_output(args)
diff --git a/NodeRRD.py b/NodeRRD.py
index ca24c0d..a4ec092 100644
--- a/NodeRRD.py
+++ b/NodeRRD.py
@@ -2,19 +2,24 @@ import os
import subprocess
from RRD import RRD, DS, RRA
+
class NodeRRD(RRD):
ds_list = [
DS('upstate', 'GAUGE', 120, 0, 1),
DS('clients', 'GAUGE', 120, 0, float('NaN')),
]
rra_list = [
- RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
- RRA('AVERAGE', 0.5, 5, 1440), # 5 days of 5 minute samples
- RRA('AVERAGE', 0.5, 60, 720), # 30 days of 1 hour samples
- RRA('AVERAGE', 0.5, 720, 730), # 1 year of 12 hour samples
+ # 2 hours of 1 minute samples
+ RRA('AVERAGE', 0.5, 1, 120),
+ # 5 days of 5 minute samples
+ RRA('AVERAGE', 0.5, 5, 1440),
+ # 30 days of 1 hour samples
+ RRA('AVERAGE', 0.5, 60, 720),
+ # 1 year of 12 hour samples
+ RRA('AVERAGE', 0.5, 720, 730),
]
- def __init__(self, filename, node = None):
+ def __init__(self, filename, node=None):
"""
Create a new RRD for a given node.
@@ -22,12 +27,13 @@ class NodeRRD(RRD):
"""
self.node = node
super().__init__(filename)
- self.ensureSanity(self.ds_list, self.rra_list, step=60)
+ self.ensure_sanity(self.ds_list, self.rra_list, step=60)
@property
def imagename(self):
- return os.path.basename(self.filename).rsplit('.', 2)[0] + ".png"
+ return "{basename}.png".format(basename=os.path.basename(self.filename).rsplit('.', 2)[0])
+ # TODO: fix this, python does not support function overloading
def update(self):
super().update({'upstate': int(self.node['flags']['online']), 'clients': self.node['statistics']['clients']})
@@ -36,8 +42,8 @@ class NodeRRD(RRD):
Create a graph in the given directory. The file will be named
basename.png if the RRD file is named basename.rrd
"""
- args = ['rrdtool','graph', os.path.join(directory, self.imagename),
- '-s', '-' + timeframe ,
+ args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
+ '-s', '-' + timeframe,
'-w', '800',
'-h', '400',
'-l', '0',
@@ -48,6 +54,5 @@ class NodeRRD(RRD):
'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
'AREA:c#0F0:up\\l',
'AREA:d#F00:down\\l',
- 'LINE1:c#00F:clients connected\\l',
- ]
+ 'LINE1:c#00F:clients connected\\l']
subprocess.check_output(args)
diff --git a/RRD.py b/RRD.py
index 9bb87a0..799338c 100644
--- a/RRD.py
+++ b/RRD.py
@@ -1,19 +1,20 @@
import subprocess
import re
-import io
import os
-from tempfile import TemporaryFile
from operator import xor, eq
from functools import reduce
from itertools import starmap
import math
+
class RRDIncompatibleException(Exception):
"""
Is raised when an RRD doesn't have the desired definition and cannot be
upgraded to it.
"""
pass
+
+
class RRDOutdatedException(Exception):
"""
Is raised when an RRD doesn't have the desired definition, but can be
@@ -25,7 +26,8 @@ if not hasattr(__builtins__, "FileNotFoundError"):
class FileNotFoundError(Exception):
pass
-class RRD:
+
+class RRD(object):
"""
An RRD is a Round Robin Database, a database which forgets old data and
aggregates multiple records into new ones.
@@ -49,7 +51,7 @@ class RRD:
def _exec_rrdtool(self, cmd, *args, **kwargs):
pargs = ["rrdtool", cmd, self.filename]
- for k,v in kwargs.items():
+ for k, v in kwargs.items():
pargs.extend(["--" + k, str(v)])
pargs.extend(args)
subprocess.check_output(pargs)
@@ -57,7 +59,7 @@ class RRD:
def __init__(self, filename):
self.filename = filename
- def ensureSanity(self, ds_list, rra_list, **kwargs):
+ def ensure_sanity(self, ds_list, rra_list, **kwargs):
"""
Create or upgrade the RRD file if necessary to contain all DS in
ds_list. If it needs to be created, the RRAs in rra_list and any kwargs
@@ -65,13 +67,13 @@ class RRD:
database are NOT modified!
"""
try:
- self.checkSanity(ds_list)
+ self.check_sanity(ds_list)
except FileNotFoundError:
self.create(ds_list, rra_list, **kwargs)
except RRDOutdatedException:
self.upgrade(ds_list)
- def checkSanity(self, ds_list=()):
+ def check_sanity(self, ds_list=()):
"""
Check if the RRD file exists and contains (at least) the DS listed in
ds_list.
@@ -82,7 +84,8 @@ class RRD:
if set(ds_list) - set(info['ds'].values()) != set():
for ds in ds_list:
if ds.name in info['ds'] and ds.type != info['ds'][ds.name].type:
- raise RRDIncompatibleException("%s is %s but should be %s" % (ds.name, ds.type, info['ds'][ds.name].type))
+ raise RRDIncompatibleException("%s is %s but should be %s" %
+ (ds.name, ds.type, info['ds'][ds.name].type))
else:
raise RRDOutdatedException()
@@ -106,7 +109,7 @@ class RRD:
old_ds = info['ds'][ds.name]
if info['ds'][ds.name].type != ds.type:
raise RuntimeError('Cannot convert existing DS "%s" from type "%s" to "%s"' %
- (ds.name, old_ds.type, ds.type))
+ (ds.name, old_ds.type, ds.type))
ds.index = old_ds.index
new_ds[ds.index] = ds
else:
@@ -116,12 +119,11 @@ class RRD:
dump = subprocess.Popen(
["rrdtool", "dump", self.filename],
- stdout=subprocess.PIPE
- )
+ stdout=subprocess.PIPE)
+
restore = subprocess.Popen(
["rrdtool", "restore", "-", self.filename + ".new"],
- stdin=subprocess.PIPE
- )
+ stdin=subprocess.PIPE)
echo = True
ds_definitions = True
for line in dump.stdout:
@@ -143,16 +145,14 @@ class RRD:
%s
%i
- """ % (
- ds.name,
- ds.type,
- ds.args[0],
- ds.args[1],
- ds.args[2],
- ds.last_ds,
- ds.value,
- ds.unknown_sec)
- , "utf-8"))
+ """ % (ds.name,
+ ds.type,
+ ds.args[0],
+ ds.args[1],
+ ds.args[2],
+ ds.last_ds,
+ ds.value,
+ ds.unknown_sec), "utf-8"))
if b'' in line:
restore.stdin.write(added_ds_num*b"""
@@ -272,7 +272,8 @@ class RRD:
self._cached_info = info
return info
-class DS:
+
+class DS(object):
"""
DS stands for Data Source and represents one line of data points in a Round
Robin Database (RRD).
@@ -284,6 +285,7 @@ class DS:
last_ds = 'U'
value = 0
unknown_sec = 0
+
def __init__(self, name, dst, *args):
self.name = name
self.type = dst
@@ -293,7 +295,7 @@ class DS:
return "DS:%s:%s:%s" % (
self.name,
self.type,
- ":".join(map(str, self._nan_to_U_args()))
+ ":".join(map(str, self._nan_to_u_args()))
)
def __repr__(self):
@@ -305,22 +307,23 @@ class DS:
)
def __eq__(self, other):
- return all(starmap(eq, zip(self._compare_keys(), other._compare_keys())))
+ return all(starmap(eq, zip(self.compare_keys(), other.compare_keys())))
def __hash__(self):
- return reduce(xor, map(hash, self._compare_keys()))
+ return reduce(xor, map(hash, self.compare_keys()))
- def _nan_to_U_args(self):
+ def _nan_to_u_args(self):
return tuple(
'U' if type(arg) is float and math.isnan(arg)
else arg
for arg in self.args
)
- def _compare_keys(self):
- return (self.name, self.type, self._nan_to_U_args())
+ def compare_keys(self):
+ return self.name, self.type, self._nan_to_u_args()
-class RRA:
+
+class RRA(object):
def __init__(self, cf, *args):
self.cf = cf
self.args = args
diff --git a/alfred.py b/alfred.py
index f71b8da..3c08b78 100644
--- a/alfred.py
+++ b/alfred.py
@@ -2,44 +2,49 @@
import subprocess
import json
+
def _fetch(data_type):
- output = subprocess.check_output(["alfred-json", "-z", "-f", "json", "-r", str(data_type)])
- return json.loads(output.decode("utf-8")).values()
+ output = subprocess.check_output(["alfred-json", "-z", "-f", "json", "-r", str(data_type)])
+ return json.loads(output.decode("utf-8")).values()
+
def nodeinfo():
- return _fetch(158)
+ return _fetch(158)
+
def statistics():
- return _fetch(159)
+ return _fetch(159)
+
def vis():
- return _fetch(160)
+ return _fetch(160)
+
def aliases():
- alias = {}
- for node in nodeinfo():
- node_alias = {}
- if 'location' in node:
- try:
- node_alias['gps'] = str(node['location']['latitude']) + ' ' + str(node['location']['longitude'])
- except:
- pass
+ alias = {}
+ for node in nodeinfo():
+ node_alias = {}
+ if 'location' in node:
+ try:
+ node_alias['gps'] = str(node['location']['latitude']) + ' ' + str(node['location']['longitude'])
+ except KeyError:
+ pass
- try:
- node_alias['firmware'] = node['software']['firmware']['release']
- except KeyError:
- pass
+ try:
+ node_alias['firmware'] = node['software']['firmware']['release']
+ except KeyError:
+ pass
- try:
- node_alias['id'] = node['network']['mac']
- except KeyError:
- pass
+ try:
+ node_alias['id'] = node['network']['mac']
+ except KeyError:
+ pass
- if 'hostname' in node:
- node_alias['name'] = node['hostname']
- elif 'name' in node:
- node_alias['name'] = node['name']
- if len(node_alias):
- alias[node['network']['mac']] = node_alias
+ if 'hostname' in node:
+ node_alias['name'] = node['hostname']
+ elif 'name' in node:
+ node_alias['name'] = node['name']
+ if len(node_alias):
+ alias[node['network']['mac']] = node_alias
- return alias
+ return alias
diff --git a/backend.py b/backend.py
index 7544709..ff943ec 100755
--- a/backend.py
+++ b/backend.py
@@ -1,9 +1,11 @@
#!/usr/bin/env python3
-
+"""
+backend.py - ffmap-backend runner
+https://github.com/ffnord/ffmap-backend
+"""
import argparse
import json
import os
-import sys
import networkx as nx
from datetime import datetime
from networkx.readwrite import json_graph
@@ -11,91 +13,89 @@ from networkx.readwrite import json_graph
import alfred
import nodes
import graph
-from batman import batman
-from rrddb import rrd
+from batman import Batman
+from rrddb import RRD
-parser = argparse.ArgumentParser()
-parser.add_argument('-a', '--aliases',
- help='read aliases from FILE',
- default=[],
- action='append',
- metavar='FILE')
+def main(params):
+ if not params['mesh']:
+ params['mesh'] = ['bat0']
-parser.add_argument('-m', '--mesh', action='append',
- help='batman mesh interface')
+ nodes_fn = os.path.join(params['destination_directory'], 'nodes.json')
+ graph_fn = os.path.join(params['destination_directory'], 'graph.json')
-parser.add_argument('-d', '--destination-directory', action='store',
- help='destination directory for generated files',required=True)
+ now = datetime.utcnow().replace(microsecond=0)
-parser.add_argument('--vpn', action='append', metavar='MAC',
- help='assume MAC to be part of the VPN')
+ with open(nodes_fn, 'r') as nodedb_handle:
+ nodedb = json.load(nodedb_handle)
-parser.add_argument('--prune', metavar='DAYS',
- help='forget nodes offline for at least DAYS')
+ # flush nodedb if it uses the old format
+ if 'links' in nodedb:
+ nodedb = {'nodes': dict()}
-args = parser.parse_args()
+ nodedb['timestamp'] = now.isoformat()
-options = vars(args)
+ for node_id, node in nodedb['nodes'].items():
+ node['flags']['online'] = False
-if not options['mesh']:
- options['mesh'] = ['bat0']
+ nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(), now, assume_online=True)
-nodes_fn = os.path.join(options['destination_directory'], 'nodes.json')
-graph_fn = os.path.join(options['destination_directory'], 'graph.json')
+ for aliases in params['aliases']:
+ with open(aliases, 'r') as f:
+ nodes.import_nodeinfo(nodedb['nodes'], json.load(f), now, assume_online=False)
-now = datetime.utcnow().replace(microsecond=0)
+ nodes.reset_statistics(nodedb['nodes'])
+ nodes.import_statistics(nodedb['nodes'], alfred.statistics())
-try:
- nodedb = json.load(open(nodes_fn))
+ bm = list(map(lambda d: (d.vis_data(True), d.gateway_list()), map(Batman, params['mesh'])))
+ for vis_data, gateway_list in bm:
+ nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vis_data)
+ nodes.import_vis_clientcount(nodedb['nodes'], vis_data)
+ nodes.mark_vis_data_online(nodedb['nodes'], vis_data, now)
+ nodes.mark_gateways(nodedb['nodes'], gateway_list)
- # ignore if old format
- if 'links' in nodedb:
- raise
-except:
- nodedb = {'nodes': dict()}
+ if params['prune']:
+ nodes.prune_nodes(nodedb['nodes'], now, int(params['prune']))
-nodedb['timestamp'] = now.isoformat()
+ batadv_graph = nx.DiGraph()
+ for vis_data, gateway_list in bm:
+ graph.import_vis_data(batadv_graph, nodedb['nodes'], vis_data)
-for node_id, node in nodedb['nodes'].items():
- node['flags']['online'] = False
+ if params['vpn']:
+ graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
-nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(), now, assume_online=True)
+ batadv_graph = graph.merge_nodes(batadv_graph)
+ batadv_graph = graph.to_undirected(batadv_graph)
-for aliases in options['aliases']:
- with open(aliases, 'r') as f:
- nodes.import_nodeinfo(nodedb['nodes'], json.load(f), now, assume_online=False)
+ with open(nodes_fn, 'w') as f:
+ json.dump(nodedb, f)
-nodes.reset_statistics(nodedb['nodes'])
-nodes.import_statistics(nodedb['nodes'], alfred.statistics())
+ with open(graph_fn, 'w') as f:
+ json.dump({'batadv': json_graph.node_link_data(batadv_graph)}, f)
-bm = list(map(lambda d: (d.vis_data(True), d.gateway_list()), map(batman, options['mesh'])))
-for vis_data, gateway_list in bm:
- nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vis_data)
- nodes.import_vis_clientcount(nodedb['nodes'], vis_data)
- nodes.mark_vis_data_online(nodedb['nodes'], vis_data, now)
- nodes.mark_gateways(nodedb['nodes'], gateway_list)
+ scriptdir = os.path.dirname(os.path.realpath(__file__))
+ rrd = RRD(scriptdir + '/nodedb/', params['destination_directory'] + '/nodes')
+ rrd.update_database(nodedb['nodes'])
+ rrd.update_images()
-if options['prune']:
- nodes.prune_nodes(nodedb['nodes'], now, int(options['prune']))
-batadv_graph = nx.DiGraph()
-for vis_data, gateway_list in bm:
- graph.import_vis_data(batadv_graph, nodedb['nodes'], vis_data)
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
-if options['vpn']:
- graph.mark_vpn(batadv_graph, frozenset(options['vpn']))
+ parser.add_argument('-a', '--aliases',
+ help='read aliases from FILE',
+ default=[], action='append',
+ metavar='FILE')
+ parser.add_argument('-m', '--mesh', action='append',
+ help='batman mesh interface')
+ parser.add_argument('-d', '--destination-directory', action='store',
+ help='destination directory for generated files',
+ required=True)
+ parser.add_argument('--vpn', action='append', metavar='MAC',
+ help='assume MAC to be part of the VPN')
+ parser.add_argument('--prune', metavar='DAYS',
+ help='forget nodes offline for at least DAYS')
-batadv_graph = graph.merge_nodes(batadv_graph)
-batadv_graph = graph.to_undirected(batadv_graph)
+ options = vars(parser.parse_args())
-with open(nodes_fn, 'w') as f:
- json.dump(nodedb, f)
-
-with open(graph_fn, 'w') as f:
- json.dump({'batadv': json_graph.node_link_data(batadv_graph)}, f)
-
-scriptdir = os.path.dirname(os.path.realpath(__file__))
-rrd = rrd(scriptdir + '/nodedb/', options['destination_directory'] + '/nodes')
-rrd.update_database(nodedb['nodes'])
-rrd.update_images()
+ main(options)
diff --git a/batman.py b/batman.py
index 94229ad..86ad4fe 100644
--- a/batman.py
+++ b/batman.py
@@ -1,82 +1,90 @@
-#!/usr/bin/env python3
import subprocess
import json
import re
-class batman:
- """ Bindings for B.A.T.M.A.N. advanced batctl tool
- """
- def __init__(self, mesh_interface = "bat0"):
- self.mesh_interface = mesh_interface
- def vis_data(self,batadv_vis=False):
- vds = self.vis_data_batctl_legacy()
- if batadv_vis:
- vds += self.vis_data_batadv_vis()
- return vds
-
- def vis_data_helper(self,lines):
- vd = []
- for line in lines:
- try:
- utf8_line = line.decode("utf-8")
- vd.append(json.loads(utf8_line))
- except e:
- pass
- return vd
-
- def vis_data_batctl_legacy(self):
- """ Parse "batctl -m vd json -n" into an array of dictionaries.
+class Batman(object):
"""
- output = subprocess.check_output(["batctl","-m",self.mesh_interface,"vd","json","-n"])
- lines = output.splitlines()
- vds = self.vis_data_helper(lines)
- return vds
-
- def vis_data_batadv_vis(self):
- """ Parse "batadv-vis -i -f json" into an array of dictionaries.
+ Bindings for B.A.T.M.A.N. Advanced
+ commandline interface "batctl"
"""
- output = subprocess.check_output(["batadv-vis","-i",self.mesh_interface,"-f","json"])
- lines = output.splitlines()
- return self.vis_data_helper(lines)
+ def __init__(self, mesh_interface='bat0'):
+ self.mesh_interface = mesh_interface
- def gateway_list(self):
- """ Parse "batctl -m gwl -n" into an array of dictionaries.
- """
- output = subprocess.check_output(["batctl","-m",self.mesh_interface,"gwl","-n"])
- output_utf8 = output.decode("utf-8")
- lines = output_utf8.splitlines()
+ def vis_data(self, batadv_vis=False):
+ vds = self.vis_data_batctl_legacy()
+ if batadv_vis:
+ vds += self.vis_data_batadv_vis()
+ return vds
- own_mac = re.match(r"^.*MainIF/MAC: [^/]+/([0-9a-f:]+).*$", lines[0]).group(1)
+ @staticmethod
+ def vis_data_helper(lines):
+ vd_tmp = []
+ for line in lines:
+ try:
+ utf8_line = line.decode('utf-8')
+ vd_tmp.append(json.loads(utf8_line))
+ except UnicodeDecodeError:
+ pass
+ return vd_tmp
- gw = []
- gw_mode = self.gateway_mode()
- if gw_mode['mode'] == 'server':
- gw.append(own_mac)
+ def vis_data_batctl_legacy(self):
+ """
+ Parse "batctl -m vd json -n" into an array of dictionaries.
+ """
+ output = subprocess.check_output(['batctl', '-m', self.mesh_interface, 'vd', 'json', '-n'])
+ lines = output.splitlines()
+ vds = self.vis_data_helper(lines)
+ return vds
- for line in lines:
- gw_line = re.match(r"^(?:=>)? +([0-9a-f:]+) ", line)
- if gw_line:
- gw.append(gw_line.group(1))
+ def vis_data_batadv_vis(self):
+ """
+ Parse "batadv-vis -i -f json" into an array of dictionaries.
+ """
+ output = subprocess.check_output(['batadv-vis', '-i', self.mesh_interface, '-f', 'json'])
+ lines = output.splitlines()
+ return self.vis_data_helper(lines)
- return gw
+ def gateway_list(self):
+ """
+ Parse "batctl -m gwl -n" into an array of dictionaries.
+ """
+ output = subprocess.check_output(['batctl', '-m', self.mesh_interface, 'gwl', '-n'])
+ output_utf8 = output.decode('utf-8')
+ lines = output_utf8.splitlines()
- def gateway_mode(self):
- """ Parse "batctl -m gw"
- """
- output = subprocess.check_output(["batctl","-m",self.mesh_interface,"gw"])
- elements = output.decode("utf-8").split()
- mode = elements[0]
- if mode == "server":
- return {'mode': 'server', 'bandwidth': elements[3]}
- else:
- return {'mode': mode}
+ own_mac = re.match(r"^.*MainIF/MAC: [^/]+/([0-9a-f:]+).*$", lines[0]).group(1)
+
+ gateways = []
+ gw_mode = self.gateway_mode()
+ if gw_mode['mode'] == 'server':
+ gateways.append(own_mac)
+
+ for line in lines:
+ gw_line = re.match(r"^(?:=>)? +([0-9a-f:]+) ", line)
+ if gw_line:
+ gateways.append(gw_line.group(1))
+
+ return gateways
+
+ def gateway_mode(self):
+ """
+ Parse "batctl -m gw"
+ """
+ output = subprocess.check_output(['batctl', '-m', self.mesh_interface, 'gw'])
+ elements = output.decode("utf-8").split()
+ mode = elements[0]
+ if mode == 'server':
+ return {'mode': 'server',
+ 'bandwidth': elements[3]}
+ else:
+ return {'mode': mode}
if __name__ == "__main__":
- bc = batman()
- vd = bc.vis_data()
- gw = bc.gateway_list()
- for x in vd:
- print(x)
- print(gw)
- print(bc.gateway_mode())
+ bc = Batman()
+ vd = bc.vis_data()
+ gw = bc.gateway_list()
+ for x in vd:
+ print(x)
+ print(gw)
+ print(bc.gateway_mode())
diff --git a/graph.py b/graph.py
index b6e86aa..460e327 100644
--- a/graph.py
+++ b/graph.py
@@ -1,66 +1,74 @@
import networkx as nx
-from copy import deepcopy
from functools import reduce
from itertools import chain
from nodes import build_mac_table
-def import_vis_data(graph, nodes, vis_data):
- macs = build_mac_table(nodes)
- nodes_a = map(lambda d: 2*[d['primary']], filter(lambda d: 'primary' in d, vis_data))
- nodes_b = map(lambda d: [d['secondary'], d['of']], filter(lambda d: 'secondary' in d, vis_data))
- graph.add_nodes_from(map(lambda a, b: (a, dict(primary=b, node_id=macs.get(b))), *zip(*chain(nodes_a, nodes_b))))
- edges = filter(lambda d: 'neighbor' in d, vis_data)
- graph.add_edges_from(map(lambda d: (d['router'], d['neighbor'], dict(tq=float(d['label']))), edges))
+def import_vis_data(graph, nodes, vis_data):
+ macs = build_mac_table(nodes)
+ nodes_a = map(lambda d: 2*[d['primary']], filter(lambda d: 'primary' in d, vis_data))
+ nodes_b = map(lambda d: [d['secondary'], d['of']], filter(lambda d: 'secondary' in d, vis_data))
+ graph.add_nodes_from(map(lambda a, b: (a, dict(primary=b, node_id=macs.get(b))), *zip(*chain(nodes_a, nodes_b))))
+
+ edges = filter(lambda d: 'neighbor' in d, vis_data)
+ graph.add_edges_from(map(lambda d: (d['router'], d['neighbor'], dict(tq=float(d['label']))), edges))
+
def mark_vpn(graph, vpn_macs):
- components = map(frozenset, nx.weakly_connected_components(graph))
- components = filter(vpn_macs.intersection, components)
- nodes = reduce(lambda a, b: a | b, components, set())
+ components = map(frozenset, nx.weakly_connected_components(graph))
+ components = filter(vpn_macs.intersection, components)
+ nodes = reduce(lambda a, b: a | b, components, set())
+
+ for node in nodes:
+ for k, v in graph[node].items():
+ v['vpn'] = True
- for node in nodes:
- for k, v in graph[node].items():
- v['vpn'] = True
def to_multigraph(graph):
- def f(a):
- node = graph.node[a]
- return node['primary'] if node else a
+ def f(a):
+ node = graph.node[a]
+ return node['primary'] if node else a
- G = nx.MultiDiGraph()
- map_node = lambda node, data: (data['primary'], dict(node_id=data['node_id'])) if data else (node, dict())
- G.add_nodes_from(map(map_node, *zip(*graph.nodes_iter(data=True))))
- G.add_edges_from(map(lambda a, b, data: (f(a), f(b), data), *zip(*graph.edges_iter(data=True))))
+ def map_node(node, data):
+ return (data['primary'], dict(node_id=data['node_id'])) if data else (node, dict())
+
+ digraph = nx.MultiDiGraph()
+ digraph.add_nodes_from(map(map_node, *zip(*graph.nodes_iter(data=True))))
+ digraph.add_edges_from(map(lambda a, b, data: (f(a), f(b), data), *zip(*graph.edges_iter(data=True))))
+
+ return digraph
- return G
def merge_nodes(graph):
- def merge_edges(data):
- tq = min(map(lambda d: d['tq'], data))
- vpn = all(map(lambda d: d.get('vpn', False), data))
- return dict(tq=tq, vpn=vpn)
+ def merge_edges(data):
+ tq = min(map(lambda d: d['tq'], data))
+ vpn = all(map(lambda d: d.get('vpn', False), data))
+ return dict(tq=tq, vpn=vpn)
- G = to_multigraph(graph)
- H = nx.DiGraph()
- H.add_nodes_from(G.nodes_iter(data=True))
- edges = chain.from_iterable([[(e, d, merge_edges(G[e][d].values())) for d in G[e]] for e in G])
- H.add_edges_from(edges)
+ multigraph = to_multigraph(graph)
+ digraph = nx.DiGraph()
+ digraph.add_nodes_from(multigraph.nodes_iter(data=True))
+ edges = chain.from_iterable([[(e, d, merge_edges(multigraph[e][d].values()))
+ for d in multigraph[e]] for e in multigraph])
+ digraph.add_edges_from(edges)
+
+ return digraph
- return H
def to_undirected(graph):
- G = nx.MultiGraph()
- G.add_nodes_from(graph.nodes_iter(data=True))
- G.add_edges_from(graph.edges_iter(data=True))
+ multigraph = nx.MultiGraph()
+ multigraph.add_nodes_from(graph.nodes_iter(data=True))
+ multigraph.add_edges_from(graph.edges_iter(data=True))
- def merge_edges(data):
- tq = max(map(lambda d: d['tq'], data))
- vpn = all(map(lambda d: d.get('vpn', False), data))
- return dict(tq=tq, vpn=vpn, bidirect=len(data) == 2)
+ def merge_edges(data):
+ tq = max(map(lambda d: d['tq'], data))
+ vpn = all(map(lambda d: d.get('vpn', False), data))
+ return dict(tq=tq, vpn=vpn, bidirect=len(data) == 2)
- H = nx.Graph()
- H.add_nodes_from(G.nodes_iter(data=True))
- edges = chain.from_iterable([[(e, d, merge_edges(G[e][d].values())) for d in G[e]] for e in G])
- H.add_edges_from(edges)
+ graph = nx.Graph()
+ graph.add_nodes_from(multigraph.nodes_iter(data=True))
+ edges = chain.from_iterable([[(e, d, merge_edges(multigraph[e][d].values()))
+ for d in multigraph[e]] for e in multigraph])
+ graph.add_edges_from(edges)
- return H
+ return graph
diff --git a/nodes.py b/nodes.py
index 61949e1..d2b9df8 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2,128 +2,137 @@ from collections import Counter, defaultdict
from datetime import datetime
from functools import reduce
-def build_mac_table(nodes):
- macs = dict()
- for node_id, node in nodes.items():
- try:
- for mac in node['nodeinfo']['network']['mesh_interfaces']:
- macs[mac] = node_id
- except KeyError:
- pass
- return macs
+def build_mac_table(nodes):
+ macs = dict()
+ for node_id, node in nodes.items():
+ try:
+ for mac in node['nodeinfo']['network']['mesh_interfaces']:
+ macs[mac] = node_id
+ except KeyError:
+ pass
+ return macs
+
def prune_nodes(nodes, now, days):
- prune = []
- for node_id, node in nodes.items():
- if not 'lastseen' in node:
- prune.append(node_id)
- continue
+ prune = []
+ for node_id, node in nodes.items():
+ if 'lastseen' not in node:
+ prune.append(node_id)
+ continue
- lastseen = datetime.strptime(node['lastseen'], '%Y-%m-%dT%H:%M:%S')
- delta = (now - lastseen).seconds
+ lastseen = datetime.strptime(node['lastseen'], '%Y-%m-%dT%H:%M:%S')
+ delta = (now - lastseen).seconds
- if delta >= days * 86400:
- prune.append(node_id)
+ if delta >= days * 86400:
+ prune.append(node_id)
+
+ for prune_key in prune:
+ del nodes[prune_key]
- for node_id in prune:
- del nodes[node_id]
def mark_online(node, now):
- node['lastseen'] = now.isoformat()
- node.setdefault('firstseen', now.isoformat())
- node['flags']['online'] = True
+ node['lastseen'] = now.isoformat()
+ node.setdefault('firstseen', now.isoformat())
+ node['flags']['online'] = True
+
def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
- for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
- node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
- node['nodeinfo'] = nodeinfo
- node['flags']['online'] = False
- node['flags']['gateway'] = False
+ for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
+ node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
+ node['nodeinfo'] = nodeinfo
+ node['flags']['online'] = False
+ node['flags']['gateway'] = False
+
+ if assume_online:
+ mark_online(node, now)
- if assume_online:
- mark_online(node, now)
def reset_statistics(nodes):
- for node in nodes.values():
- node['statistics'] = { 'clients': 0 }
+ for node in nodes.values():
+ node['statistics'] = {'clients': 0}
+
def import_statistics(nodes, statistics):
- def add(node, statistics, target, source, f=lambda d: d):
- try:
- node['statistics'][target] = f(reduce(dict.__getitem__, source, statistics))
- except (KeyError,TypeError):
- pass
+ def add(node, statistics, target, source, f=lambda d: d):
+ try:
+ node['statistics'][target] = f(reduce(dict.__getitem__, source, statistics))
+ except (KeyError, TypeError):
+ pass
+
+ macs = build_mac_table(nodes)
+ statistics = filter(lambda d: 'node_id' in d, statistics)
+ statistics = filter(lambda d: d['node_id'] in nodes, statistics)
+ for node, statistics in map(lambda d: (nodes[d['node_id']], d), statistics):
+ add(node, statistics, 'clients', ['clients', 'total'])
+ add(node, statistics, 'gateway', ['gateway'], lambda d: macs.get(d, d))
+ add(node, statistics, 'uptime', ['uptime'])
+ add(node, statistics, 'loadavg', ['loadavg'])
+ add(node, statistics, 'memory_usage', ['memory'], lambda d: 1 - d['free'] / d['total'])
+ add(node, statistics, 'rootfs_usage', ['rootfs_usage'])
- macs = build_mac_table(nodes)
- statistics = filter(lambda d: 'node_id' in d, statistics)
- statistics = filter(lambda d: d['node_id'] in nodes, statistics)
- for node, statistics in map(lambda d: (nodes[d['node_id']], d), statistics):
- add(node, statistics, 'clients', ['clients', 'total'])
- add(node, statistics, 'gateway', ['gateway'], lambda d: macs.get(d, d))
- add(node, statistics, 'uptime', ['uptime'])
- add(node, statistics, 'loadavg', ['loadavg'])
- add(node, statistics, 'memory_usage', ['memory'], lambda d: 1 - d['free'] / d['total'])
- add(node, statistics, 'rootfs_usage', ['rootfs_usage'])
def import_mesh_ifs_vis_data(nodes, vis_data):
- macs = build_mac_table(nodes)
+ macs = build_mac_table(nodes)
- mesh_ifs = defaultdict(lambda: set())
- for line in filter(lambda d: 'secondary' in d, vis_data):
- primary = line['of']
- mesh_ifs[primary].add(primary)
- mesh_ifs[primary].add(line['secondary'])
+ mesh_ifs = defaultdict(lambda: set())
+ for line in filter(lambda d: 'secondary' in d, vis_data):
+ primary = line['of']
+ mesh_ifs[primary].add(primary)
+ mesh_ifs[primary].add(line['secondary'])
- def if_to_node(ifs):
- a = filter(lambda d: d in macs, ifs)
- a = map(lambda d: nodes[macs[d]], a)
- try:
- return (next(a), ifs)
- except StopIteration:
- return None
+ def if_to_node(ifs):
+ a = filter(lambda d: d in macs, ifs)
+ a = map(lambda d: nodes[macs[d]], a)
+ try:
+ return next(a), ifs
+ except StopIteration:
+ return None
- mesh_nodes = filter(lambda d: d, map(if_to_node, mesh_ifs.values()))
+ mesh_nodes = filter(lambda d: d, map(if_to_node, mesh_ifs.values()))
- for v in mesh_nodes:
- node = v[0]
+ for v in mesh_nodes:
+ node = v[0]
- try:
- mesh_ifs = set(node['nodeinfo']['network']['mesh_interfaces'])
- except KeyError:
- mesh_ifs = set()
+ try:
+ mesh_ifs = set(node['nodeinfo']['network']['mesh_interfaces'])
+ except KeyError:
+ mesh_ifs = set()
+
+ node['nodeinfo']['network']['mesh_interfaces'] = list(mesh_ifs | v[1])
- node['nodeinfo']['network']['mesh_interfaces'] = list(mesh_ifs | v[1])
def import_vis_clientcount(nodes, vis_data):
- macs = build_mac_table(nodes)
- data = filter(lambda d: d.get('label', None) == 'TT', vis_data)
- data = filter(lambda d: d['router'] in macs, data)
- data = map(lambda d: macs[d['router']], data)
+ macs = build_mac_table(nodes)
+ data = filter(lambda d: d.get('label', None) == 'TT', vis_data)
+ data = filter(lambda d: d['router'] in macs, data)
+ data = map(lambda d: macs[d['router']], data)
+
+ for node_id, clientcount in Counter(data).items():
+ nodes[node_id]['statistics'].setdefault('clients', clientcount)
- for node_id, clientcount in Counter(data).items():
- nodes[node_id]['statistics'].setdefault('clients', clientcount)
def mark_gateways(nodes, gateways):
- macs = build_mac_table(nodes)
- gateways = filter(lambda d: d in macs, gateways)
+ macs = build_mac_table(nodes)
+ gateways = filter(lambda d: d in macs, gateways)
+
+ for node in map(lambda d: nodes[macs[d]], gateways):
+ node['flags']['gateway'] = True
- for node in map(lambda d: nodes[macs[d]], gateways):
- node['flags']['gateway'] = True
def mark_vis_data_online(nodes, vis_data, now):
- macs = build_mac_table(nodes)
+ macs = build_mac_table(nodes)
- online = set()
- for line in vis_data:
- if 'primary' in line:
- online.add(line['primary'])
- elif 'secondary' in line:
- online.add(line['secondary'])
- elif 'gateway' in line:
- # This matches clients' MACs.
- # On pre-Gluon nodes the primary MAC will be one of it.
- online.add(line['gateway'])
+ online = set()
+ for line in vis_data:
+ if 'primary' in line:
+ online.add(line['primary'])
+ elif 'secondary' in line:
+ online.add(line['secondary'])
+ elif 'gateway' in line:
+ # This matches clients' MACs.
+ # On pre-Gluon nodes the primary MAC will be one of it.
+ online.add(line['gateway'])
- for mac in filter(lambda d: d in macs, online):
- mark_online(nodes[macs[mac]], now)
+ for mac in filter(lambda d: d in macs, online):
+ mark_online(nodes[macs[mac]], now)
diff --git a/rrddb.py b/rrddb.py
index 2fccff4..b023e6b 100644
--- a/rrddb.py
+++ b/rrddb.py
@@ -1,50 +1,50 @@
#!/usr/bin/env python3
-import subprocess
import time
import os
from GlobalRRD import GlobalRRD
from NodeRRD import NodeRRD
-class rrd:
- def __init__( self
- , databaseDirectory
- , imagePath
- , displayTimeGlobal = "7d"
- , displayTimeNode = "1d"
- ):
- self.dbPath = databaseDirectory
- self.globalDb = GlobalRRD(self.dbPath)
- self.imagePath = imagePath
- self.displayTimeGlobal = displayTimeGlobal
- self.displayTimeNode = displayTimeNode
- self.currentTimeInt = (int(time.time())/60)*60
- self.currentTime = str(self.currentTimeInt)
+class RRD(object):
+ def __init__(self,
+ database_directory,
+ image_path,
+ display_time_global="7d",
+ display_time_node="1d"):
- try:
- os.stat(self.imagePath)
- except:
- os.mkdir(self.imagePath)
+ self.dbPath = database_directory
+ self.globalDb = GlobalRRD(self.dbPath)
+ self.imagePath = image_path
+ self.displayTimeGlobal = display_time_global
+ self.displayTimeNode = display_time_node
- def update_database(self, nodes):
- online_nodes = dict(filter(lambda d: d[1]['flags']['online'], nodes.items()))
- client_count = sum(map(lambda d: d['statistics']['clients'], online_nodes.values()))
+ self.currentTimeInt = (int(time.time())/60)*60
+ self.currentTime = str(self.currentTimeInt)
- self.globalDb.update(len(online_nodes), client_count)
- for node_id, node in online_nodes.items():
- rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
- rrd.update()
+ try:
+ os.stat(self.imagePath)
+ except OSError:
+ os.mkdir(self.imagePath)
- def update_images(self):
- self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"), self.displayTimeGlobal)
+ def update_database(self, nodes):
+ online_nodes = dict(filter(lambda d: d[1]['flags']['online'], nodes.items()))
+ client_count = sum(map(lambda d: d['statistics']['clients'], online_nodes.values()))
- nodeDbFiles = os.listdir(self.dbPath)
+ self.globalDb.update(len(online_nodes), client_count)
+ for node_id, node in online_nodes.items():
+ rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
+ rrd.update()
- for fileName in nodeDbFiles:
- if not os.path.isfile(os.path.join(self.dbPath, fileName)):
- continue
+ def update_images(self):
+ self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"), self.displayTimeGlobal)
- nodeName = os.path.basename(fileName).split('.')
- if nodeName[1] == 'rrd' and not nodeName[0] == "nodes":
- rrd = NodeRRD(os.path.join(self.dbPath, fileName))
- rrd.graph(self.imagePath, self.displayTimeNode)
+ nodedb_files = os.listdir(self.dbPath)
+
+ for file_name in nodedb_files:
+ if not os.path.isfile(os.path.join(self.dbPath, file_name)):
+ continue
+
+ node_name = os.path.basename(file_name).split('.')
+ if node_name[1] == 'rrd' and not node_name[0] == "nodes":
+ rrd = NodeRRD(os.path.join(self.dbPath, file_name))
+ rrd.graph(self.imagePath, self.displayTimeNode)