thooge
/
ffmap-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: thooge:master
Branches Tags
thooge:master
thooge:dev
thooge:respondd
thooge:detailed-rrds
thooge:influxdb
thooge:legacy
thooge:rewrite
thooge:alfred-data
thooge:v2
thooge:v1
..
compare: thooge:influxdb
Branches Tags
thooge:master
thooge:dev
thooge:respondd
thooge:detailed-rrds
thooge:influxdb
thooge:legacy
thooge:rewrite
thooge:alfred-data
thooge:v2
thooge:v1

1 Commits
master ... influxdb

Author SHA1 Message Date
Nils Schneider 820da07451 remove RRD support 2015-04-23 19:16:07 +02:00
17 changed files with 27 additions and 1283 deletions
Show Stats Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,6 +1,3 @@
# backups
*~
# script-generated
aliases*.json
nodedb/

22
README.md
View File

@ -18,13 +18,6 @@ For the script's regular execution add the following to the crontab:
* * * * * backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a d2:d0:93:63:f7:da
</pre>
# Dependencies
- Python 3
- Python 3 Package [Networkx](https://networkx.github.io/)
- [alfred-json](https://github.com/tcatm/alfred-json)
- rrdtool (if run with `--with-rrd`)
# Running as unprivileged user
Some information collected by ffmap-backend requires access to specific system resources.
@ -79,21 +72,6 @@ will prefix `sudo` where necessary.
- online
- gateway
## Old data format
If you want to still use the old [ffmap-d3](https://github.com/ffnord/ffmap-d3)
front end, you can use the file `ffmap-d3.jq` to convert the new output to the
old one:
```
jq -n -f ffmap-d3.jq \
--argfile nodes nodedb/nodes.json \
--argfile graph nodedb/graph.json \
> nodedb/ffmap-d3.json
```
Then point your ffmap-d3 instance to the `ffmap-d3.json` file.
# Removing owner information
If you'd like to redact information about the node owner from `nodes.json`,

24
aliases.json_sample
View File

@ -7,30 +7,18 @@
"latitude": 53.86
},
"network": {
"mesh": {
"bat0": {
"interfaces": {
"tunnel": [
"00:25:86:e6:f1:bf"
]
}
}
}
"mesh_interfaces": [
"00:25:86:e6:f1:bf"
]
}
},
{
"node_id": "gw1",
"hostname": "burgtor",
"network": {
"mesh": {
"bat0": {
"interfaces": {
"tunnel": [
"52:54:00:f3:62:d9"
]
}
}
}
"mesh_interfaces": [
"52:54:00:f3:62:d9"
]
}
}
]

132
backend.py
View File

@ -2,20 +2,11 @@
"""
backend.py - ffmap-backend runner
https://github.com/ffnord/ffmap-backend
Erweiterte Version von Freifunk Pinneberg
- Graphiken aus RRD-Daten nur auf Anforderung erzeugen
- Verzeichnis für die RRD-Nodedb als Kommandozeilenparameter
- Statistikerzeugung korrigiert: Initialisierung und Befüllung
zu passenden Zeitpunkten
"""
import argparse
import configparser
import json
import os
import sys
import logging, logging.handlers
from datetime import datetime
import networkx as nx
@ -24,25 +15,11 @@ from networkx.readwrite import json_graph
from lib import graph, nodes
from lib.alfred import Alfred
from lib.batman import Batman
from lib.rrddb import RRD
from lib.nodelist import export_nodelist
from lib.validate import validate_nodeinfos
NODES_VERSION = 1
GRAPH_VERSION = 1
cfg = {
'cfgfile': '/etc/ffmap/ffmap-test.cfg',
'logfile': '/var/log/ffmap.log',
'loglevel': 5,
'dest_dir': '/var/lib/ffmap/mapdata',
'aliases': [],
'prune': 0,
'nodedb': '/var/lib/ffmap/nodedb',
'rrd_data': False,
'rrd_graphs': False,
'redis': False
}
def main(params):
os.makedirs(params['dest_dir'], exist_ok=True)
@ -77,11 +54,11 @@ def main(params):
'Unparseable value "{0}" in --mesh parameter.'.
format(value))
# read nodedb state from nodes.json
# read nodedb state from node.json
try:
with open(nodes_fn, 'r') as nodedb_handle:
nodedb = json.load(nodedb_handle)
except (IOError, ValueError):
except IOError:
nodedb = {'nodes': dict()}
# flush nodedb if it uses the old format
@ -98,19 +75,18 @@ def main(params):
# integrate alfred nodeinfo
for alfred in alfred_instances:
nodeinfo = validate_nodeinfos(alfred.nodeinfo())
nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(),
now, assume_online=True)
# integrate static aliases data
for aliases in params['aliases']:
with open(aliases, 'r') as f:
nodeinfo = validate_nodeinfos(json.load(f))
nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
now, assume_online=False)
# prepare statistics collection
nodes.reset_statistics(nodedb['nodes'])
for alfred in alfred_instances:
nodes.import_statistics(nodedb['nodes'], alfred.statistics())
# acquire gwl and visdata for each batman instance
mesh_info = []
@ -127,10 +103,6 @@ def main(params):
nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
nodes.mark_gateways(nodedb['nodes'], gwl)
# get alfred statistics
for alfred in alfred_instances:
nodes.import_statistics(nodedb['nodes'], alfred.statistics())
# clear the nodedb from nodes that have not been online in $prune days
if params['prune']:
nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
@ -144,19 +116,6 @@ def main(params):
if params['vpn']:
graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
def extract_tunnel(nodes):
macs = set()
for id, node in nodes.items():
try:
for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
macs.add(mac)
except KeyError:
pass
return macs
graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
batadv_graph = graph.merge_nodes(batadv_graph)
batadv_graph = graph.to_undirected(batadv_graph)
@ -173,97 +132,26 @@ def main(params):
with open(nodelist_fn, 'w') as f:
json.dump(export_nodelist(now, nodedb), f)
# optional rrd graphs (trigger with --rrd)
if params['rrd']:
if params['nodedb']:
rrd = RRD(params['nodedb'], os.path.join(params['dest_dir'], 'nodes'))
else:
script_directory = os.path.dirname(os.path.realpath(__file__))
rrd = RRD(os.path.join(script_directory, 'nodedb'),
os.path.join(params['dest_dir'], 'nodes'))
rrd.update_database(nodedb['nodes'])
if params['img']:
rrd.update_images()
def set_loglevel(nr):
"""
Umsetzen der Nummer auf einen für "logging" passenden Wert
Die Nummer kann ein Wert zwischen 0 - kein Logging und 5 - Debug sein
"""
level = (None, logging.CRITICAL, logging.ERROR, logging.WARNING,
logging.INFO, logging.DEBUG)
if nr > 5:
nr = 5
elif nr < 0:
nr = 0
return level[nr]
if __name__ == '__main__':
# get options from command line
parser = argparse.ArgumentParser(
description = "Collect data for ffmap: creates json files and "
"optional rrd data and graphs")
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--aliases',
help='Read aliases from FILE',
nargs='+', default=[], metavar='FILE')
parser.add_argument('-m', '--mesh',
default=['bat0'], nargs='+',
help='Use given batman-adv mesh interface(s) (defaults '
help='Use given batman-adv mesh interface(s) (defaults'
'to bat0); specify alfred unix socket like '
'bat0:/run/alfred0.sock.')
parser.add_argument('-d', '--dest-dir', action='store',
help='Write output to destination directory',
required=False)
parser.add_argument('-c', '--config', action='store', metavar='FILE',
help='read configuration from FILE')
required=True)
parser.add_argument('-V', '--vpn', nargs='+', metavar='MAC',
help='Assume MAC addresses are part of vpn')
parser.add_argument('-p', '--prune', metavar='DAYS', type=int,
help='Forget nodes offline for at least DAYS')
parser.add_argument('-r', '--with-rrd', dest='rrd', action='store_true',
default=False,
help='Enable the collection of RRD data')
parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
help='Directory for node RRD data files')
parser.add_argument('-i', '--with-img', dest='img', action='store_true',
default=False,
help='Enable the rendering of RRD graphs (cpu '
'intensive)')
help='forget nodes offline for at least DAYS')
options = vars(parser.parse_args())
if options['config']:
cfg['cfgfile'] = options['config']
config = configparser.ConfigParser(cfg)
if config.read(cfg['cfgfile']):
if not options['nodedb']:
options['nodedb'] = config.get('rrd', 'nodedb')
if not options['dest_dir']:
options['dest_dir'] = config.get('global', 'dest_dir')
if not options['rrd']:
options['rrd'] = config.getboolean('rrd', 'enabled')
if not options['img']:
options['img'] = config.getboolean('rrd', 'graphs')
cfg['logfile'] = config.get('global', 'logfile')
cfg['loglevel'] = config.getint('global', 'loglevel')
# At this point global configuration is available. Time to enable logging
# Logging is handled by the operating system, so use WatchedFileHandler
handler = logging.handlers.WatchedFileHandler(cfg['logfile'])
handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'))
log = logging.getLogger()
log.addHandler(handler)
loglevel = set_loglevel(cfg['loglevel'])
if loglevel:
log.setLevel(loglevel)
else:
log.disabled = True
log.info("%s started" % sys.argv[0])
if os.path.isfile(cfg['cfgfile']):
log.info("using configuration from '%s'" % cfg['cfgfile'])
main(options)
log.info("%s finished" % sys.argv[0])

52
ffmap-d3.jq
View File

@ -1,52 +0,0 @@
{
"meta": {
"timestamp": $nodes.timestamp
},
"nodes": (
$graph.batadv.nodes
| map(
if has("node_id") and .node_id
then (
$nodes.nodes[.node_id] as $node
| {
"id": .id,
"uptime": $node.statistics.uptime,
"flags": ($node.flags + {"client": false}),
"name": $node.nodeinfo.hostname,
"clientcount": (if $node.statistics.clients >= 0 then $node.statistics.clients else 0 end),
"hardware": $node.nodeinfo.hardware.model,
"firmware": $node.nodeinfo.software.firmware.release,
"geo": (if $node.nodeinfo.location then [$node.nodeinfo.location.latitude, $node.nodeinfo.location.longitude] else null end),
#"lastseen": $node.lastseen,
"network": $node.nodeinfo.network
}
)
else
{
"flags": {},
"id": .id,
"geo": null,
"clientcount": 0
}
end
)
),
"links": (
$graph.batadv.links
| map(
$graph.batadv.nodes[.source].node_id as $source_id
| $graph.batadv.nodes[.target].node_id as $target_id
| select(
$source_id and $target_id and
($nodes.nodes | (has($source_id) and has($target_id)))
)
| {
"target": .target,
"source": .source,
"quality": "\(.tq), \(.tq)",
"id": ($source_id + "-" + $target_id),
"type": (if .vpn then "vpn" else null end)
}
)
)
}

185
ffmap-modify.py
View File

@ -1,185 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Knotendaten manuell ändern
Das ist z.B. für ausgeschaltete Knoten interessant, die nur
temporär nicht zur Verfügung stehen. Die können ausgeblendet
werden.
Das ist besser als löschen, weil so die Statistik nicht
verschwindet
Änderungsprotokoll
==================
Version Datum Änderung(en) von
-------- ----------- ------------------------------------------------------ ----
1.0 2017-08-03 Programm in das ffmap-backend Projekt integriert tho
"""
import argparse
import configparser
import json
import os
import sys
import glob
# Einstellungen werden in folgender Reihenfolge verarbeitet
# später gesetzte Werte überschreiben frühere
# 1. im Programm hart codiert
# 2. aus der zentralen Konfigurationsdatei gelesen
# 3. als Kommandozeilenoptionen angegeben
cfg = {
'cfgfile': '/etc/ffmap/ffmap.cfg',
'logfile': '/var/log/ffmap.log',
'loglevel': 2,
'dest_dir': '/var/lib/ffmap/mapdata',
'nodedb': '/var/lib/ffmap/nodedb',
'imgpath': '/var/www/meshviewer/stats/img'
}
roles_defined = ('node', 'temp', 'mobile', 'offloader', 'service', 'test', 'gate', 'plan', 'hidden')
def main(cfg):
# Pfade zu den beteiligten Dateien
nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
# 1. Knotendaten (NodeDB)
# 1.1 Daten laden
try:
with open(nodes_fn, 'r') as nodedb_handle:
nodedb = json.load(nodedb_handle)
except IOError:
print("Error reading nodedb file %s" % nodes_fn)
nodedb = {'nodes': dict()}
# 1.2 Knoten bearbeiten
changed = False
for n in cfg['nodeid']:
if n in nodedb['nodes']:
print("Modify %s in nodedb" % n)
if 'role' in cfg and cfg['role'] in roles_defined:
try:
oldrole = nodedb['nodes'][n]['nodeinfo']['system']['role']
except KeyError:
oldrole = '<unset>'
print(" - change role from '%s' to '%s'" % (oldrole, cfg['role']))
nodedb['nodes'][n]['nodeinfo']['system']['role'] = cfg['role']
changed = True
if 'location' in cfg:
print(" - remove location")
# del nodedb['nodes'][n]['nodeinfo']['location']
changed = True
else:
print("Node %s not found in nodedb" % n)
# 1.3 Geänderte Daten zurückschreiben
if changed:
try:
with open(nodes_fn, 'w') as nodedb_handle:
json.dump(nodedb, nodedb_handle)
except IOError:
print("Error writing nodedb file %s" % nodes_fn)
# 2. Knotenliste (NodeList)
try:
with open(nodelist_fn, 'r') as nodelist_handle:
nodelist = json.load(nodelist_handle)
except IOError:
print("Error reading nodelist file %s" % nodelist_fn)
nodelist = {'nodelist': dict()}
# 2.1 Knoten bearbeiten
changed = False
ixlist = []
for nodeid in cfg['nodeid']:
found = False
for ix, node in enumerate(nodelist['nodes']):
if node['id'] == nodeid:
found = True
break
if found:
print("Modify %s in nodelist" % nodeid)
if 'role' in cfg and cfg['role'] in roles_defined:
try:
oldrole = nodelist['nodes'][ix]['role']
except KeyError:
oldrole = '<unset>'
print(" - change role from '%s' to '%s'" % (oldrole, cfg['role']))
nodelist['nodes'][ix]['role'] = cfg['role']
if 'location' in cfg:
print(" - remove location")
try:
#del nodelist['nodes'][ix]['position']
pass
except KeyError:
pass
changed = True
else:
print ("Node %s not found in nodelist" % nodeid)
# 2.3 Geänderte Daten zurückschreiben
if changed:
try:
with open(nodelist_fn, 'w') as nodelist_handle:
json.dump(nodelist, nodelist_handle)
except IOError:
print("Error writing nodelist file %s" % nodelist_fn)
if __name__ == "__main__":
# Optionen von der Kommandozeile lesen
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', action='store',
help='Configuration file')
parser.add_argument('-d', '--dest-dir', action='store',
help='Directory with JSON data files',
required=False)
parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
nargs='+', required=True,
help='Node id to modify')
parser.add_argument('-l', '--location', action='store_true',
help='Clear location information (hides node)',
required=False)
parser.add_argument('-r', '--role', action='store',
help='Set new role',
required=False)
# TODO
# Optionen was genau gemacht werden soll
# -p Position entfernen, Knoten wird nicht mehr angezeigt
# -r <rolle> Rolle einstellen
options = vars(parser.parse_args())
# Konfigurationsdatei einlesen
if options['config']:
cfg['cfgfile'] = options['config']
config = configparser.ConfigParser(cfg)
# config.read liefert eine Liste der geparsten Dateien
# zurück. Wenn sie leer ist, war z.B. die Datei nicht
# vorhanden
if config.read(cfg['cfgfile']):
if 'global' in config:
cfg['logfile'] = config['global']['logfile']
cfg['loglevel'] = config['global']['loglevel']
cfg['dest_dir'] = config['global']['dest_dir']
else:
print('Config file %s not parsed' % cfg['cfgfile'])
# Optionen von der Kommandozeile haben höchste Priorität
cfg['nodeid'] = options['nodeid']
if options['dest_dir']:
cfg['dest_dir'] = options['dest_dir']
if options['location']:
cfg['location'] = True
if options['role']:
cfg['role'] = options['role']
# Alles initialisiert, auf geht's
main(cfg)

225
ffmap-remove.py
View File

@ -1,225 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Lösche einen Knoten manuell aus dem Backend:
- JSON
- NodeDB
- NodeList
- Graph
- RRD-Dateien
- Bilder vom Webserver
Änderungsprotokoll
==================
Version Datum Änderung(en) von
-------- ----------- ------------------------------------------------------ ----
1.0 2017-01-06 Programm in das ffmap-backend Projekt integriert tho
"""
import argparse
import configparser
import json
import os
import sys
import glob
# Einstellungen werden in folgender Reihenfolge verarbeitet
# später gesetzte Werte überschreiben frühere
# 1. im Programm hart codiert
# 2. aus der zentralen Konfigurationsdatei gelesen
# 3. als Kommandozeilenoptionen angegeben
cfg = {
'cfgfile': '/etc/ffmap/ffmap.cfg',
'logfile': '/var/log/ffmap.log',
'loglevel': 2,
'dest_dir': '/var/lib/ffmap/mapdata',
'nodedb': '/var/lib/ffmap/nodedb',
'imgpath': '/var/www/meshviewer/stats/img'
}
def main(cfg):
# Pfade zu den beteiligten Dateien
nodes_fn = os.path.join(cfg['dest_dir'], 'nodes.json')
graph_fn = os.path.join(cfg['dest_dir'], 'graph.json')
nodelist_fn = os.path.join(cfg['dest_dir'], 'nodelist.json')
# 1. Knotendaten (NodeDB) bereinigen
# 1.1 Daten laden
try:
with open(nodes_fn, 'r') as nodedb_handle:
nodedb = json.load(nodedb_handle)
except IOError:
print("Error reading nodedb file %s" % nodes_fn)
nodedb = {'nodes': dict()}
# 1.2 Knoten entfernen
changed = False
for n in cfg['nodeid']:
if n in nodedb['nodes']:
print("Remove %s from nodedb" % n)
del nodedb['nodes'][n]
changed = True
else:
print("Node %s not found in nodedb" % n)
# 1.3 Geänderte Daten zurückschreiben
if changed:
try:
with open(nodes_fn, 'w') as nodedb_handle:
json.dump(nodedb, nodedb_handle)
except IOError:
print("Error writing nodedb file %s" % nodes_fn)
# 2. Knotenliste (NodeList) bereinigen
try:
with open(nodelist_fn, 'r') as nodelist_handle:
nodelist = json.load(nodelist_handle)
except IOError:
print("Error reading nodelist file %s" % nodelist_fn)
nodelist = {'nodelist': dict()}
# 2.1 Knoten entfernen
changed = False
ixlist = []
for nodeid in cfg['nodeid']:
found = False
for ix, node in enumerate(nodelist['nodes']):
if node['id'] == nodeid:
found = True
break
if found:
print("Remove %s from nodelist" % nodeid)
del nodelist['nodes'][ix]
changed = True
else:
print ("Node %s not found in nodelist" % nodeid)
# 2.3 Geänderte Daten zurückschreiben
if changed:
try:
with open(nodelist_fn, 'w') as nodelist_handle:
json.dump(nodelist, nodelist_handle)
except IOError:
print("Error writing nodelist file %s" % nodelist_fn)
# 3. Graph (NodeGraph) bereinigen
# 3.1 Graph laden
try:
with open(graph_fn, 'r') as graph_handle:
graph = json.load(graph_handle)
except IOError:
print("Error reading graph file %s" % graph_fn)
graph = {'graph': dict()}
# 3.2 Finde Knoten und Links
# Nodes und Links gehören zusammen
changed = False
for nodeid in cfg['nodeid']:
found = False
for ixn, node in enumerate(graph["batadv"]["nodes"]):
# Es kann nodes ohne "node_id" geben
try:
if node["node_id"] == nodeid:
found = True
break
except KeyError:
pass
if found:
print("Found %s in graph nodes at index %d" % (nodeid, ixn))
del graph["batadv"]["nodes"][ixn]
# Suche Link source oder target dem gefundenen Index entsprechen
ixlist = []
for ixg, link in enumerate(graph["batadv"]["links"]):
if link["source"] == ixn:
print("Found source link at index %d" % ixg)
print(" -> %s" % graph["batadv"]["nodes"][link["target"]])
ixlist.append(ixg)
if link["target"] == ixn:
print("Found target link at index %d" % ixg)
print(" -> %s" % graph["batadv"]["nodes"][link["source"]])
ixlist.append(ixg)
for ix in ixlist:
del graph["batadv"]["nodes"][ix]
changed = True
else:
print("Node %s not found in graph nodes" % nodeid)
# 3.3 Zurückschreiben der geänderten Daten
if changed:
try:
with open(graph_fn, 'w') as graph_handle:
json.dump(graph, graph_handle)
except IOError:
print("Error writing graph file %s" % graph_fn)
# 4. Entferne RRD-Dateien
for nodeid in cfg['nodeid']:
rrdfile = os.path.join(cfg['nodedb'], nodeid+'.rrd')
if os.path.isfile(rrdfile):
print("Removing RRD database file %s" % os.path.basename(rrdfile))
else:
print("RRD database file %s not found" % os.path.basename(rrdfile))
try:
os.remove(rrdfile)
except OSError:
pass
# 5. Entferne Bilder vom Webserver
count_deleted = 0
for nodeid in cfg['nodeid']:
for imagefile in glob.glob(os.path.join(cfg['imgpath'], nodeid+'_*.png')):
print("Removing stats image %s" % os.path.basename(imagefile))
try:
os.remove(imagefile)
count_deleted += 1
except OSError:
pass
if count_deleted == 0:
print("No stats images found in %s" % cfg['imgpath'])
if __name__ == "__main__":
# Optionen von der Kommandozeile lesen
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', action='store',
help='Configuration file')
parser.add_argument('-d', '--dest-dir', action='store',
help='Directory with JSON data files',
required=False)
parser.add_argument('-i', '--nodeid', metavar='ID', action='store',
nargs='+', required=True,
help='Node id to remove')
parser.add_argument('-n', '--nodedb', metavar='RRD_DIR', action='store',
help='Directory for node RRD data files')
options = vars(parser.parse_args())
# Konfigurationsdatei einlesen
if options['config']:
cfg['cfgfile'] = options['config']
config = configparser.ConfigParser(cfg)
# config.read liefert eine Liste der geparsten Dateien
# zurück. Wenn sie leer ist, war z.B. die Datei nicht
# vorhanden
if config.read(cfg['cfgfile']):
if 'global' in config:
cfg['logfile'] = config['global']['logfile']
cfg['loglevel'] = config['global']['loglevel']
cfg['dest_dir'] = config['global']['dest_dir']
if 'rrd' in config:
cfg['nodedb'] = config['rrd']['nodedb']
else:
print('Config file %s not parsed' % cfg['cfgfile'])
# Optionen von der Kommandozeile haben höchste Priorität
cfg['nodeid'] = options['nodeid']
if options['dest_dir']:
cfg['dest_dir'] = options['dest_dir']
if options['nodedb']:
cfg['nodedb'] = options['nodedb']
# Alles initialisiert, auf geht's
main(cfg)

74
lib/GateRRD.py
View File

@ -1,74 +0,0 @@
"""
RRD for gateways
"""
import os
import subprocess
from lib.RRD import DS, RRA, RRD
class GateRRD(RRD):
ds_list = [
DS('upstate', 'GAUGE', 120, 0, 1),
DS('clients', 'GAUGE', 120, 0, float('NaN')),
DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
DS('leases', 'GAUGE', 120, 0, float('NaN')),
]
rra_list = [
RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
RRA('AVERAGE', 0.5, 5, 1440), # 5 days of 5 minute samples
RRA('AVERAGE', 0.5, 15, 672), # 7 days of 15 minute samples
RRA('AVERAGE', 0.5, 60, 720), # 30 days of 1 hour samples
RRA('AVERAGE', 0.5, 720, 730), # 1 year of 12 hour samples
]
def __init__(self, filename, node=None):
"""
Create a new RRD for a given node.
If the RRD isn't supposed to be updated, the node can be omitted.
"""
self.node = node
super().__init__(filename)
self.ensure_sanity(self.ds_list, self.rra_list, step=60)
@property
def imagename(self):
return "{basename}.png".format(
basename=os.path.basename(self.filename).rsplit('.', 2)[0])
# TODO: fix this, python does not support function overloading
def update(self):
values = {
'upstate': int(self.node['flags']['online']),
'clients': float(self.node['statistics']['clients']),
}
if 'loadavg' in self.node['statistics']:
values['loadavg'] = float(self.node['statistics'].get('loadavg', 0))
# Gateways can send the peer count. We use the clients field to store data
if 'peers' in self.node['statistics']:
values['clients'] = self.node['statistics']['peers']
if 'leases' in self.node['statistics']:
values['leases'] = self.node['statistics']['leases']
super().update(values)
def graph(self, directory, timeframe):
"""
Create a graph in the given directory. The file will be named
basename.png if the RRD file is named basename.rrd
"""
args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
'-s', '-' + timeframe,
'-w', '800',
'-h', '400',
'-l', '0',
'-y', '1:1',
'DEF:clients=' + self.filename + ':clients:AVERAGE',
'VDEF:maxc=clients,MAXIMUM',
'CDEF:c=0,clients,ADDNAN',
'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
'AREA:c#0F0:up\\l',
'AREA:d#F00:down\\l',
'LINE1:c#00F:clients connected\\l']
subprocess.check_output(args)

40
lib/GlobalRRD.py
View File

@ -1,40 +0,0 @@
import os
import subprocess
from lib.RRD import DS, RRA, RRD
class GlobalRRD(RRD):
ds_list = [
# Number of nodes available
DS('nodes', 'GAUGE', 120, 0, float('NaN')),
# Number of client available
DS('clients', 'GAUGE', 120, 0, float('NaN')),
]
rra_list = [
# 2 hours of 1 minute samples
RRA('AVERAGE', 0.5, 1, 120),
# 31 days of 1 hour samples
RRA('AVERAGE', 0.5, 60, 744),
# ~5 years of 1 day samples
RRA('AVERAGE', 0.5, 1440, 1780),
]
def __init__(self, directory):
super().__init__(os.path.join(directory, "nodes.rrd"))
self.ensure_sanity(self.ds_list, self.rra_list, step=60)
# TODO: fix this, python does not support function overloading
def update(self, node_count, client_count):
super().update({'nodes': node_count, 'clients': client_count})
def graph(self, filename, timeframe):
args = ["rrdtool", 'graph', filename,
'-s', '-' + timeframe,
'-w', '800',
'-h' '400',
'DEF:nodes=' + self.filename + ':nodes:AVERAGE',
'LINE1:nodes#F00:nodes\\l',
'DEF:clients=' + self.filename + ':clients:AVERAGE',
'LINE2:clients#00F:clients']
subprocess.check_output(args)

67
lib/NodeRRD.py
View File

@ -1,67 +0,0 @@
"""
RRD for nodes
"""
import os
import subprocess
from lib.RRD import DS, RRA, RRD
class NodeRRD(RRD):
ds_list = [
DS('upstate', 'GAUGE', 120, 0, 1),
DS('clients', 'GAUGE', 120, 0, float('NaN')),
DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
]
rra_list = [
RRA('AVERAGE', 0.5, 1, 120), # 2 hours of 1 minute samples
RRA('AVERAGE', 0.5, 5, 1440), # 5 days of 5 minute samples
RRA('AVERAGE', 0.5, 60, 720), # 30 days of 1 hour samples
RRA('AVERAGE', 0.5, 720, 730), # 1 year of 12 hour samples
]
def __init__(self, filename, node=None):
"""
Create a new RRD for a given node.
If the RRD isn't supposed to be updated, the node can be omitted.
"""
self.node = node
super().__init__(filename)
self.ensure_sanity(self.ds_list, self.rra_list, step=60)
@property
def imagename(self):
return "{basename}.png".format(
basename=os.path.basename(self.filename).rsplit('.', 2)[0])
# TODO: fix this, python does not support function overloading
def update(self):
values = {
'upstate': int(self.node['flags']['online']),
'clients': self.node['statistics']['clients']
}
if 'loadavg' in self.node['statistics']:
values['loadavg'] = float(self.node['statistics']['loadavg'])
super().update(values)
def graph(self, directory, timeframe):
"""
Create a graph in the given directory. The file will be named
basename.png if the RRD file is named basename.rrd
"""
args = ['rrdtool', 'graph', os.path.join(directory, self.imagename),
'-s', '-' + timeframe,
'-w', '800',
'-h', '400',
'-l', '0',
'-y', '1:1',
'DEF:clients=' + self.filename + ':clients:AVERAGE',
'VDEF:maxc=clients,MAXIMUM',
'CDEF:c=0,clients,ADDNAN',
'CDEF:d=clients,UN,maxc,UN,1,maxc,IF,*',
'AREA:c#0F0:up\\l',
'AREA:d#F00:down\\l',
'LINE1:c#00F:clients connected\\l']
subprocess.check_output(args)

346
lib/RRD.py
View File

@ -1,346 +0,0 @@
import subprocess
import re
import os
from operator import xor, eq
from functools import reduce
from itertools import starmap
import math
class RRDIncompatibleException(Exception):
"""
Is raised when an RRD doesn't have the desired definition and cannot be
upgraded to it.
"""
pass
class RRDOutdatedException(Exception):
"""
Is raised when an RRD doesn't have the desired definition, but can be
upgraded to it.
"""
pass
if not hasattr(__builtins__, "FileNotFoundError"):
class FileNotFoundError(Exception):
pass
class RRD(object):
"""
An RRD is a Round Robin Database, a database which forgets old data and
aggregates multiple records into new ones.
It contains multiple Data Sources (DS) which can be thought of as columns,
and Round Robin Archives (RRA) which can be thought of as tables with the
DS as columns and time-dependant rows.
"""
# rra[2].cdp_prep[0].value = 1,8583033333e+03
_info_regex = re.compile("""
(?P<section>[a-z_]+)
\[ (?P<key>[a-zA-Z0-9_]+) \]
\.
|
(?P<name>[a-z_]+)
\s*=\s*
"? (?P<value>.*?) "?
$""", re.X)
_cached_info = None
def _exec_rrdtool(self, cmd, *args, **kwargs):
pargs = ["rrdtool", cmd, self.filename]
for k, v in kwargs.items():
pargs.extend(["--" + k, str(v)])
pargs.extend(args)
subprocess.check_output(pargs)
def __init__(self, filename):
self.filename = filename
def ensure_sanity(self, ds_list, rra_list, **kwargs):
"""
Create or upgrade the RRD file if necessary to contain all DS in
ds_list. If it needs to be created, the RRAs in rra_list and any kwargs
will be used for creation. Note that RRAs and options of an existing
database are NOT modified!
"""
try:
self.check_sanity(ds_list)
except FileNotFoundError:
self.create(ds_list, rra_list, **kwargs)
except RRDOutdatedException:
self.upgrade(ds_list)
def check_sanity(self, ds_list=()):
"""
Check if the RRD file exists and contains (at least) the DS listed in
ds_list.
"""
if not os.path.exists(self.filename):
raise FileNotFoundError(self.filename)
info = self.info()
if set(ds_list) - set(info['ds'].values()) != set():
for ds in ds_list:
if ds.name in info['ds'] and\
ds.type != info['ds'][ds.name].type:
raise RRDIncompatibleException(
"{} is {} but should be {}".format(
ds.name, ds.type, info['ds'][ds.name].type))
else:
raise RRDOutdatedException()
def upgrade(self, dss):
"""
Upgrade the DS definitions (!) of this RRD.
(To update its values, use update())
The list dss contains DSS objects to be updated or added. The
parameters of a DS can be changed, but not its type. New DS are always
added at the end in the order of their appearance in the list.
This is done internally via an rrdtool dump -> rrdtool restore and
modifying the dump on the fly.
"""
info = self.info()
new_ds = list(info['ds'].values())
new_ds.sort(key=lambda ds: ds.index)
for ds in dss:
if ds.name in info['ds']:
old_ds = info['ds'][ds.name]
if info['ds'][ds.name].type != ds.type:
raise RuntimeError(
"Cannot convert existing DS '{}'"
"from type '{}' to '{}'".format(
ds.name, old_ds.type, ds.type))
ds.index = old_ds.index
new_ds[ds.index] = ds
else:
ds.index = len(new_ds)
new_ds.append(ds)
added_ds_num = len(new_ds) - len(info['ds'])
dump = subprocess.Popen(
["rrdtool", "dump", self.filename],
stdout=subprocess.PIPE)
restore = subprocess.Popen(
["rrdtool", "restore", "-", self.filename + ".new"],
stdin=subprocess.PIPE)
echo = True
ds_definitions = True
for line in dump.stdout:
if ds_definitions and b'<ds>' in line:
echo = False
if b'<!-- Round Robin Archives -->' in line:
ds_definitions = False
for ds in new_ds:
restore.stdin.write(bytes("""
<ds>
<name> %s </name>
<type> %s </type>
<minimal_heartbeat>%i</minimal_heartbeat>
<min>%s</min>
<max>%s</max>
<!-- PDP Status -->
<last_ds>%s</last_ds>
<value>%s</value>
<unknown_sec> %i </unknown_sec>
</ds>
""" % (ds.name,
ds.type,
ds.args[0],
ds.args[1],
ds.args[2],
ds.last_ds,
ds.value,
ds.unknown_sec), "utf-8"))
if b'</cdp_prep>' in line:
restore.stdin.write(added_ds_num * b"""
<ds>
<primary_value> NaN </primary_value>
<secondary_value> NaN </secondary_value>
<value> NaN </value>
<unknown_datapoints> 0 </unknown_datapoints>
</ds>
""")
# echoing of input line
if echo:
restore.stdin.write(
line.replace(
b'</row>',
(added_ds_num * b'<v>NaN</v>') + b'</row>'
)
)
if ds_definitions and b'</ds>' in line:
echo = True
dump.stdout.close()
restore.stdin.close()
dump.wait()
restore.wait()
os.rename(self.filename + ".new", self.filename)
self._cached_info = None
def create(self, ds_list, rra_list, **kwargs):
"""
Create a new RRD file with the specified list of RRAs and DSs.
Any kwargs are passed as --key=value to rrdtool create.
"""
self._exec_rrdtool(
"create",
*map(str, rra_list + ds_list),
**kwargs
)
self._cached_info = None
def update(self, V):
"""
Update the RRD with new values V.
V can be either list or dict:
* If it is a dict, its keys must be DS names in the RRD and it is
ensured that the correct DS are updated with the correct values, by
passing a "template" to rrdtool update (see man rrdupdate).
* If it is a list, no template is generated and the order of the
values in V must be the same as that of the DS in the RRD.
"""
try:
args = ['N:' + ':'.join(map(str, V.values()))]
kwargs = {'template': ':'.join(V.keys())}
except AttributeError:
args = ['N:' + ':'.join(map(str, V))]
kwargs = {}
self._exec_rrdtool("update", *args, **kwargs)
self._cached_info = None
def info(self):
"""
Return a dictionary with information about the RRD.
See `man rrdinfo` for more details.
"""
if self._cached_info:
return self._cached_info
env = os.environ.copy()
env["LC_ALL"] = "C"
proc = subprocess.Popen(
["rrdtool", "info", self.filename],
stdout=subprocess.PIPE,
env=env
)
out, err = proc.communicate()
out = out.decode()
info = {}
for line in out.splitlines():
base = info
for match in self._info_regex.finditer(line):
section, key, name, value = match.group(
"section", "key", "name", "value")
if section and key:
try:
key = int(key)
except ValueError:
pass
if section not in base:
base[section] = {}
if key not in base[section]:
base[section][key] = {}
base = base[section][key]
if name and value:
try:
base[name] = int(value)
except ValueError:
try:
base[name] = float(value)
except:
base[name] = value
dss = {}
for name, ds in info['ds'].items():
ds_obj = DS(name, ds['type'], ds['minimal_heartbeat'],
ds['min'], ds['max'])
ds_obj.index = ds['index']
ds_obj.last_ds = ds['last_ds']
ds_obj.value = ds['value']
ds_obj.unknown_sec = ds['unknown_sec']
dss[name] = ds_obj
info['ds'] = dss
rras = []
for rra in info['rra'].values():
rras.append(RRA(rra['cf'], rra['xff'],
rra['pdp_per_row'], rra['rows']))
info['rra'] = rras
self._cached_info = info
return info
class DS(object):
"""
DS stands for Data Source and represents one line of data points in a Round
Robin Database (RRD).
"""
name = None
type = None
args = []
index = -1
last_ds = 'U'
value = 0
unknown_sec = 0
def __init__(self, name, dst, *args):
self.name = name
self.type = dst
self.args = args
def __str__(self):
return "DS:%s:%s:%s" % (
self.name,
self.type,
":".join(map(str, self._nan_to_u_args()))
)
def __repr__(self):
return "%s(%r, %r, %s)" % (
self.__class__.__name__,
self.name,
self.type,
", ".join(map(repr, self.args))
)
def __eq__(self, other):
return all(starmap(eq, zip(self.compare_keys(), other.compare_keys())))
def __hash__(self):
return reduce(xor, map(hash, self.compare_keys()))
def _nan_to_u_args(self):
return tuple(
'U' if type(arg) is float and math.isnan(arg)
else arg
for arg in self.args
)
def compare_keys(self):
return self.name, self.type, self._nan_to_u_args()
class RRA(object):
def __init__(self, cf, *args):
self.cf = cf
self.args = args
def __str__(self):
return "RRA:%s:%s" % (self.cf, ":".join(map(str, self.args)))
def __repr__(self):
return "%s(%r, %s)" % (
self.__class__.__name__,
self.cf,
", ".join(map(repr, self.args))
)

6
lib/alfred.py
View File

@ -20,11 +20,7 @@ class Alfred(object):
if self.unix_sock:
cmd.extend(['-s', self.unix_sock])
# There should not be any warnings which would be sent by cron
# every minute. Therefore suppress error output of called program
FNULL = open(os.devnull, 'w')
output = subprocess.check_output(cmd, stderr=FNULL)
FNULL.close()
output = subprocess.check_output(cmd)
return json.loads(output.decode("utf-8")).values()
def nodeinfo(self):

11
lib/batman.py
View File

@ -3,6 +3,7 @@ import json
import os
import re
class Batman(object):
"""
Bindings for B.A.T.M.A.N. Advanced
@ -21,7 +22,7 @@ class Batman(object):
self.environ = env
# compile regular expressions only once on startup
self.mac_addr_pattern = re.compile(r'(([a-f0-9]{2}:){5}[a-f0-9]{2})')
self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})')
def vis_data(self):
return self.vis_data_batadv_vis()
@ -51,10 +52,10 @@ class Batman(object):
def gateway_list(self):
"""
Parse "batctl meshif <mesh_interface> gwl -n"
Parse "batctl -m <mesh_interface> gwl -n"
into an array of dictionaries.
"""
cmd = ['batctl', 'meshif', self.mesh_interface, 'gwl', '-n']
cmd = ['batctl', '-m', self.mesh_interface, 'gwl', '-n']
if os.geteuid() > 0:
cmd.insert(0, 'sudo')
output = subprocess.check_output(cmd, env=self.environ)
@ -80,10 +81,10 @@ class Batman(object):
def gateway_mode(self):
"""
Parse "batctl meshif <mesh_interface> gw"
Parse "batctl -m <mesh_interface> gw"
return: tuple mode, bandwidth, if mode != server then bandwidth is None
"""
cmd = ['batctl', 'meshif', self.mesh_interface, 'gw']
cmd = ['batctl', '-m', self.mesh_interface, 'gw']
if os.geteuid() > 0:
cmd.insert(0, 'sudo')
output = subprocess.check_output(cmd, env=self.environ)

8
lib/nodelist.py
View File

@ -13,20 +13,12 @@ def export_nodelist(now, nodedb):
node_out["status"] = dict()
node_out["status"]["online"] = node["flags"]["online"]
if "firstseen" in node:
node_out["status"]["firstcontact"] = node["firstseen"]
if "lastseen" in node:
node_out["status"]["lastcontact"] = node["lastseen"]
if "clients" in node["statistics"]:
node_out["status"]["clients"] = node["statistics"]["clients"]
if "role" in node["nodeinfo"]["system"]:
node_out["role"] = node["nodeinfo"]["system"]["role"]
else:
node_out["role"] = "node"
nodelist.append(node_out)
return {"version": "1.0.1", "nodes": nodelist, "updated_at": now.isoformat()}

35
lib/nodes.py
View File

@ -11,15 +11,6 @@ def build_mac_table(nodes):
macs[mac] = node_id
except KeyError:
pass
try:
for upper_if in node['nodeinfo']['network']['mesh'].values():
for lower_if in upper_if['interfaces'].values():
for mac in lower_if:
macs[mac] = node_id
except KeyError:
pass
return macs
@ -68,7 +59,7 @@ def import_statistics(nodes, stats):
node['statistics'][target] = f(reduce(dict.__getitem__,
source,
statistics))
except (KeyError, TypeError, ZeroDivisionError):
except (KeyError, TypeError):
pass
macs = build_mac_table(nodes)
@ -82,7 +73,6 @@ def import_statistics(nodes, stats):
add(node, stats, 'memory_usage', ['memory'],
lambda d: 1 - d['free'] / d['total'])
add(node, stats, 'rootfs_usage', ['rootfs_usage'])
add(node, stats, 'traffic', ['traffic'])
def import_mesh_ifs_vis_data(nodes, vis_data):
@ -107,29 +97,12 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
for v in mesh_nodes:
node = v[0]
ifs = set()
try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh_interfaces']))
mesh_ifs = set(node['nodeinfo']['network']['mesh_interfaces'])
except KeyError:
pass
mesh_ifs = set()
try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']))
except KeyError:
pass
try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']))
except KeyError:
pass
try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
except KeyError:
pass
node['nodeinfo']['network']['mesh_interfaces'] = list(ifs | v[1])
node['nodeinfo']['network']['mesh_interfaces'] = list(mesh_ifs | v[1])
def import_vis_clientcount(nodes, vis_data):

61
lib/rrddb.py
View File

@ -1,61 +0,0 @@
#!/usr/bin/env python3
import time
import os
from lib.GlobalRRD import GlobalRRD
from lib.NodeRRD import NodeRRD
from lib.GateRRD import GateRRD
class RRD(object):
def __init__(self,
database_directory,
image_path,
display_time_global="7d",
display_time_node="1d"):
self.dbPath = database_directory
self.globalDb = GlobalRRD(self.dbPath)
self.imagePath = image_path
self.displayTimeGlobal = display_time_global
self.displayTimeNode = display_time_node
self.currentTimeInt = (int(time.time()) / 60) * 60
self.currentTime = str(self.currentTimeInt)
def update_database(self, nodes):
online_nodes = dict(filter(
lambda d: d[1]['flags']['online'], nodes.items()))
client_count = sum(map(
lambda d: d['statistics']['clients'], online_nodes.values()))
# Refresh global database
self.globalDb.update(len(online_nodes), client_count)
# Refresh databases for all single nodes
for node_id, node in online_nodes.items():
if node['flags']['gateway']:
rrd = GateRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
else:
rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
rrd.update()
def update_images(self):
# Create image path if it does not exist
try:
os.stat(self.imagePath)
except OSError:
os.mkdir(self.imagePath)
self.globalDb.graph(os.path.join(self.imagePath, "globalGraph.png"),
self.displayTimeGlobal)
nodedb_files = os.listdir(self.dbPath)
for file_name in nodedb_files:
if not os.path.isfile(os.path.join(self.dbPath, file_name)):
continue
node_name = os.path.basename(file_name).split('.')
if node_name[1] == 'rrd' and not node_name[0] == "nodes":
rrd = NodeRRD(os.path.join(self.dbPath, file_name))
rrd.graph(self.imagePath, self.displayTimeNode)

19
lib/validate.py
View File

@ -1,19 +0,0 @@
import json
def validate_nodeinfos(nodeinfos):
result = []
for nodeinfo in nodeinfos:
if validate_nodeinfo(nodeinfo):
result.append(nodeinfo)
return result
def validate_nodeinfo(nodeinfo):
if 'location' in nodeinfo:
if 'latitude' not in nodeinfo['location'] or 'longitude' not in nodeinfo['location']:
return False
return True