author | Tero Marttila <terom@paivola.fi> |
Sat, 21 Dec 2013 22:57:44 +0200 | |
changeset 329 | b3778c190aa5 |
parent 235 | a34e7260568b |
child 425 | 4e828d47421a |
permissions | -rw-r--r-- |
235
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
1 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
2 |
Handle host-interface mappings for stats. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
3 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
4 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
5 |
import shlex |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
6 |
import os.path |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
7 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
8 |
import logging; log = logging.getLogger('pvl.rrd.hosts') |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
9 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
10 |
def hostjoin (*hosts) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
11 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
12 |
DNS hostname join. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
13 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
14 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
15 |
return '.'.join(hosts) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
16 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
17 |
def hostreverse (host) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
18 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
19 |
Reverse hostname. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
20 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
21 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
22 |
return '.'.join(reversed(host.split('.'))) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
23 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
24 |
def load (file, domain=None) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
25 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
26 |
Parse hosts from file, yielding |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
27 |
(host, node, iface, tag) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
28 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
29 |
file - read host/ifaces from file |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
30 |
domain - append given domain to hostname |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
31 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
32 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
33 |
host = None |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
34 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
35 |
for idx, line in enumerate(file, 1) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
36 |
line = line.rstrip() |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
37 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
38 |
if not line : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
39 |
continue |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
40 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
41 |
# comment? |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
42 |
if line.startswith('#') : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
43 |
continue |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
44 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
45 |
# line |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
46 |
parts = shlex.split(line) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
47 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
48 |
if not line[0].isspace() : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
49 |
host = parts.pop(0) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
50 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
51 |
# host-spec? |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
52 |
if '=' in host : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
53 |
host, node = host.split('=') |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
54 |
else : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
55 |
node = host |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
56 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
57 |
# host has domain in collectd? |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
58 |
if domain : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
59 |
host = hostjoin(host, domain) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
60 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
61 |
if not parts : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
62 |
# keep host for following lines |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
63 |
continue |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
64 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
65 |
iface = parts.pop(0) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
66 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
67 |
# possibly multiple tags.. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
68 |
for tag in parts : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
69 |
yield host, node, iface, tag |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
70 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
71 |
def map_interfaces (options, file): |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
72 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
73 |
Read (hostname, interface}: (nodename, tag) pairs from file. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
74 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
75 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
76 |
for host, node, iface, tag in load(file) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
77 |
log.debug("%s/%s -> %s/%s", host, iface, node, tag) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
78 |
yield (host, iface), (node, tag) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
79 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
80 |
def collectd_interfaces (options, file, collectd_domain, collectd_plugin) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
81 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
82 |
Read collectd (host, type-instance, name) items, and yield (collectd-rrd, out-rrd) tuples. |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
83 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
84 |
file - read host/ports from file |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
85 |
collectd_domain - append given domain to collectd hostname |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
86 |
collectd_plugin - use given collectd plugin's type-instances |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
87 |
""" |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
88 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
89 |
log.info("scanning %s/<host>.%s/%s/%s-<port>.rrd", options.collectd_rrd, collectd_domain, collectd_plugin, options.collectd_type) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
90 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
91 |
for collectd_host, interface_host, port, tag in load(file, domain=collectd_domain) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
92 |
# flip from DNS-ordering -> path-ordering |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
93 |
if options.reverse_host : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
94 |
interface_host = hostreverse(interface_host) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
95 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
96 |
if options.collectd_instance == 'type' : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
97 |
type = options.collectd_type + '-' + port |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
98 |
else : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
99 |
type = options.collectd_type |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
100 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
101 |
if options.collectd_instance == 'plugin' : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
102 |
plugin = collectd_plugin + '-' + port |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
103 |
else : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
104 |
plugin = collectd_plugin |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
105 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
106 |
collectd_rrd = os.path.join(options.collectd_rrd, collectd_host, plugin, type) + '.rrd' |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
107 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
108 |
if not os.path.exists(collectd_rrd) : |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
109 |
log.warn("%s/%s: missing collectd rrd: %s", collectd_host, port, collectd_rrd) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
110 |
continue |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
111 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
112 |
# out |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
113 |
interface_rrd = os.path.join(interface_host, tag + '.rrd') |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
114 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
115 |
log.debug("%s: %s", interface_rrd, collectd_rrd) |
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
116 |
|
a34e7260568b
split pvl.rrd-interface into pvl.rrd.hosts
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
117 |
yield collectd_rrd, interface_rrd |