from pvl.verkko import db, web
from pvl.verkko.utils import parse_timedelta, IPv4Network
from pvl.html import tags as html
import re
import datetime
import socket # dns
import math
import logging; log = logging.getLogger('pvl.verkko.hosts')
# XXX: this should actually be DHCPHost
class Host (object) :
DATE_FMT = '%Y%m%d'
TIME_FMT = '%H:%M:%S'
MAC_HEX = r'([A-Za-z0-9]{2})'
MAC_SEP = r'[-:.]?'
MAC_RE = re.compile(MAC_SEP.join([MAC_HEX] * 6))
@classmethod
def query (cls, session, seen=None) :
"""
seen - select hosts seen during given timedelta period
"""
query = session.query(cls)
return query
@classmethod
def normalize_mac (cls, mac) :
match = cls.MAC_RE.search(mac)
if not match :
raise ValueError(mac)
else :
return ':'.join(hh.lower() for hh in match.groups())
def __init__ (self, ip, mac, name=None) :
self.ip = ip
self.mac = mac
self.name = name
def render_mac (self) :
if not self.mac :
return None
elif len(self.mac) > (6 * 2 + 5) :
return u'???'
else :
return unicode(self.mac)
def render_name (self) :
if self.name :
return self.name.decode('ascii', 'replace')
else :
return None
STATES = {
'DHCPACK': 'ack',
'DHCPNAK': 'nak',
'DHCPRELEASE': 'release',
'DHCPDISCOVER': 'search',
'DHCPREQUEST': 'search',
'DHCPOFFER': 'search',
}
def state_class (self) :
if self.error :
return 'dhcp-error'
elif self.state in self.STATES :
return 'dhcp-' + self.STATES[self.state]
else :
return None
def state_title (self) :
return self.error # or None
def render_state (self) :
if self.error :
return "{self.state}: {self.error}".format(self=self)
else :
return self.state
@classmethod
def format_datetime (cls, dt) :
if (datetime.datetime.now() - dt).days :
return dt.strftime(cls.DATE_FMT)
else :
return dt.strftime(cls.TIME_FMT)
def seen (self) :
return (
html.span(title=self.first_seen)(self.format_datetime(self.first_seen)),
'-',
html.span(title=self.last_seen)(self.format_datetime(self.last_seen))
)
def dns (self) :
"""
Reverse-DNS lookup.
"""
if not self.ip :
return None
sockaddrs = set(sockaddr for family, socktype, proto, canonname, sockaddr in socket.getaddrinfo(self.ip, 0, 0, 0, 0, socket.AI_NUMERICHOST))
for sockaddr in sockaddrs :
try :
host, port = socket.getnameinfo(sockaddr, socket.NI_NAMEREQD)
except socket.gaierror :
continue
return host
def __unicode__ (self) :
return u"{host.ip} ({host.mac})".format(host=self)
db.mapper(Host, db.dhcp_hosts, properties=dict(
#id = db.dhcp_hosts.c.rowid,
#state = db.dhcp_hosts.c.,
))
class BaseHandler (web.Handler) :
HOST_ATTRS = {
'id': Host.id,
'net': Host.gw,
'ip': Host.ip,
'mac': Host.mac,
'name': Host.name,
'seen': Host.last_seen,
'state': Host.state,
'count': Host.count,
}
HOST_SORT = Host.last_seen.desc()
def query (self) :
return self.db.query(Host)
def sort (self, hosts, default=HOST_SORT) :
sort = self.request.args.get('sort')
if sort :
name = sort.lstrip('+-')
else :
name = None
if name :
order_by = self.HOST_ATTRS[name]
else :
order_by = default
# prefix
if not sort :
pass
elif sort.startswith('+') :
order_by = order_by.asc()
elif sort.startswith('-') :
order_by = order_by.desc()
else :
pass
log.debug("sort: %s", order_by)
hosts = hosts.order_by(order_by)
# k
return sort, hosts
def filter_attr (self, attr, value) :
"""
Return filter expression for given attr == value
"""
if attr == 'seen' :
column = Host.last_seen
if value.isdigit() :
# specific date
date = datetime.datetime.strptime(value, Host.DATE_FMT).date()
return db.between(date.strftime(Host.DATE_FMT),
db.func.strftime(Host.DATE_FMT, Host.first_seen),
db.func.strftime(Host.DATE_FMT, Host.last_seen)
)
else :
# recent
timedelta = parse_timedelta(value)
return ((db.func.now() - Host.last_seen) < timedelta)
# XXX: for sqlite, pgsql should handle this natively?
# to seconds
#timeout = timedelta.days * (24 * 60 * 60) + timedelta.seconds
# WHERE strftime('%s', 'now') - strftime('%s', last_seen) < :timeout
#filter = (db.func.strftime('%s', 'now') - db.func.strftime('%s', Host.last_seen) < timeout)
elif attr == 'ip' :
column = Host.ip
# column is IPv4 string literal format...
if '/' in value :
return (db.func.inet(Host.ip).op('<<')(db.func.cidr(value)))
else :
return (db.func.inet(Host.ip) == db.func.inet(value))
else :
# preprocess
like = False
if value.endswith('*') :
like = value.replace('*', '%')
elif attr == 'mac' :
value = Host.normalize_mac(value)
# filter
column = self.HOST_ATTRS[attr]
if like :
return (column.like(like))
else :
return (column == value)
def filter (self, hosts) :
"""
Apply filters from request.args against given hosts.
Returns (filters, hosts).
"""
# filter?
filters = {}
for attr in self.HOST_ATTRS :
values = self.request.args.getlist(attr)
if not values :
continue
filter = db.or_(*[self.filter_attr(attr, value) for value in values])
log.debug("filter %s: %s", attr, filter)
hosts = hosts.filter(filter)
filters[attr] = values
return filters, hosts
def filters_title (self) :
"""
Return a string representing the applied filters.
"""
return ', '.join(value for values in self.filters.itervalues() for value in values)
def render_hosts (self, hosts, title=None, filters=False, page=None, hilight=None) :
COLS = (
#title sort filter class
('IP', 'ip', 'ip', 'ip' ),
('MAC', 'mac', 'mac', 'mac' ),
('Hostname', 'name', False, False ),
('Network', 'net', 'net', False ),
('Seen', 'seen', 'seen', 'seen' ),
('State', 'state', 'state', False ),
)
def url (**opts) :
args = dict()
if filters :
args.update(filters)
args.update(opts)
return self.url(**args)
def sortlink (attr) :
if not self.sorts :
sort = attr
elif self.sorts.lstrip('+-') != attr :
sort = attr
elif self.sorts.startswith('-') :
sort = "+" + attr
else :
sort = "-" + attr
return html.a(href=url(sort=sort))
def paginate (page, count=None) :
"""
Render pagination.
"""
if count is not None :
pages = int(math.ceil(count / self.PAGE)) # XXX: bad self.PAGE
else :
pages = None
if page > 0 :
yield html.a(href=url(page=0))(html("«« First"))
yield html.a(href=url(page=(page - 1)))(html("« Prev"))
yield html.span("Page {page} of {pages}".format(page=(page + 1), pages=(pages or '???')))
yield html.a(href=url(page=(page + 1)))(html("» Next"))
def render_filter (filter) :
value = filters.get(filter)
if value :
# XXX: multi-valued filters?
value = value[0]
else :
value = None
return html.input(type='text', name=filter, value=value)
def render_cell (attr, value, cssclass=True, filter=None, htmlvalue=None) :
if htmlvalue :
cell = htmlvalue
else :
cell = value
if filter :
cell = html.a(href=self.url(ListHandler, **{attr: value}))(cell)
if cssclass is True :
cssclass = attr
css = (cssclass, 'hilight' if (hilight and attr in hilight and value in hilight[attr]) else None)
css = ' '.join(cls for cls in css if cls)
return html.td(class_=css)(cell)
table = html.table(
html.caption(title) if title else None,
html.thead(
html.tr(
html.th('#'),
(
html.th(
sortlink(sort)(title) if sort else (title)
) for title, sort, filter, class_ in COLS
)
),
html.tr(class_='filter')(
html.td(
html.input(type='submit', value=u'\u00BF'),
),
(
html.td(class_=class_)(
render_filter(filter) if filter else None
) for title, sort, filter, class_ in COLS
)
) if filters is not False else None
),
html.tbody(
html.tr(class_=('alternate' if i % 2 else None), id=host.id)(
html.th(
html.a(href=self.url(ItemHandler, id=host.id))(
'#' #host.id
)
),
render_cell('ip', host.ip, filter=True),
render_cell('mac', host.mac, filter=True, htmlvalue=host.render_mac()),
render_cell('name', host.name, htmlvalue=host.render_name()),
render_cell('gw', host.gw),
render_cell('seen', host.seen()),
html.td(class_=host.state_class(), title=host.state_title())(host.state),
) for i, host in enumerate(hosts)
),
html.tfoot(
html.tr(
html.td(colspan=(1 + len(COLS)))(
paginate(page) if page is not None else (
# XXX: does separate SELECT count()
"{count} hosts".format(count=hosts.count())
)
)
)
)
)
if filters is False :
return table
else :
return html.form(method='get', action=self.url())(
html.input(type='hidden', name='sort', value=self.sorts),
table,
)
class ItemHandler (BaseHandler) :
def process (self, id) :
self.hosts = self.query()
self.host = self.hosts.get(id)
if not self.host :
raise web.NotFound("No such host: {id}".format(id=id))
self.sorts, self.hosts = self.sort(self.hosts.filter((Host.ip == self.host.ip) | (Host.mac == self.host.mac)))
def title (self) :
return u"DHCP Host: {self.host}".format(self=self)
def render_host (self, host) :
"""
Details for specific host.
"""
attrs = (
('Network', host.gw),
('IP', host.ip),
('MAC', host.mac),
('Hostname', host.name),
('DNS', host.dns()),
('First seen', host.first_seen),
('Last seen', host.last_seen),
('Last state', host.render_state()),
('Total messages', host.count),
)
return (
html.dl(
(html.dt(title), html.dd(value)) for title, value in attrs
)
)
def render (self) :
return (
html.h2('Host'),
self.render_host(self.host),
html.h2('Related'),
self.render_hosts(self.hosts, hilight=dict(ip=self.host.ip, mac=self.host.mac)),
html.a(href=self.url(ListHandler))(html('«'), 'Back'),
)
class ListHandler (BaseHandler) :
# pagination
PAGE = 10
# views
VIEWS = (
("Last hour", dict(seen='1h')),
("Last day", dict(seen='24h')),
("All", dict()),
) + tuple(
("Network " + network, dict(ip=network)) for network in (
'194.197.235.0/24',
'10.1.0.0/16',
'10.4.0.0/16',
'10.5.0.0/16',
'10.6.0.0/16',
'10.10.0.0/16',
)
) + (
("Valid", dict(state=('DHCPACK', 'DHCPRELEASE'))),
("Incomplete", dict(state=('DHCPDISCOVER', 'DHCPOFFER', 'DHCPREQUEST'))),
("Invalid", dict(state=('DHCPNAK', ))),
)
def process (self) :
hosts = self.query()
# filter
self.filters, hosts = self.filter(hosts)
# sort XXX: default per filter column?
self.sorts, hosts = self.sort(hosts)
# page?
self.page = self.request.args.get('page')
if self.page :
self.page = int(self.page)
hosts = hosts.offset(self.page * self.PAGE).limit(self.PAGE)
self.hosts = hosts
def title (self) :
if self.filters :
return "DHCP Hosts: {filters}".format(filters=self.filters_title())
else :
return "DHCP Hosts"
def render (self) :
return (
self.render_hosts(self.hosts, filters=self.filters, page=self.page),
html.a(href=self.url())(html('«'), 'Back') if self.filters else None,
)
import json
import time
def dt2ts (dt) :
return int(time.mktime(dt.timetuple()))
def ts2dt (ts) :
return datetime.datetime.fromtimestamp(ts)
class RealtimeHandler (BaseHandler) :
TITLE = "Pseudo-Realtime hosts.."
CSS = web.Handler.CSS + (
'http://code.jquery.com/ui/1.9.0/themes/base/jquery-ui.css',
)
JS = (
#"/static/jquery/jquery.js",
'http://code.jquery.com/jquery-1.8.2.js',
'http://code.jquery.com/ui/1.9.0/jquery-ui.js',
'/static/js/spin.js',
'/static/hosts.js',
)
COLUMNS = (
( 'ip', 'IP', lambda host: host.ip ),
( 'mac', 'MAC', lambda host: host.mac ),
( 'name', 'Hostname', lambda host: host.name ),
( 'gw', 'Network', lambda host: host.gw ),
( 'seen', 'Seen', Host.seen, ),
( 'state', 'State', lambda host: host.state ),
)
def process (self) :
"""
Either return JSON (if ?t=...), or fetch hosts/t for rendering.
"""
hosts = self.db.query(Host)
t = self.request.args.get('t')
# always sorted by last_seen
hosts = hosts.order_by(Host.last_seen.desc())
# filter
self.filters, hosts = self.filter(hosts)
def host_params (host) :
yield 'id', host.id
for name, title, fvalue in self.COLUMNS :
value = fvalue(host)
if name == 'seen' :
# XXX: hackfix html() rendering
value = unicode(html.div(value))
yield name, value
# special
yield 'state_class', host.state_class()
if t :
# return json
t = ts2dt(int(t))
hosts = hosts.filter(Host.last_seen > t)
hosts = list(hosts)
hosts.reverse()
if hosts :
t = hosts[-1].last_seen
hosts = [dict(host_params(host)) for host in hosts]
else :
hosts = []
data = dict(
t = dt2ts(t),
hosts = hosts,
)
return web.Response(json.dumps(data), mimetype='text/json')
else :
# render html
hosts = hosts.limit(10)
# XXX: testing
hosts = hosts.offset(1)
self.hosts = list(hosts)
if self.hosts :
self.t = self.hosts[0].last_seen
else :
self.t = datetime.datetime.now()
def title (self) :
if self.filters :
return "{title}: {filters}".format(title=self.TITLE, filters=self.filters_title())
else :
return self.TITLE
def render (self) :
"""
Render page HTML and initial <table>, along with bootstrap JS (t0, configuration).
"""
def column (name, title, fvalue, host) :
cls = name
if name == 'state' :
cls = host.state_class()
return html.td(class_=cls)(fvalue(host))
params = dict(
url = self.url(),
filters = self.filters,
t = dt2ts(self.t),
host = self.url(ItemHandler, id='0'),
columns = [name for name, title, fvalue in self.COLUMNS]
)
params = json.dumps(params)
return html.div(id='wrapper')(
html.input(type='submit', id='refresh', value="Refresh"),
html.table(id='hosts')(
html.thead(
html.tr(
html.th('#'),
(
html.th(class_=name)(title) for name, title, fvalue in self.COLUMNS
)
),
),
html.tbody(
html.tr(id=host.id)(
html.td(html.a(href=self.url(ItemHandler, id=host.id))('#')),
(
column(name, title, fvalue, host) for name, title, fvalue in self.COLUMNS
),
) for host in self.hosts
)
),
html.script(type='text/javascript')("""
$(document).ready(hosts_realtime({params}));
""".format(params=params)
)
)