add search form to menu, remove styling from inputs/selects other than the submit button
"""
A source of IRC log files
"""
import codecs
from datetime import date, datetime, timedelta
import pytz
# for SEEK_*, errno
import os, errno
class LogSource (object) :
"""
A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events
"""
def get_latest (self, count) :
"""
Yield the latest events, up to `count` of them.
"""
abstract
class LogFile (LogSource) :
"""
A file containing LogEvents
"""
def __init__ (self, path, charset='utf-8', sep='\n') :
"""
Open the file at the given path, which contains data of the given codec, as lines separated by the given separator
"""
# store
self.path = path
self.charset = charset
self.sep = sep
# open
self.file = open(path, 'rb')
def __iter__ (self) :
"""
Yields a series of lines, as read from the top of the file
"""
# seek to beginning
self.file.seek(0)
# iterate over lines
return iter(self.file)
def get_latest (self, count) :
"""
Returns up to <count> lines from the end of the file, or less, if the file doesn't contain that many lines
"""
# the list of lines
lines = []
# seek to end of file
self.file.seek(0, os.SEEK_END)
# read offset
# XXX: hack -1 to get rid of trailing newline
size = offset = self.file.tell() - 1
# use this blocksize
BLOCKSIZE = 1024
# trailing data
buf = ''
# read a block at a time, backwards
while len(lines) < count and offset > 0:
# calc new offset + size
if offset > BLOCKSIZE :
# full block
offset -= BLOCKSIZE
read_size = BLOCKSIZE
else :
# partial block
read_size = offset
offset = 0
# seek to offset
self.file.seek(offset)
# read the data we want
read_buf = self.file.read(read_size)
read_len = len(read_buf)
# sanity check
assert read_len == read_size
# add in our previous buf
buf = read_buf + buf
# split out lines
buf_lines = buf.split(self.sep)
# keep the first one as our buffer, as it's incomplete
buf = buf_lines[0]
# prepend up to count lines from the end to our lines buffer
lines = buf_lines[-min(count, len(buf_lines) - 1):] + lines
# decode
# XXX: better queue implementation, plz
lines = [line.decode(self.charset) for line in lines]
# return the line list
return lines
class LogDirectory (LogSource) :
"""
A directory containing a series of timestamped LogFiles
"""
def __init__ (self, path, tz, charset='utf-8', filename_fmt='%Y-%m-%d') :
"""
Load the logfiles at the given path.
The files contain data in the given charset, and are named according the the date in the given timezone and
date format.
"""
# store
self.path = path
self.tz = tz
self.charset = charset
self.filename_fmt = filename_fmt
def _get_logfile_datetime (self, dt) :
"""
Get the logfile corresponding to the given datetime
"""
# convert to target timezone
dtz = dt.astimezone(self.tz)
# convert to date and use that
return self._get_logfile_date(dtz.date())
def _get_logfile_date (self, d) :
"""
Get the logfile corresponding to the given naive date in our timezone
"""
# format filename
filename = d.strftime(self.filename_fmt)
# build path
path = os.path.join(self.path, filename)
# return the LogFile
return LogFile(path, self.charset)
def _iter_backwards (self, dt=None) :
"""
Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the
given *datetime*, or the the current date, if none given
"""
# default to now
if not dt :
dt = datetime.now(pytz.utc)
# convert to target timezone
dtz = dt.astimezone(self.tz)
# our timedelta
ONE_DAY = timedelta(1)
# iterate unto infinity
while True :
# yield
yield dtz.date()
# one day sdrawkcab
dtz -= ONE_DAY
def get_latest (self, count) :
"""
Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed
"""
# iterate backwards from now
day_iter = self._iter_backwards()
# number of files read
files = 0
# only read up to 100 files or so
MAX_FILES = 100
# read the lines into here
lines = []
# loop until done
while len(lines) < count :
logfile = None
try :
# get next logfile
files += 1
# open
logfile = self._get_logfile_date(day_iter.next())
except IOError, e :
# skip nonexistant days if we haven't found any logs yet
if e.errno != errno.ENOENT :
raise
if files > MAX_FILES :
raise Exception("No recent logfiles found")
else :
# skip to next day
continue
# read the lines
lines = logfile.get_latest(count) + lines
# return the lines
return lines