log_source.py
author Tero Marttila <terom@fixme.fi>
Sun, 08 Feb 2009 04:41:00 +0200
changeset 48 7858b7b8ffe3
parent 46 185504387370
child 50 f13cf27a360b
permissions -rw-r--r--
fix bugs with file tailing
"""
    A source of IRC log files
"""

import codecs
from datetime import date, datetime, timedelta
import pytz

# for SEEK_*, errno
import os, errno

class LogSource (object) :
    """
        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events
    """
    
    def get_latest (self, count) :
        """
            Yield the latest events, up to `count` of them.
        """

        abstract

class LogFile (LogSource) :
    """
        A file containing LogEvents
    """

    def __init__ (self, path, charset='utf-8', sep='\n') :
        """
            Open the file at the given path, which contains data of the given codec, as lines separated by the given separator
        """
        
        # store
        self.path = path
        self.charset = charset
        self.sep = sep

        # open
        self.file = open(path, 'rb')
    
    def __iter__ (self) :
        """
            Yields a series of lines, as read from the top of the file
        """
        
        # seek to beginning
        self.file.seek(0)

        # iterate over lines
        return iter(self.file)
    
    def get_latest (self, count) :
        """
            Returns up to <count> lines from the end of the file, or less, if the file doesn't contain that many lines
        """

        # the list of lines
        lines = []

        # seek to end of file
        self.file.seek(0, os.SEEK_END)

        # read offset
        # XXX: hack -1 to get rid of trailing newline
        size = offset = self.file.tell() - 1

        # use this blocksize
        BLOCKSIZE = 1024

        # trailing data
        buf = ''

        # read a block at a time, backwards
        while len(lines) < count and offset > 0:
            # calc new offset + size
            if offset > BLOCKSIZE :
                # full block
                offset -= BLOCKSIZE
                read_size = BLOCKSIZE

            else :
                # partial block
                read_size = offset
                offset = 0

            # seek to offset
            self.file.seek(offset)

            # read the data we want
            read_buf = self.file.read(read_size)
            read_len = len(read_buf)

            # sanity check
            assert read_len == read_size

            # add in our previous buf
            buf = read_buf + buf
            
            # split out lines
            buf_lines = buf.split(self.sep)

            # keep the first one as our buffer, as it's incomplete
            buf = buf_lines[0]

            # prepend up to count lines from the end to our lines buffer
            lines = buf_lines[-min(count, len(buf_lines) - 1):] + lines
        
        # decode
        # XXX: better queue implementation, plz
        lines = [line.decode(self.charset) for line in lines]

        # return the line list
        return lines

class LogDirectory (LogSource) :
    """
        A directory containing a series of timestamped LogFiles
    """

    def __init__ (self, path, tz, charset='utf-8', filename_fmt='%Y-%m-%d') :
        """
            Load the logfiles at the given path.
            
            The files contain data in the given charset, and are named according the the date in the given timezone and
            date format.
        """

        # store
        self.path = path
        self.tz = tz
        self.charset = charset
        self.filename_fmt = filename_fmt

    def _get_logfile_datetime (self, dt) :
        """
            Get the logfile corresponding to the given datetime
        """

        # convert to target timezone
        dtz = dt.astimezone(self.tz)
        
        # convert to date and use that
        return self._get_logfile_date(dtz.date())

    def _get_logfile_date (self, d) :
        """
            Get the logfile corresponding to the given naive date in our timezone
        """

        # format filename
        filename = d.strftime(self.filename_fmt)

        # build path
        path = os.path.join(self.path, filename)

        # return the LogFile
        return LogFile(path, self.charset)
    
    def _iter_backwards (self, dt=None) :
        """
            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the
            given *datetime*, or the the current date, if none given
        """
        
        # default to now
        if not dt :
            dt = datetime.now(pytz.utc)
        
        # convert to target timezone
        dtz = dt.astimezone(self.tz)

        # our timedelta
        ONE_DAY = timedelta(1)
        
        # iterate unto infinity
        while True :
            # yield
            yield dtz.date()
            
            # one day sdrawkcab
            dtz -= ONE_DAY
    
    def get_latest (self, count) :
        """
            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed
        """
        
        # iterate backwards from now
        day_iter = self._iter_backwards()

        # number of files read
        files = 0

        # only read up to 100 files or so
        MAX_FILES = 100

        # read the lines into here
        lines = []
        
        # loop until done
        while len(lines) < count :
            logfile = None

            try :
                # get next logfile
                files += 1
                
                # open
                logfile = self._get_logfile_date(day_iter.next())
            
            except IOError, e :
                # skip nonexistant days if we haven't found any logs yet
                if e.errno != errno.ENOENT :
                    raise

                if files > MAX_FILES :
                    raise Exception("No recent logfiles found")
                
                else :
                    # skip to next day
                    continue
            
            # read the lines
            lines = logfile.get_latest(count) + lines
        
        # return the lines
        return lines