sites/irclogs.qmsk.net/log_source.py
author Tero Marttila <terom@fixme.fi>
Sun, 08 Feb 2009 02:55:53 +0200
branchsites
changeset 184 a3d9aa76790d
parent 182 53e376e572ce
permissions -rw-r--r--
implement channel_view count, the query stuff, css, layout all need some cleanup :(
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     1
"""
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     2
    A source of IRC log files
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     5
import codecs
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     6
from datetime import date, datetime, timedelta
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     7
import pytz
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     9
# for SEEK_*, errno
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    10
import os, errno
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    12
class LogSource (object) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    13
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    14
        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    15
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    16
    
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    17
    def get_latest (self, count) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    18
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    19
            Yield the latest events, up to `count` of them.
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    20
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    21
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    22
        abstract
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    23
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    24
class LogFile (LogSource) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    25
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    26
        A file containing LogEvents
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    27
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    28
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    29
    def __init__ (self, path, charset='utf-8', sep='\n') :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    30
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    31
            Open the file at the given path, which contains data of the given codec, as lines separated by the given separator
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    32
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    33
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    34
        # store
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    35
        self.path = path
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    36
        self.charset = charset
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    37
        self.sep = sep
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    38
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    39
        # open
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    40
        self.file = codecs.open(path, 'r', charset)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    41
    
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    42
    def __iter__ (self) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    43
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    44
            Yields a series of lines, as read from the top of the file
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    45
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    46
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    47
        # seek to beginning
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    48
        self.file.seek(0)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    49
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    50
        # iterate over lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    51
        return iter(self.file)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    52
    
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    53
    def get_latest (self, count) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    54
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    55
            Returns up to <count> lines from the end of the file, or less, if the file doesn't contain that many lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    56
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    57
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    58
        # the list of lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    59
        lines = []
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    60
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    61
        # seek to end of file
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    62
        self.file.seek(0, os.SEEK_END)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    63
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    64
        # read offset
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    65
        # XXX; why -2 ?
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    66
        size = offset = self.file.tell() - 2
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    67
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    68
        # use this blocksize
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    69
        BLOCKSIZE = 1024
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    70
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    71
        # trailing data
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    72
        buf = ''
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    73
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    74
        # read a block at a time, backwards
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    75
        while  count > 0 and offset >= 0:
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    76
            # update offset back one block
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    77
            offset -= BLOCKSIZE
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    78
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    79
            # normalize to zero
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    80
            if offset < 0 :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    81
                offset = 0
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    82
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    83
            # seek to offset
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    84
            self.file.seek(offset)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    85
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    86
            # add the new block to our buffer
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    87
            read_buf = self.file.read(BLOCKSIZE)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    88
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    89
            # XXX: trim off extra...
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    90
            if len(read_buf) > BLOCKSIZE :
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    91
                read_buf = read_buf[:BLOCKSIZE]
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    92
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    93
            # make sure we got the right amount of data
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
    94
            assert len(read_buf) == BLOCKSIZE, "read(%d) @ %d/%d -> %d" % (BLOCKSIZE, offset, size, len(read_buf))
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    95
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    96
            # add in our previous buf
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    97
            buf = read_buf + buf
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    98
            
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    99
            # split out lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   100
            buf_lines = buf.split(self.sep)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   101
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   102
            # keep the first one as our buffer, as it's incomplete
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   103
            buf = buf_lines[0]
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   104
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   105
            # add up to count lines to our lines buffer
184
a3d9aa76790d implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 182
diff changeset
   106
            lines = buf_lines[-min(count, len(buf_lines) - 1):] + lines
182
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   107
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   108
            # update count
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   109
            count -= (len(buf_lines) - 1)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   110
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   111
        # return the line list
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   112
        return lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   113
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   114
class LogDirectory (LogSource) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   115
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   116
        A directory containing a series of timestamped LogFiles
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   117
    """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   118
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   119
    def __init__ (self, path, tz, charset='utf-8', filename_fmt='%Y-%m-%d') :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   120
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   121
            Load the logfiles at the given path.
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   122
            
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   123
            The files contain data in the given charset, and are named according the the date in the given timezone and
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   124
            date format.
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   125
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   126
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   127
        # store
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   128
        self.path = path
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   129
        self.tz = tz
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   130
        self.charset = charset
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   131
        self.filename_fmt = filename_fmt
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   132
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   133
    def _get_logfile_datetime (self, dt) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   134
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   135
            Get the logfile corresponding to the given datetime
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   136
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   137
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   138
        # convert to target timezone
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   139
        dtz = dt.astimezone(self.tz)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   140
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   141
        # convert to date and use that
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   142
        return self._get_logfile_date(dtz.date())
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   143
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   144
    def _get_logfile_date (self, d) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   145
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   146
            Get the logfile corresponding to the given naive date in our timezone
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   147
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   148
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   149
        # format filename
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   150
        filename = d.strftime(self.filename_fmt)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   151
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   152
        # build path
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   153
        path = os.path.join(self.path, filename)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   154
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   155
        # return the LogFile
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   156
        return LogFile(path, self.charset)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   157
    
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   158
    def _iter_backwards (self, dt=None) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   159
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   160
            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   161
            given *datetime*, or the the current date, if none given
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   162
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   163
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   164
        # default to now
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   165
        if not dt :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   166
            dt = datetime.now(pytz.utc)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   167
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   168
        # convert to target timezone
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   169
        dtz = dt.astimezone(self.tz)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   170
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   171
        # our timedelta
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   172
        ONE_DAY = timedelta(1)
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   173
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   174
        # iterate unto infinity
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   175
        while True :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   176
            # yield
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   177
            yield dtz.date()
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   178
            
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   179
            # one day sdrawkcab
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   180
            dtz -= ONE_DAY
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   181
    
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   182
    def get_latest (self, count) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   183
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   184
            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   185
        """
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   186
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   187
        # iterate backwards from now
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   188
        day_iter = self._iter_backwards()
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   189
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   190
        # number of files read
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   191
        files = 0
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   192
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   193
        # only read up to 100 files or so
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   194
        MAX_FILES = 100
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   195
        
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   196
        # loop until done
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   197
        while count > 0 :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   198
            logfile = None
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   199
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   200
            try :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   201
                # get next logfile
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   202
                files += 1
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   203
                
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   204
                # open
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   205
                logfile = self._get_logfile_date(day_iter.next())
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   206
            
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   207
            except IOError, e :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   208
                # skip nonexistant days if we haven't found any logs yet
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   209
                if e.errno != errno.ENOENT :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   210
                    raise
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   211
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   212
                if files > MAX_FILES :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   213
                    raise Exception("No recent logfiles found")
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   214
                
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   215
                else :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   216
                    # skip to next day
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   217
                    continue
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   218
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   219
            # yield lines
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   220
            for line in logfile.get_latest(count) :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   221
                # yield while we still need to, otherwise, stop
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   222
                if count > 0 :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   223
                    # decrement
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   224
                    count -= 1
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   225
 
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   226
                    yield line
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   227
            
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   228
                else :
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   229
                    break
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   230
53e376e572ce working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   231