log_source.py
author Tero Marttila <terom@fixme.fi>
Mon, 09 Feb 2009 22:17:10 +0200
changeset 72 5ade0288f2ec
parent 65 8b50694f841e
child 73 5a7188bf2894
permissions -rw-r--r--
implement line-links as UTC timestamps
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     1
"""
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     2
    A source of IRC log files
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
     5
import datetime, calendar, itertools
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
     6
import os, errno
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     7
import pytz
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     9
class LogSource (object) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    10
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    12
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    13
    
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    14
    def get_latest (self, count) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    15
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    16
            Yield the latest events, up to `count` of them.
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    17
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    18
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    19
        abstract
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    20
    
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    21
    def get_date (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    22
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    23
            Get logs for the given date (as a datetime)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    24
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    25
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    26
        abstract
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    27
    
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    28
    def get_month_days (self, dt) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    29
        """
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    30
            Get a set of dates, telling which days in the given month (as a datetime) have logs available
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    31
        """
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    32
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    33
        abstract
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
    34
    
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    35
class LogFile (object) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    36
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    37
        A file containing LogEvents
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    38
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    39
        XXX: modify to implement LogSource?
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    40
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    41
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    42
    def __init__ (self, path, parser, start_date=None, charset='utf-8', sep='\n') :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    43
        """
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    44
            Open the file at the given path, which contains data with the given charset, as lines separated by the
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    45
            given separator. Lines are parsed using the given parser, using the given date as an initial date, see
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    46
            LogParser for more info. XXX: currently we assume start_date also for the end of the file
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    47
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    48
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    49
        # store
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    50
        self.path = path
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    51
        self.parser = parser
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    52
        self.start_date = start_date
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    53
        self.charset = charset
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    54
        self.sep = sep
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    55
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    56
        # open
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    57
        self.file = open(path, 'rb')
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    58
    
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    59
    def __iter__ (self) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    60
        """
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    61
            Yields a series of unicode lines, as read from the top of the file
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    62
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    63
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    64
        # seek to beginning
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    65
        self.file.seek(0)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    66
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    67
        # iterate over lines, decoding them as well
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents: 64
diff changeset
    68
        return (line.decode(self.charset).rstrip(self.sep) for line in self.file)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    69
    
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    70
    def read_full (self) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    71
        """
64
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
    72
            Reads all LogLines. The LogLines will have a valid offset
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    73
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    74
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    75
        # just use our __iter__
64
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
    76
        return self.parser.parse_lines(self, self.start_date, starting_offset=1)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    77
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    78
    def read_from (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    79
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    80
            Reads all LogLines from the given naive timestamp onwards
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    81
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    82
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    83
        # start reading at beginning
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    84
        events = self.read_full()
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    85
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    86
        # skip unwanted events
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    87
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    88
            if event.timestamp < dt :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    89
                continue
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    90
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    91
            else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    92
                # include this line as well
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    93
                yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    94
                break
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    95
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    96
        # yield the rest as-is
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    97
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    98
            yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    99
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   100
    def read_until (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   101
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   102
            Reads all LogLines up until the given naive timestamp
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   103
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   104
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   105
        # start reading events at the beginning
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   106
        events = self.read_full()
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   107
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   108
        # yield events until we hit the given timestamp
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   109
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   110
            if event.timestamp <= dt :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   111
                yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   112
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   113
            else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   114
                break
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   115
            
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   116
        # ignore the rest
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   117
        return
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   118
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   119
    def _read_blocks_reverse (self, blocksize=1024) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   120
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   121
            Yields blocks of file data in reverse order, starting at the end of the file
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   122
        """
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   123
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   124
        # seek to end of file
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   125
        self.file.seek(0, os.SEEK_END)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   126
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   127
        # read offset
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   128
        # XXX: hack -1 to get rid of trailing newline
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   129
        size = offset = self.file.tell() - 1
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   130
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   131
        # do not try to read past the beginning of the file
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   132
        while offset > 0:
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   133
            # calc new offset + size
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   134
            if offset > blocksize :
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   135
                # full block
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   136
                offset -= blocksize
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   137
                read_size = blocksize
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   138
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   139
            else :
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   140
                # partial block
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   141
                read_size = offset
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   142
                offset = 0
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   143
43
fc11c4e86a82 implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 41
diff changeset
   144
            # seek to offset
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   145
            self.file.seek(offset)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   146
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   147
            # read the data we want
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   148
            block = self.file.read(read_size)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   149
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   150
            # sanity check
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   151
            assert len(block) == read_size
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   152
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   153
            # yield 
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   154
            yield block
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   155
    
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   156
    def _read_lines_reverse (self) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   157
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   158
            Yields decoded lines from the end of the file, in reverse order.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   159
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   160
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   161
        # partial lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   162
        buf = ''
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   163
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   164
        # read from end of file, a block at a time
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   165
        for block in self._read_blocks_reverse() :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   166
            # add in our previous buf
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   167
            buf = block + buf
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   168
            
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   169
            # split up lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   170
            lines = buf.split(self.sep)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   171
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   172
            # keep the first one as our buffer, as it's incomplete
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   173
            buf = lines[0]
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   174
           
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   175
            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   176
            # XXX: use something like islice, this has to build a slice object
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   177
            for line in lines[:0:-1] :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   178
                yield line.decode(self.charset)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   179
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   180
    def read_latest (self, count) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   181
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   182
            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   183
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   184
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   185
        # the list of lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   186
        lines = []
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   187
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   188
        # start reading lines into lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   189
        for line in self._read_lines_reverse() :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   190
            # append
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   191
            lines.append(line)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   192
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   193
            # done?
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   194
            if len(lines) >= count :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   195
                break
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   196
        
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   197
        # decode in reverse order, using our starting date....
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   198
        # XXX: use lines[::-1] or reversed?
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   199
        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   200
        return self.parser.parse_lines(reversed(lines), self.start_date)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   201
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   202
class LogDirectory (LogSource) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   203
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   204
        A directory containing a series of timestamped LogFiles
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   205
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   206
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   207
    def __init__ (self, path, tz, parser, charset='utf-8', filename_fmt='%Y-%m-%d') :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   208
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   209
            Load the logfiles at the given path.
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   210
            
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   211
            The files contain data in the given charset, and are named according the the date in the given timezone and
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   212
            date format, and will be parsed using the given parser.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   213
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   214
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   215
        # store
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   216
        self.path = path
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   217
        self.tz = tz
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   218
        self.parser = parser
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   219
        self.charset = charset
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   220
        self.filename_fmt = filename_fmt
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   221
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   222
    def _get_logfile_datetime (self, dt) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   223
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   224
            Get the logfile corresponding to the given datetime
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   225
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   226
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   227
        # convert to target timezone
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   228
        dtz = dt.astimezone(self.tz)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   229
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   230
        # convert to date and use that
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   231
        return self._get_logfile_date(dtz.date())
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   232
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   233
    def _get_logfile_date (self, d, load=True) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   234
        """
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   235
            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   236
            presence of the logfile, do not actually open it.
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   237
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   238
            Returns None if the logfile does not exist.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   239
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   240
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   241
        # format filename
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   242
        filename = d.strftime(self.filename_fmt)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   243
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   244
        # build path
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   245
        path = os.path.join(self.path, filename)
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   246
        
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   247
        try :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   248
            if load :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   249
                # open+return the LogFile
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   250
                return LogFile(path, self.parser, d, self.charset)
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   251
            
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   252
            else :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   253
                # test
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   254
                return os.path.exists(path)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   255
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   256
        # XXX: move to LogFile
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   257
        except IOError, e :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   258
            # return None for missing files
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   259
            if e.errno == errno.ENOENT :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   260
                return None
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   261
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   262
            else :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   263
                raise
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   264
    
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   265
    def _iter_date_reverse (self, dt=None) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   266
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   267
            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   268
            given *datetime*, or the the current date, if none given
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   269
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   270
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   271
        # default to now
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   272
        if not dt :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   273
            dt = datetime.datetime.now(pytz.utc)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   274
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   275
        # convert to target timezone
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   276
        dtz = dt.astimezone(self.tz)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   277
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   278
        # our timedelta
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   279
        ONE_DAY = datetime.timedelta(1)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   280
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   281
        # iterate unto infinity
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   282
        while True :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   283
            # yield
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   284
            yield dtz.date()
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   285
            
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   286
            # one day sdrawkcab
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   287
            dtz -= ONE_DAY
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   288
    
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   289
    def _iter_logfile_reverse (self, dt=None, max_files=100) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   290
        """
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   291
            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   292
            current date, if none given.
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   293
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   294
            Reads/probes at most max_files files.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   295
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   296
        
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   297
        # start counting at zero...
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   298
        file_count = 0
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   299
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   300
        # iterate backwards over days
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   301
        for day in self._iter_date_reverse(dt) :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   302
            # stop if we've handled enough files by now
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   303
            if file_count > max_files :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   304
                break
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   305
            
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   306
            # try and open the next logfile
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   307
            logfile = None
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   308
            
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   309
            file_count += 1
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   310
            logfile = self._get_logfile_date(day)
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   311
            
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   312
            # no logfile there?
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   313
            if not logfile :
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   314
                # if we didn't find any logfiles at all, terminate rudely
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   315
                if file_count > max_files :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   316
                    raise Exception("No recent logfiles found")
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   317
                
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   318
                else :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   319
                    # skip to next day
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   320
                    continue
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   321
            
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   322
            # yield it
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   323
            yield logfile
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   324
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   325
    def get_latest (self, count) :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   326
        """
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   327
            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   328
        """
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   329
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   330
        # iterate over logfiles
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   331
        iter = self._iter_logfile_reverse()
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   332
        
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   333
        # read the events into here
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   334
        lines = []
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   335
        
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   336
        # loop until done
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   337
        while len(lines) < count :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   338
            # next logfile
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   339
            logfile = iter.next()
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   340
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   341
            # read the events
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   342
            # XXX: use a queue
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   343
            lines = list(logfile.read_latest(count)) + lines
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   344
        
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   345
        # return the events
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   346
        return lines
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   347
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   348
    def get_date (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   349
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   350
            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   351
            differs from our native datetime, this may involve lines from more than one logfile.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   352
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   353
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   354
        # begin/end of 24h period, in target timezone
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   355
        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   356
        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   357
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   358
        # as dates
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   359
        d_begin = dtz_begin.date() 
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   360
        d_end = dtz_end.date()
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   361
        
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   362
#        print
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   363
#        print "LogDirectory.get_date - %s" % dt
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   364
#        print "\t   %s %s" % (d_begin, dtz_begin)
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   365
#        print "\t-> %s %s" % (d_end, dtz_end)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   366
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   367
        # if they're the same, just pull the full log for that date
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   368
        if d_begin == d_end :
64
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   369
            # open that log
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   370
            logfile = self._get_logfile_date(d_begin)
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   371
            
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   372
            if not logfile :
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   373
                raise Exception("No logfile for date=%r" % (dt, ))
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   374
            
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   375
            # return the full data
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   376
            return logfile.read_full()
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   377
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   378
        # otherwise, we need to pull two partial logs
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   379
        else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   380
            # open both of them
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   381
            f_begin = self._get_logfile_date(d_begin)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   382
            f_end = self._get_logfile_date(d_end)
55
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   383
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   384
            # chain together the two sources
55
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   385
            return itertools.chain(
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   386
                f_begin.read_from(dtz_begin), 
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   387
                f_end.read_until(dtz_end) if f_end else []
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   388
            )
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   389
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   390
    def get_month_days (self, month) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   391
        """
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   392
            Returns a set of dates for which logfiles are available in the given datetime's month
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   393
        """
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   394
        
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   395
        # the set of days
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   396
        days = set()
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   397
        
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   398
        # iterate over month's days using Calendar
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   399
        for date in calendar.Calendar().itermonthdates(month.year, month.month) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   400
            # convert date to target datetime
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   401
            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   402
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   403
            # date in our target timezone
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   404
            log_date = dtz.date()
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   405
            
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   406
            # test for it
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   407
            if self._get_logfile_date(log_date, load=False) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   408
                # add to set
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   409
                days.add(date)
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   410
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   411
        # return set
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   412
        return days
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   413