log_source.py
author Tero Marttila <terom@fixme.fi>
Wed, 11 Feb 2009 03:05:11 +0200
changeset 97 6165f1ba458d
parent 94 6673de9bc911
child 102 e396613bc873
permissions -rw-r--r--
implement parser/formatter netsplits and day-change
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     1
"""
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     2
    A source of IRC log files
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
77
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
     5
import datetime, calendar, itertools, functools, math
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
     6
import os, os.path, errno
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     7
import pytz
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
     9
import config, utils
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    10
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    11
class LogSourceDecoder (object) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    12
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    13
        Handles decoding of LogSource lines
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    14
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    15
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    16
    def __init__ (self, encoding_list) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    17
        """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    18
            Will try each of the given (charset, errors) items in turn, until one succeeds
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    19
        """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    20
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    21
        self.encoding_list = encoding_list
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    22
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    23
    def decode (self, line) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    24
        """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    25
            Decode the line of str() text into an unicode object
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    26
        """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    27
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    28
        # list of errors encountered
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    29
        error_list = []
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    30
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    31
        # try each in turn
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    32
        for charset, errors in self.encoding_list :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    33
            # trap UnicodeDecodeError to try with the next one
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    34
            try :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    35
                return line.decode(charset, errors)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    36
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    37
            except UnicodeDecodeError, e :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    38
                error_list.append("%s:%s - %s" % (charset, errors, e))
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    39
                continue
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    40
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    41
        # failure
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    42
        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    43
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    44
class LogSource (object) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    45
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    46
        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    47
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    48
    
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    49
    def __init__ (self, decoder, channel=None) :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    50
        """
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    51
            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    52
            known, then it can be given as None, and set later with bind_channel.
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    53
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    54
            Uses the given LogSourceDecoder to decode the lines.
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    55
        """
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    56
        
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    57
        self.channel = channel
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    58
        self.decoder = decoder
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    59
    
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    60
    def bind_channel (self, channel) :
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    61
        """
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    62
            Set this source's channel, where None was set before
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    63
        """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
    64
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    65
        assert not self.channel
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    66
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    67
        self.channel = channel
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    68
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    69
    def get_latest (self, count) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    70
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    71
            Yield the latest events, up to `count` of them.
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    72
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    73
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    74
        abstract
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    75
    
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    76
    def get_date (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    77
        """
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    78
            Get logs for the given date (as a datetime).
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    79
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    80
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
    81
        abstract
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    82
    
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    83
    def get_date_paged (self, dt, count, page=None) :
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    84
        """
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    85
            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    86
            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    87
            then the lines for the page containing the given timestamp is returned.
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    88
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    89
            The return value is a (page, max, lines) tuple.
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    90
        """
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    91
        
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    92
        # how to act?
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    93
        if page :
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    94
            # constant skip
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    95
            skip = (page - 1) * count
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    96
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    97
        else :
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    98
            skip = None
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    99
78
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   100
            # go through the logs a page at a time
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   101
            this_page = 1
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   102
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   103
            # last line's timestamp
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   104
            last_ts = None
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   105
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   106
            # found it yet?
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   107
            found = False
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   108
77
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   109
        # count the full number of lines
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   110
        line_count = 0
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   111
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   112
        # collect lines
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   113
        lines = []
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   114
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   115
        # iterate using get_date
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   116
        for line in self.get_date(dt) :
77
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   117
            # count them
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   118
            line_count += 1
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   119
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   120
            # skip?
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   121
            if skip :
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   122
                skip -= 1
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   123
                continue
78
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   124
            
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   125
            # is this page all that we want/need?
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   126
            if page or found :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   127
                # already full?
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   128
                if len(lines) >= count :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   129
                    continue
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   130
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   131
            # specfic timestamp
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   132
            else :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   133
                # didn't find it in this page?
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   134
                if len(lines) >= count :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   135
                    # reset to next page
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   136
                    lines = []
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   137
                    this_page += 1
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   138
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   139
                # is dt between these two timestamps?
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   140
                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   141
                    # found!
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   142
                    found = True
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   143
                    page = this_page
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   144
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   145
                else :
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   146
                    # keep looking
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   147
                    last_ts = line.timestamp
85345abbd46a implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents: 77
diff changeset
   148
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   149
            # store line
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   150
            lines.append(line)
77
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   151
        
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   152
        # calculate max_pages
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   153
        max_pages = math.ceil(float(line_count) / count)
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   154
        
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   155
        # return
4287fb77e312 implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents: 76
diff changeset
   156
        return (page, max_pages, lines)
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   157
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   158
    def get_month_days (self, dt) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   159
        """
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   160
            Return a sequence of dates, telling which days in the given month (as a datetime) have logs available
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   161
        """
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   162
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   163
        abstract
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   164
    
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   165
    def get_modified (self, dt=None, after=None) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   166
        """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   167
            Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime.
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   168
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   169
            If the datetime is not given, *all* lines are returned.
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   170
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   171
            If after is given, only lines after said date will be returned, regardless of modification.
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   172
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   173
            The LogLines should be in time order.
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   174
        """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   175
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   176
        abstract
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   177
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   178
class LogFile (object) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   179
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   180
        A file containing LogEvents
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   181
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   182
        XXX: modify to implement LogSource?
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   183
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   184
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   185
    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   186
        """
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   187
            Open the file at the given path, which contains lines as separated by the given separator. Lines are
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   188
            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   189
            as the initial date for this log's first line.
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   190
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   191
            XXX: currently we assume start_date also for the end of the file
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   192
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   193
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   194
        # store
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   195
        self.channel = channel
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   196
        self.path = path
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   197
        self.parser = parser
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   198
        self.start_date = start_date
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   199
        self.decoder = decoder
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   200
        self.sep = sep
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   201
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   202
        # open
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   203
        self.file = open(path, 'rb')
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   204
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   205
    def __iter__ (self) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   206
        """
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   207
            Yields a series of unicode lines, as read from the top of the file
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   208
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   209
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   210
        # seek to beginning
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   211
        self.file.seek(0)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   212
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   213
        # iterate over lines, decoding them as well
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   214
        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   215
    
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   216
    def read_full (self) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   217
        """
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   218
            Reads all LogLines. The LogLines will have a valid offset.
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   219
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   220
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   221
        # just use our __iter__
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   222
        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   223
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   224
    def read_from (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   225
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   226
            Reads all LogLines from the given naive timestamp onwards
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   227
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   228
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   229
        # start reading at beginning
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   230
        events = self.read_full()
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   231
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   232
        # skip unwanted events
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   233
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   234
            if event.timestamp < dt :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   235
                continue
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   236
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   237
            else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   238
                # include this line as well
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   239
                yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   240
                break
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   241
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   242
        # yield the rest as-is
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   243
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   244
            yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   245
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   246
    def read_until (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   247
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   248
            Reads all LogLines up until the given naive timestamp
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   249
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   250
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   251
        # start reading events at the beginning
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   252
        events = self.read_full()
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   253
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   254
        # yield events until we hit the given timestamp
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   255
        for event in events :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   256
            if event.timestamp <= dt :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   257
                yield event
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   258
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   259
            else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   260
                break
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   261
            
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   262
        # ignore the rest
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   263
        return
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   264
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   265
    def _read_blocks_reverse (self, blocksize=1024) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   266
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   267
            Yields blocks of file data in reverse order, starting at the end of the file
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   268
        """
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   269
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   270
        # seek to end of file
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   271
        self.file.seek(0, os.SEEK_END)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   272
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   273
        # read offset
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   274
        # XXX: hack -1 to get rid of trailing newline
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   275
        size = offset = self.file.tell() - 1
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   276
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   277
        # do not try to read past the beginning of the file
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   278
        while offset > 0:
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   279
            # calc new offset + size
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   280
            if offset > blocksize :
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   281
                # full block
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   282
                offset -= blocksize
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   283
                read_size = blocksize
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   284
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   285
            else :
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   286
                # partial block
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   287
                read_size = offset
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   288
                offset = 0
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   289
43
fc11c4e86a82 implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents: 41
diff changeset
   290
            # seek to offset
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   291
            self.file.seek(offset)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   292
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   293
            # read the data we want
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   294
            block = self.file.read(read_size)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   295
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   296
            # sanity check
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   297
            assert len(block) == read_size
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   298
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   299
            # yield 
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   300
            yield block
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   301
    
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   302
    def _read_lines_reverse (self) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   303
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   304
            Yields decoded lines from the end of the file, in reverse order.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   305
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   306
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   307
        # partial lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   308
        buf = ''
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   309
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   310
        # read from end of file, a block at a time
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   311
        for block in self._read_blocks_reverse() :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   312
            # add in our previous buf
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   313
            buf = block + buf
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   314
            
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   315
            # split up lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   316
            lines = buf.split(self.sep)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   317
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   318
            # keep the first one as our buffer, as it's incomplete
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   319
            buf = lines[0]
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   320
           
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   321
            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   322
            # XXX: use something like islice, this has to build a slice object
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   323
            for line in lines[:0:-1] :
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   324
                yield self.decoder.decode(line)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   325
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   326
    def read_latest (self, count) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   327
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   328
            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   329
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   330
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   331
        # the list of lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   332
        lines = []
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   333
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   334
        # start reading lines into lines
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   335
        for line in self._read_lines_reverse() :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   336
            # append
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   337
            lines.append(line)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   338
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   339
            # done?
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   340
            if len(lines) >= count :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   341
                break
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   342
        
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   343
        # decode in reverse order, using our starting date....
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   344
        # XXX: use lines[::-1] or reversed?
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   345
        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   346
        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   347
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   348
class LogDirectory (LogSource) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   349
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   350
        A directory containing a series of timestamped LogFiles
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   351
    """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   352
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   353
    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   354
        """
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   355
            Load the logfiles at the given path, which are for the given LogChannel
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   356
            
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   357
            Decode the file lines using the given decoder, the files are named according the the date in the given
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   358
            timezone and date format, and will be parsed using the given parser.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   359
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   360
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   361
        # store
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   362
        self.channel = channel
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   363
        self.path = path
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   364
        self.tz = tz
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   365
        self.parser = parser
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 81
diff changeset
   366
        self.decoder = decoder
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   367
        self.filename_fmt = filename_fmt
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   368
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   369
    def _get_logfile_datetime (self, dt) :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   370
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   371
            Get the logfile corresponding to the given datetime
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   372
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   373
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   374
        # convert to target timezone
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   375
        dtz = dt.astimezone(self.tz)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   376
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   377
        # convert to date and use that
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   378
        return self._get_logfile_date(dtz.date())
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   379
97
6165f1ba458d implement parser/formatter netsplits and day-change
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   380
    def _get_logfile_date (self, d, load=True, stat=False, ignore_missing=True) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   381
        """
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   382
            Get the logfile corresponding to the given naive date in our timezone. 
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   383
            
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   384
            If load is False, only test for the presence of the logfile, do not actually open it. If stat is given,
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   385
            then this returns the stat() result
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   386
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   387
            Returns None if the logfile does not exist, unless ignore_missing is given as False.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   388
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   389
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   390
        # format filename
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   391
        filename = d.strftime(self.filename_fmt)
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   392
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   393
        # build path
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   394
        path = os.path.join(self.path, filename)
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   395
        
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   396
        try :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   397
            if load :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   398
                # open+return the LogFile
86
645cf9c4441e implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   399
                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   400
            
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   401
            elif stat :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   402
                # stat
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   403
                return os.stat(path)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   404
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   405
            else :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   406
                # test
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   407
                return os.path.exists(path)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   408
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   409
        # XXX: move to LogFile
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   410
        except IOError, e :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   411
            # return None for missing files
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   412
            if e.errno == errno.ENOENT and ignore_missing :
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   413
                return None
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   414
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   415
            else :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   416
                raise
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   417
    
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   418
    def _iter_logfile_dates (self, after=None) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   419
        """
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   420
            Yields a series of naive datetime objects representing the logfiles that are available, in time order.
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   421
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   422
            If after is given, only dates after said date will be returned
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   423
        """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   424
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   425
        # listdir
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   426
        filenames = os.listdir(self.path)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   427
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   428
        # sort
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   429
        filenames.sort()
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   430
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   431
        # iter files
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   432
        for filename in filenames :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   433
            try :
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   434
                # parse date
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   435
                date = datetime.datetime.strptime(filename, self.filename_fmt).replace(tzinfo=self.tz)
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   436
            
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   437
            except :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   438
                # ignore
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   439
                continue
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   440
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   441
            else :
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   442
                # ignore after?
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   443
                if after and date < after :
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   444
                    continue
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   445
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   446
                else :
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   447
                    # yield
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   448
                    yield date
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   449
            
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   450
    def _iter_date_reverse (self, dt=None) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   451
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   452
            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   453
            given *datetime*, or the the current date, if none given
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   454
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   455
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   456
        # default to now
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   457
        if not dt :
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   458
            dtz = self.tz.localize(datetime.datetime.now())
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   459
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   460
        else :
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   461
            # convert to target timezone
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   462
            dtz = dt.astimezone(self.tz)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   463
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   464
        # our timedelta
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   465
        ONE_DAY = datetime.timedelta(1)
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   466
        
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   467
        # iterate unto infinity
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   468
        while True :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   469
            # yield
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   470
            yield dtz.date()
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   471
            
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   472
            # one day sdrawkcab
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   473
            dtz -= ONE_DAY
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   474
    
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   475
    def _iter_logfile_reverse (self, dt=None, max_files=100) :
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   476
        """
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   477
            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   478
            current date, if none given.
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   479
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   480
            Reads/probes at most max_files files.
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   481
        """
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   482
        
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   483
        # start counting at zero...
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   484
        file_count = 0
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   485
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   486
        # have we found any files at all so far?
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   487
        have_found = False
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   488
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   489
        # iterate backwards over days
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   490
        for day in self._iter_date_reverse(dt) :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   491
            # stop if we've handled enough files by now
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   492
            if file_count > max_files :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   493
                break
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   494
            
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   495
            # try and open the next logfile
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   496
            logfile = None
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   497
            
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   498
            file_count += 1
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   499
            logfile = self._get_logfile_date(day)
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   500
            
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   501
            # no logfile there?
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   502
            if not logfile :
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   503
                # hit our limit?
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   504
                if file_count > max_files :
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   505
                    # if we didn't find any logfiles at all, terminate rudely
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   506
                    if not have_found :
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   507
                        raise Exception("No recent logfiles found")
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   508
                    
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   509
                    else :
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   510
                        # stop looking, deal with what we've got
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   511
                        return
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   512
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   513
                else :
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   514
                    # skip to next day
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   515
                    continue
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   516
            
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   517
            # mark have_found
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   518
            have_found = True
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   519
63
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   520
            # yield it
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   521
            yield logfile
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   522
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   523
    def get_latest (self, count) :
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   524
        """
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   525
            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   526
        """
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   527
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   528
        # read the events into here
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   529
        lines = []
416560b82116 rudimentary search
Tero Marttila <terom@fixme.fi>
parents: 55
diff changeset
   530
        
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   531
        # start reading in those logfiles
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   532
        for logfile in self._iter_logfile_reverse() :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   533
            # read the events
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   534
            # XXX: use a queue
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   535
            lines = list(logfile.read_latest(count)) + lines
81
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   536
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   537
            # done?
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   538
            if len(lines) >= count :
745032a57803 add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents: 78
diff changeset
   539
                break
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   540
        
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   541
        # return the events
48
7858b7b8ffe3 fix bugs with file tailing
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   542
        return lines
41
9585441a4bfb working basic logs stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   543
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   544
    def get_date (self, dt) :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   545
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   546
            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   547
            differs from our native datetime, this may involve lines from more than one logfile.
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   548
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   549
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   550
        # begin/end of 24h period, in target timezone
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   551
        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   552
        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   553
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   554
        # as dates
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   555
        d_begin = dtz_begin.date() 
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   556
        d_end = dtz_end.date()
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   557
        
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   558
#        print
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   559
#        print "LogDirectory.get_date - %s" % dt
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   560
#        print "\t   %s %s" % (d_begin, dtz_begin)
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   561
#        print "\t-> %s %s" % (d_end, dtz_end)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   562
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   563
        # if they're the same, just pull the full log for that date
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   564
        if d_begin == d_end :
64
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   565
            # open that log
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   566
            logfile = self._get_logfile_date(d_begin)
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   567
            
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   568
            if not logfile :
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   569
                raise Exception("No logfile for date=%r" % (dt, ))
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   570
            
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   571
            # return the full data
cdb6403c2498 beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents: 63
diff changeset
   572
            return logfile.read_full()
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   573
        
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   574
        # otherwise, we need to pull two partial logs
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   575
        else :
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   576
            # open both of them
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   577
            f_begin = self._get_logfile_date(d_begin)
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   578
            f_end = self._get_logfile_date(d_end)
76
cc3ab2c39ded fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   579
            
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   580
            # chain together the two sources
55
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   581
            return itertools.chain(
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   582
                f_begin.read_from(dtz_begin), 
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   583
                f_end.read_until(dtz_end) if f_end else []
5667d2bbdc50 improve calendar further
Tero Marttila <terom@fixme.fi>
parents: 54
diff changeset
   584
            )
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   585
    
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   586
    def _iter_month_days (self, month) :
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   587
        """
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   588
            Iterates over the days of a month as dt objects with time=0
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   589
        """
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   590
        
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   591
        # there's at most 31 days in a month...
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   592
        for day in xrange(1, 32) :
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   593
            try :
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   594
                # try and build the datetime
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   595
                dt = datetime.datetime(month.year, month.month, day)
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   596
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   597
            except :
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   598
                # stop
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   599
                return
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   600
            
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   601
            else :
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   602
                # fix timezones + yield
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   603
                yield month.tzinfo.localize(dt)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents: 48
diff changeset
   604
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   605
    def get_month_days (self, month) :
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   606
        """
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   607
            Returns a set of dates for which logfiles are available in the given datetime's month
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   608
        """
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   609
        
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   610
        # iterate over month's days
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   611
        for dt in self._iter_month_days(month) :
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   612
            # date in our target timezone
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   613
            log_date = dt.astimezone(self.tz).date()
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   614
            
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   615
            # test for it
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   616
            if self._get_logfile_date(log_date, load=False) :
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   617
                # valid
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   618
                yield dt.date()
54
b65a95eb9f6b implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
   619
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   620
    def get_modified (self, dt=None, after=None) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   621
        """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   622
            Returns the contents off all logfiles with mtimes past the given date
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   623
        """
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   624
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   625
        # iterate through all available logfiles in date order, as datetimes, from the given date on
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   626
        for log_date in self._iter_logfile_dates(after) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   627
            # compare against dt?
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   628
            if dt :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   629
                # stat
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   630
                st = self._get_logfile_date(log_date, load=False, stat=True)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   631
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   632
                # not modified?
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   633
                if utils.from_utc_timestamp(st.st_mtime) < dt :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   634
                    # skip
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   635
                    continue
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   636
                
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   637
            # open
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   638
            logfile = self._get_logfile_date(log_date, ignore_missing=False)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   639
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   640
            # yield all lines
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   641
            for line in logfile.read_full() :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   642
                yield line
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 86
diff changeset
   643