author | Tero Marttila <terom@fixme.fi> |
Wed, 11 Feb 2009 22:24:55 +0200 | |
changeset 111 | 95c0c49d76aa |
parent 106 | 0690d715385d |
child 112 | 090192b64d7e |
permissions | -rw-r--r-- |
41 | 1 |
""" |
2 |
A source of IRC log files |
|
3 |
""" |
|
4 |
||
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
5 |
import datetime, calendar, itertools, functools, math |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
6 |
import os, os.path, errno |
41 | 7 |
import pytz |
8 |
||
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
9 |
import config, utils |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
10 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
11 |
# a timedelta that represents one day |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
12 |
ONE_DAY = datetime.timedelta(days=1) |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
13 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
14 |
class LogSourceDecoder (object) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
15 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
16 |
Handles decoding of LogSource lines |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
17 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
18 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
19 |
def __init__ (self, encoding_list) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
20 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
21 |
Will try each of the given (charset, errors) items in turn, until one succeeds |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
22 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
23 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
24 |
self.encoding_list = encoding_list |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
25 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
26 |
def decode (self, line) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
27 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
28 |
Decode the line of str() text into an unicode object |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
29 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
30 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
31 |
# list of errors encountered |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
32 |
error_list = [] |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
33 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
34 |
# try each in turn |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
35 |
for charset, errors in self.encoding_list : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
36 |
# trap UnicodeDecodeError to try with the next one |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
37 |
try : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
38 |
return line.decode(charset, errors) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
39 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
40 |
except UnicodeDecodeError, e : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
41 |
error_list.append("%s:%s - %s" % (charset, errors, e)) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
42 |
continue |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
43 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
44 |
# failure |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
45 |
raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list))) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
46 |
|
41 | 47 |
class LogSource (object) : |
48 |
""" |
|
49 |
A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events |
|
50 |
""" |
|
51 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
52 |
def __init__ (self, decoder, channel=None) : |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
53 |
""" |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
54 |
The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
55 |
known, then it can be given as None, and set later with bind_channel. |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
56 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
57 |
Uses the given LogSourceDecoder to decode the lines. |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
58 |
""" |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
59 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
60 |
self.channel = channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
61 |
self.decoder = decoder |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
62 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
63 |
def bind_channel (self, channel) : |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
64 |
""" |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
65 |
Set this source's channel, where None was set before |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
66 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
67 |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
68 |
assert not self.channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
69 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
70 |
self.channel = channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
71 |
|
41 | 72 |
def get_latest (self, count) : |
73 |
""" |
|
74 |
Yield the latest events, up to `count` of them. |
|
75 |
""" |
|
76 |
||
77 |
abstract |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
78 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
79 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
80 |
""" |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
81 |
Get logs for the given date (as a datetime). |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
82 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
83 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
84 |
abstract |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
85 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
86 |
def get_date_paged (self, dt, count, page=None) : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
87 |
""" |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
88 |
Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
89 |
portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None, |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
90 |
then the lines for the page containing the given timestamp is returned. |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
91 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
92 |
The return value is a (page, max, lines) tuple. |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
93 |
""" |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
94 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
95 |
# how to act? |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
96 |
if page : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
97 |
# constant skip |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
98 |
skip = (page - 1) * count |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
99 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
100 |
else : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
101 |
skip = None |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
102 |
|
78
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
103 |
# go through the logs a page at a time |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
104 |
this_page = 1 |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
105 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
106 |
# last line's timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
107 |
last_ts = None |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
108 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
109 |
# found it yet? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
110 |
found = False |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
111 |
|
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
112 |
# count the full number of lines |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
113 |
line_count = 0 |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
114 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
115 |
# collect lines |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
116 |
lines = [] |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
117 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
118 |
# iterate using get_date |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
119 |
for line in self.get_date(dt) : |
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
120 |
# count them |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
121 |
line_count += 1 |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
122 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
123 |
# skip? |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
124 |
if skip : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
125 |
skip -= 1 |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
126 |
continue |
78
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
127 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
128 |
# is this page all that we want/need? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
129 |
if page or found : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
130 |
# already full? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
131 |
if len(lines) >= count : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
132 |
continue |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
133 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
134 |
# specfic timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
135 |
else : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
136 |
# didn't find it in this page? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
137 |
if len(lines) >= count : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
138 |
# reset to next page |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
139 |
lines = [] |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
140 |
this_page += 1 |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
141 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
142 |
# is dt between these two timestamps? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
143 |
if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
144 |
# found! |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
145 |
found = True |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
146 |
page = this_page |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
147 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
148 |
else : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
149 |
# keep looking |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
150 |
last_ts = line.timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
151 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
152 |
# store line |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
153 |
lines.append(line) |
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
154 |
|
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
155 |
# calculate max_pages |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
156 |
max_pages = math.ceil(float(line_count) / count) |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
157 |
|
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
158 |
# return |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
159 |
return (page, max_pages, lines) |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
160 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
161 |
def get_month_days (self, dt) : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
162 |
""" |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
163 |
Return an ordered sequence of dates, telling which days in the given month (as a datetime) have logs available. |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
164 |
""" |
41 | 165 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
166 |
abstract |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
167 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
168 |
def get_modified (self, dt=None, after=None, until=None) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
169 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
170 |
Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
171 |
|
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
172 |
If the datetime is not given, *all* lines are returned. |
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
173 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
174 |
If after is given, only lines from said date onwards will be returned, regardless of modification. |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
175 |
If until is given, only lines up to and including said date will be returned, regardless of modification. |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
176 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
177 |
The LogLines should be in time order. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
178 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
179 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
180 |
abstract |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
181 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
182 |
def get_prev_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
183 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
184 |
Get the next distinct date of logs available preceeding the given date, or None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
185 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
186 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
187 |
abstract |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
188 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
189 |
def get_next_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
190 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
191 |
Get the next distinct date of logs following the given date, or None. |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
192 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
193 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
194 |
abstract |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
195 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
196 |
class LogFile (object) : |
41 | 197 |
""" |
198 |
A file containing LogEvents |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
199 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
200 |
XXX: modify to implement LogSource? |
41 | 201 |
""" |
202 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
203 |
def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') : |
41 | 204 |
""" |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
205 |
Open the file at the given path, which contains lines as separated by the given separator. Lines are |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
206 |
decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
207 |
as the initial date for this log's first line. |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
208 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
209 |
XXX: currently we assume start_date also for the end of the file |
41 | 210 |
""" |
211 |
||
212 |
# store |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
213 |
self.channel = channel |
41 | 214 |
self.path = path |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
215 |
self.parser = parser |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
216 |
self.start_date = start_date |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
217 |
self.decoder = decoder |
41 | 218 |
self.sep = sep |
219 |
||
220 |
# open |
|
48 | 221 |
self.file = open(path, 'rb') |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
222 |
|
41 | 223 |
def __iter__ (self) : |
224 |
""" |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
225 |
Yields a series of unicode lines, as read from the top of the file |
41 | 226 |
""" |
227 |
||
228 |
# seek to beginning |
|
229 |
self.file.seek(0) |
|
230 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
231 |
# iterate over lines, decoding them as well |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
232 |
return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file) |
41 | 233 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
234 |
def read_full (self) : |
41 | 235 |
""" |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
236 |
Reads all LogLines. The LogLines will have a valid offset. |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
237 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
238 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
239 |
# just use our __iter__ |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
240 |
return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
241 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
242 |
def read_from (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
243 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
244 |
Reads all LogLines from the given naive timestamp onwards |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
245 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
246 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
247 |
# start reading at beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
248 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
249 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
250 |
# skip unwanted events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
251 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
252 |
if event.timestamp < dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
253 |
continue |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
254 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
255 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
256 |
# include this line as well |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
257 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
258 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
259 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
260 |
# yield the rest as-is |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
261 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
262 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
263 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
264 |
def read_until (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
265 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
266 |
Reads all LogLines up until the given naive timestamp |
41 | 267 |
""" |
268 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
269 |
# start reading events at the beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
270 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
271 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
272 |
# yield events until we hit the given timestamp |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
273 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
274 |
if event.timestamp <= dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
275 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
276 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
277 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
278 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
279 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
280 |
# ignore the rest |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
281 |
return |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
282 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
283 |
def _read_blocks_reverse (self, blocksize=1024) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
284 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
285 |
Yields blocks of file data in reverse order, starting at the end of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
286 |
""" |
41 | 287 |
|
288 |
# seek to end of file |
|
289 |
self.file.seek(0, os.SEEK_END) |
|
290 |
||
291 |
# read offset |
|
48 | 292 |
# XXX: hack -1 to get rid of trailing newline |
293 |
size = offset = self.file.tell() - 1 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
294 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
295 |
# do not try to read past the beginning of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
296 |
while offset > 0: |
48 | 297 |
# calc new offset + size |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
298 |
if offset > blocksize : |
48 | 299 |
# full block |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
300 |
offset -= blocksize |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
301 |
read_size = blocksize |
41 | 302 |
|
48 | 303 |
else : |
304 |
# partial block |
|
305 |
read_size = offset |
|
41 | 306 |
offset = 0 |
307 |
||
43
fc11c4e86a82
implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents:
41
diff
changeset
|
308 |
# seek to offset |
41 | 309 |
self.file.seek(offset) |
310 |
||
48 | 311 |
# read the data we want |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
312 |
block = self.file.read(read_size) |
41 | 313 |
|
48 | 314 |
# sanity check |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
315 |
assert len(block) == read_size |
41 | 316 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
317 |
# yield |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
318 |
yield block |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
319 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
320 |
def _read_lines_reverse (self) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
321 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
322 |
Yields decoded lines from the end of the file, in reverse order. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
323 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
324 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
325 |
# partial lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
326 |
buf = '' |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
327 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
328 |
# read from end of file, a block at a time |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
329 |
for block in self._read_blocks_reverse() : |
41 | 330 |
# add in our previous buf |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
331 |
buf = block + buf |
41 | 332 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
333 |
# split up lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
334 |
lines = buf.split(self.sep) |
41 | 335 |
|
336 |
# keep the first one as our buffer, as it's incomplete |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
337 |
buf = lines[0] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
338 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
339 |
# yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
340 |
# XXX: use something like islice, this has to build a slice object |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
341 |
for line in lines[:0:-1] : |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
342 |
yield self.decoder.decode(line) |
41 | 343 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
344 |
def read_latest (self, count) : |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
345 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
346 |
Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
347 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
348 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
349 |
# the list of lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
350 |
lines = [] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
351 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
352 |
# start reading lines into lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
353 |
for line in self._read_lines_reverse() : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
354 |
# append |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
355 |
lines.append(line) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
356 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
357 |
# done? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
358 |
if len(lines) >= count : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
359 |
break |
48 | 360 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
361 |
# decode in reverse order, using our starting date.... |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
362 |
# XXX: use lines[::-1] or reversed? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
363 |
# XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
364 |
return self.parser.parse_lines(self.channel, reversed(lines), self.start_date) |
41 | 365 |
|
366 |
class LogDirectory (LogSource) : |
|
367 |
""" |
|
368 |
A directory containing a series of timestamped LogFiles |
|
369 |
""" |
|
370 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
371 |
def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) : |
41 | 372 |
""" |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
373 |
Load the logfiles at the given path, which are for the given LogChannel |
41 | 374 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
375 |
Decode the file lines using the given decoder, the files are named according the the date in the given |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
376 |
timezone and date format, and will be parsed using the given parser. |
41 | 377 |
""" |
378 |
||
379 |
# store |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
380 |
self.channel = channel |
41 | 381 |
self.path = path |
382 |
self.tz = tz |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
383 |
self.parser = parser |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
384 |
self.decoder = decoder |
41 | 385 |
self.filename_fmt = filename_fmt |
386 |
||
387 |
def _get_logfile_datetime (self, dt) : |
|
388 |
""" |
|
389 |
Get the logfile corresponding to the given datetime |
|
390 |
""" |
|
391 |
||
392 |
# convert to target timezone |
|
393 |
dtz = dt.astimezone(self.tz) |
|
394 |
||
395 |
# convert to date and use that |
|
396 |
return self._get_logfile_date(dtz.date()) |
|
397 |
||
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
398 |
def _get_logfile_date (self, d, load=True, mtime=False, ignore_missing=True) : |
41 | 399 |
""" |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
400 |
Get the logfile corresponding to the given naive date in our timezone. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
401 |
|
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
402 |
If load is False, only test for the presence of the logfile, do not actually open it. If mtime is given, |
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
403 |
then this returns the file's mtime |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
404 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
405 |
Returns None if the logfile does not exist, unless ignore_missing is given as False. |
41 | 406 |
""" |
407 |
||
408 |
# format filename |
|
409 |
filename = d.strftime(self.filename_fmt) |
|
410 |
||
411 |
# build path |
|
412 |
path = os.path.join(self.path, filename) |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
413 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
414 |
try : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
415 |
if load : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
416 |
# open+return the LogFile |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
417 |
return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel) |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
418 |
|
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
419 |
elif mtime : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
420 |
# stat |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
421 |
return utils.mtime(path) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
422 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
423 |
else : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
424 |
# test |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
425 |
return os.path.exists(path) |
41 | 426 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
427 |
# XXX: move to LogFile |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
428 |
except IOError, e : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
429 |
# return None for missing files |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
430 |
if e.errno == errno.ENOENT and ignore_missing : |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
431 |
return None |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
432 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
433 |
else : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
434 |
raise |
41 | 435 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
436 |
def _iter_logfile_dates (self, after=None, until=None, reverse=False) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
437 |
""" |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
438 |
Yields a series of naive datetime objects representing the logfiles that are available, in time order. |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
439 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
440 |
Parameters : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
441 |
after only dates from said date onwards will be returned |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
442 |
until only dates up to and including said date will be returned |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
443 |
reverse the dates are returned in reverse order instead. Note that the meaning of after/until doesn't change |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
444 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
445 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
446 |
# listdir |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
447 |
filenames = os.listdir(self.path) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
448 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
449 |
# sort |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
450 |
filenames.sort(reverse=reverse) |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
451 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
452 |
# iter files |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
453 |
for filename in filenames : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
454 |
try : |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
455 |
# parse date |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
456 |
date = self.tz.localize(datetime.datetime.strptime(filename, self.filename_fmt)) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
457 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
458 |
except : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
459 |
# ignore |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
460 |
continue |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
461 |
|
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
462 |
else : |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
463 |
if (after and date < after) or (until and date > until) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
464 |
# ignore |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
465 |
continue |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
466 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
467 |
else : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
468 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
469 |
# print "iter_logfile_dates: after=%s, until=%s, reverse=%s -> %s" % (after, until, reverse, date) |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
470 |
|
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
471 |
# yield |
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
472 |
yield date |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
473 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
474 |
def _iter_date_reverse (self, dt=None) : |
41 | 475 |
""" |
476 |
Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the |
|
477 |
given *datetime*, or the the current date, if none given |
|
478 |
""" |
|
479 |
||
480 |
# default to now |
|
481 |
if not dt : |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
482 |
dtz = self.tz.localize(datetime.datetime.now()) |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
483 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
484 |
else : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
485 |
# convert to target timezone |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
486 |
dtz = dt.astimezone(self.tz) |
41 | 487 |
|
488 |
# iterate unto infinity |
|
489 |
while True : |
|
490 |
# yield |
|
491 |
yield dtz.date() |
|
492 |
||
493 |
# one day sdrawkcab |
|
494 |
dtz -= ONE_DAY |
|
495 |
||
63 | 496 |
def _iter_logfile_reverse (self, dt=None, max_files=100) : |
41 | 497 |
""" |
63 | 498 |
Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the |
499 |
current date, if none given. |
|
500 |
||
501 |
Reads/probes at most max_files files. |
|
41 | 502 |
""" |
503 |
||
63 | 504 |
# start counting at zero... |
505 |
file_count = 0 |
|
48 | 506 |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
507 |
# have we found any files at all so far? |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
508 |
have_found = False |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
509 |
|
63 | 510 |
# iterate backwards over days |
511 |
for day in self._iter_date_reverse(dt) : |
|
512 |
# stop if we've handled enough files by now |
|
513 |
if file_count > max_files : |
|
514 |
break |
|
515 |
||
516 |
# try and open the next logfile |
|
41 | 517 |
logfile = None |
518 |
||
63 | 519 |
file_count += 1 |
520 |
logfile = self._get_logfile_date(day) |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
521 |
|
63 | 522 |
# no logfile there? |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
523 |
if not logfile : |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
524 |
# hit our limit? |
63 | 525 |
if file_count > max_files : |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
526 |
# if we didn't find any logfiles at all, terminate rudely |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
527 |
if not have_found : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
528 |
raise Exception("No recent logfiles found") |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
529 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
530 |
else : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
531 |
# stop looking, deal with what we've got |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
532 |
return |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
533 |
|
41 | 534 |
else : |
535 |
# skip to next day |
|
536 |
continue |
|
48 | 537 |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
538 |
# mark have_found |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
539 |
have_found = True |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
540 |
|
63 | 541 |
# yield it |
542 |
yield logfile |
|
543 |
||
544 |
def get_latest (self, count) : |
|
545 |
""" |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
546 |
Uses _logfile_reverse to read the yield the given lines from as many logfiles as needed |
63 | 547 |
""" |
548 |
||
549 |
# read the events into here |
|
550 |
lines = [] |
|
551 |
||
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
552 |
# start reading in those logfiles |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
553 |
for logfile in self._iter_logfile_reverse() : |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
554 |
# read the events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
555 |
# XXX: use a queue |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
556 |
lines = list(logfile.read_latest(count)) + lines |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
557 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
558 |
# done? |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
559 |
if len(lines) >= count : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
560 |
break |
48 | 561 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
562 |
# return the events |
48 | 563 |
return lines |
41 | 564 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
565 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
566 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
567 |
A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
568 |
differs from our native datetime, this may involve lines from more than one logfile. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
569 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
570 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
571 |
# begin/end of 24h period, in target timezone |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
572 |
dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
573 |
dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
574 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
575 |
# as dates |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
576 |
d_begin = dtz_begin.date() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
577 |
d_end = dtz_end.date() |
72
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
578 |
|
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
579 |
|
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
580 |
# print "LogDirectory.get_date - %s" % dt |
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
581 |
# print "\t %s %s" % (d_begin, dtz_begin) |
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
582 |
# print "\t-> %s %s" % (d_end, dtz_end) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
583 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
584 |
# if they're the same, just pull the full log for that date |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
585 |
if d_begin == d_end : |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
586 |
# open that log |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
587 |
logfile = self._get_logfile_date(d_begin) |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
588 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
589 |
if not logfile : |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
590 |
raise Exception("No logfile for date=%r" % (dt, )) |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
591 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
592 |
# return the full data |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
593 |
return logfile.read_full() |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
594 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
595 |
# otherwise, we need to pull two partial logs |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
596 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
597 |
# open both of them |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
598 |
f_begin = self._get_logfile_date(d_begin) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
599 |
f_end = self._get_logfile_date(d_end) |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
600 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
601 |
# chain together the two sources |
55 | 602 |
return itertools.chain( |
603 |
f_begin.read_from(dtz_begin), |
|
604 |
f_end.read_until(dtz_end) if f_end else [] |
|
605 |
) |
|
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
606 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
607 |
def _iter_month_days (self, month) : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
608 |
""" |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
609 |
Iterates over the days of a month as dt objects with time=0 |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
610 |
""" |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
611 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
612 |
# there's at most 31 days in a month... |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
613 |
for day in xrange(1, 32) : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
614 |
try : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
615 |
# try and build the datetime |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
616 |
dt = datetime.datetime(month.year, month.month, day) |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
617 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
618 |
except : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
619 |
# stop |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
620 |
return |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
621 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
622 |
else : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
623 |
# fix timezones + yield |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
624 |
yield month.tzinfo.localize(dt) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
625 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
626 |
def get_month_days (self, month) : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
627 |
""" |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
628 |
Returns a set of dates for which logfiles are available in the given datetime's month |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
629 |
""" |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
630 |
|
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
631 |
# iterate over month's days |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
632 |
for dt in self._iter_month_days(month) : |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
633 |
# date in our target timezone |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
634 |
log_date = dt.astimezone(self.tz).date() |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
635 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
636 |
# test for it |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
637 |
if self._get_logfile_date(log_date, load=False) : |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
638 |
# valid |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
639 |
yield dt.date() |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
640 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
641 |
def get_modified (self, dt=None, after=None, until=None) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
642 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
643 |
Returns the contents off all logfiles with mtimes past the given date |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
644 |
""" |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
645 |
|
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
646 |
# iterate through all available logfiles in date order, as datetimes, from the given date on |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
647 |
for log_date in self._iter_logfile_dates(after, until) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
648 |
# compare against dt? |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
649 |
if dt : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
650 |
# stat |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
651 |
mtime = self._get_logfile_date(log_date, load=False, mtime=True) |
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
652 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
653 |
# not modified? |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
654 |
if mtime < dt : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
655 |
# skip |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
656 |
continue |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
657 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
658 |
# open |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
659 |
logfile = self._get_logfile_date(log_date, ignore_missing=False) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
660 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
661 |
# yield all lines |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
662 |
for line in logfile.read_full() : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
663 |
yield line |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
664 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
665 |
def get_prev_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
666 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
667 |
Just use _iter_logfile_dates |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
668 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
669 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
670 |
# use for to "iter" once |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
671 |
for log_date in self._iter_logfile_dates(until=dt - ONE_DAY, reverse=True) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
672 |
return log_date |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
673 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
674 |
else : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
675 |
return None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
676 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
677 |
def get_next_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
678 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
679 |
Just use _iter_logfile_dates |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
680 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
681 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
682 |
# use for to "iter" once |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
683 |
for log_date in self._iter_logfile_dates(after=dt + ONE_DAY) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
684 |
return log_date |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
685 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
686 |
else : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
687 |
return None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
688 |