author | terom |
Wed, 30 Mar 2016 01:29:22 +0300 | |
changeset 153 | 47dbaa6b13b0 |
parent 146 | 22d6b9525ae0 |
permissions | -rw-r--r-- |
41 | 1 |
""" |
2 |
A source of IRC log files |
|
3 |
""" |
|
4 |
||
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
5 |
import datetime, calendar, itertools, functools, math |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
6 |
import os, os.path, errno |
41 | 7 |
import pytz |
8 |
||
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
9 |
import config, utils |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
10 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
11 |
# a timedelta that represents one day |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
12 |
ONE_DAY = datetime.timedelta(days=1) |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
13 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
14 |
class LogSourceDecoder (object) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
15 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
16 |
Handles decoding of LogSource lines |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
17 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
18 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
19 |
def __init__ (self, encoding_list) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
20 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
21 |
Will try each of the given (charset, errors) items in turn, until one succeeds |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
22 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
23 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
24 |
self.encoding_list = encoding_list |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
25 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
26 |
def decode (self, line) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
27 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
28 |
Decode the line of str() text into an unicode object |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
29 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
30 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
31 |
# list of errors encountered |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
32 |
error_list = [] |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
33 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
34 |
# try each in turn |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
35 |
for charset, errors in self.encoding_list : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
36 |
# trap UnicodeDecodeError to try with the next one |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
37 |
try : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
38 |
return line.decode(charset, errors) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
39 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
40 |
except UnicodeDecodeError, e : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
41 |
error_list.append("%s:%s - %s" % (charset, errors, e)) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
42 |
continue |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
43 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
44 |
# failure |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
45 |
raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list))) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
46 |
|
41 | 47 |
class LogSource (object) : |
48 |
""" |
|
49 |
A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events |
|
50 |
""" |
|
51 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
52 |
def __init__ (self, decoder, channel=None) : |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
53 |
""" |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
54 |
The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
55 |
known, then it can be given as None, and set later with bind_channel. |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
56 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
57 |
Uses the given LogSourceDecoder to decode the lines. |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
58 |
""" |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
59 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
60 |
self.channel = channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
61 |
self.decoder = decoder |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
62 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
63 |
def bind_channel (self, channel) : |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
64 |
""" |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
65 |
Set this source's channel, where None was set before |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
66 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
67 |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
68 |
assert not self.channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
69 |
|
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
70 |
self.channel = channel |
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
71 |
|
41 | 72 |
def get_latest (self, count) : |
73 |
""" |
|
74 |
Yield the latest events, up to `count` of them. |
|
75 |
""" |
|
76 |
||
77 |
abstract |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
78 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
79 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
80 |
""" |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
81 |
Get logs for the given date (as a datetime). |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
82 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
83 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
84 |
abstract |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
85 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
86 |
def get_date_paged (self, dt, count, page=None) : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
87 |
""" |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
88 |
Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
89 |
portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None, |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
90 |
then the lines for the page containing the given timestamp is returned. |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
91 |
|
141
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
92 |
dt - the date to get logs for. If page is None, this is also the specific timestamp to page to |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
93 |
count - number of lines per page to return |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
94 |
page - specific page to return, or None to pick the right page for the given datetime |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
95 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
96 |
The return value is a (page, max, lines) tuple. |
141
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
97 |
page - the selected page |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
98 |
max - total number of pages |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
99 |
lines - the sequence of lines for the selected page |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
100 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
101 |
""" |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
102 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
103 |
# how to act? |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
104 |
if page : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
105 |
# constant skip |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
106 |
skip = (page - 1) * count |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
107 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
108 |
else : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
109 |
skip = None |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
110 |
|
78
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
111 |
# go through the logs a page at a time |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
112 |
this_page = 1 |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
113 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
114 |
# last line's timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
115 |
last_ts = None |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
116 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
117 |
# found it yet? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
118 |
found = False |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
119 |
|
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
120 |
# count the full number of lines |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
121 |
line_count = 0 |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
122 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
123 |
# collect lines |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
124 |
lines = [] |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
125 |
|
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
126 |
# iterate using get_date |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
127 |
for line in self.get_date(dt) : |
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
128 |
# count them |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
129 |
line_count += 1 |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
130 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
131 |
# skip? |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
132 |
if skip : |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
133 |
skip -= 1 |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
134 |
continue |
78
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
135 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
136 |
# is this page all that we want/need? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
137 |
if page or found : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
138 |
# already full? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
139 |
if len(lines) >= count : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
140 |
continue |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
141 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
142 |
# specfic timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
143 |
else : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
144 |
# didn't find it in this page? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
145 |
if len(lines) >= count : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
146 |
# reset to next page |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
147 |
lines = [] |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
148 |
this_page += 1 |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
149 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
150 |
# is dt between these two timestamps? |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
151 |
if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
152 |
# found! |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
153 |
found = True |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
154 |
page = this_page |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
155 |
|
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
156 |
else : |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
157 |
# keep looking |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
158 |
last_ts = line.timestamp |
85345abbd46a
implement pagination for channel_link
Tero Marttila <terom@fixme.fi>
parents:
77
diff
changeset
|
159 |
|
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
160 |
# store line |
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
161 |
lines.append(line) |
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
162 |
|
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
163 |
# calculate max_pages |
146
22d6b9525ae0
log_source: fix get_date_paged returning max as float
Tero Marttila <terom@fixme.fi>
parents:
141
diff
changeset
|
164 |
max_pages = int(math.ceil(float(line_count) / count)) |
77
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
165 |
|
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
166 |
# return |
4287fb77e312
implement max_pages, and paginate channel_date by default now
Tero Marttila <terom@fixme.fi>
parents:
76
diff
changeset
|
167 |
return (page, max_pages, lines) |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
168 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
169 |
def get_month_days (self, dt) : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
170 |
""" |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
171 |
Return an ordered sequence of dates, telling which days in the given month (as a datetime) have logs available. |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
172 |
""" |
41 | 173 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
174 |
abstract |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
175 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
176 |
def get_modified (self, dt=None, after=None, until=None) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
177 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
178 |
Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
179 |
|
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
180 |
If the datetime is not given, *all* lines are returned. |
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
181 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
182 |
If after is given, only lines from said date onwards will be returned, regardless of modification. |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
183 |
If until is given, only lines up to and including said date will be returned, regardless of modification. |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
184 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
185 |
The LogLines should be in time order. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
186 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
187 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
188 |
abstract |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
189 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
190 |
def get_prev_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
191 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
192 |
Get the next distinct date of logs available preceeding the given date, or None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
193 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
194 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
195 |
abstract |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
196 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
197 |
def get_next_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
198 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
199 |
Get the next distinct date of logs following the given date, or None. |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
200 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
201 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
202 |
abstract |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
203 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
204 |
class LogFile (object) : |
41 | 205 |
""" |
206 |
A file containing LogEvents |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
207 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
208 |
XXX: modify to implement LogSource? |
41 | 209 |
""" |
210 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
211 |
def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') : |
41 | 212 |
""" |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
213 |
Open the file at the given path, which contains lines as separated by the given separator. Lines are |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
214 |
decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
215 |
as the initial date for this log's first line. |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
216 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
217 |
XXX: currently we assume start_date also for the end of the file |
41 | 218 |
""" |
219 |
||
220 |
# store |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
221 |
self.channel = channel |
41 | 222 |
self.path = path |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
223 |
self.parser = parser |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
224 |
self.start_date = start_date |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
225 |
self.decoder = decoder |
41 | 226 |
self.sep = sep |
227 |
||
228 |
# open |
|
48 | 229 |
self.file = open(path, 'rb') |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
230 |
|
41 | 231 |
def __iter__ (self) : |
232 |
""" |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
233 |
Yields a series of unicode lines, as read from the top of the file |
41 | 234 |
""" |
235 |
||
236 |
# seek to beginning |
|
237 |
self.file.seek(0) |
|
238 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
239 |
# iterate over lines, decoding them as well |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
240 |
return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file) |
41 | 241 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
242 |
def read_full (self) : |
41 | 243 |
""" |
76
cc3ab2c39ded
fix off-by-one with search paginate, and implement basic pagination for channel_date
Tero Marttila <terom@fixme.fi>
parents:
73
diff
changeset
|
244 |
Reads all LogLines. The LogLines will have a valid offset. |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
245 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
246 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
247 |
# just use our __iter__ |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
248 |
return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
249 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
250 |
def read_from (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
251 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
252 |
Reads all LogLines from the given naive timestamp onwards |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
253 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
254 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
255 |
# start reading at beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
256 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
257 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
258 |
# skip unwanted events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
259 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
260 |
if event.timestamp < dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
261 |
continue |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
262 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
263 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
264 |
# include this line as well |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
265 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
266 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
267 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
268 |
# yield the rest as-is |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
269 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
270 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
271 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
272 |
def read_until (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
273 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
274 |
Reads all LogLines up until the given naive timestamp |
41 | 275 |
""" |
276 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
277 |
# start reading events at the beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
278 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
279 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
280 |
# yield events until we hit the given timestamp |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
281 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
282 |
if event.timestamp <= dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
283 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
284 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
285 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
286 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
287 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
288 |
# ignore the rest |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
289 |
return |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
290 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
291 |
def _read_blocks_reverse (self, blocksize=1024) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
292 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
293 |
Yields blocks of file data in reverse order, starting at the end of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
294 |
""" |
41 | 295 |
|
296 |
# seek to end of file |
|
297 |
self.file.seek(0, os.SEEK_END) |
|
298 |
||
299 |
# read offset |
|
48 | 300 |
# XXX: hack -1 to get rid of trailing newline |
301 |
size = offset = self.file.tell() - 1 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
302 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
303 |
# do not try to read past the beginning of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
304 |
while offset > 0: |
48 | 305 |
# calc new offset + size |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
306 |
if offset > blocksize : |
48 | 307 |
# full block |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
308 |
offset -= blocksize |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
309 |
read_size = blocksize |
41 | 310 |
|
48 | 311 |
else : |
312 |
# partial block |
|
313 |
read_size = offset |
|
41 | 314 |
offset = 0 |
315 |
||
43
fc11c4e86a82
implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents:
41
diff
changeset
|
316 |
# seek to offset |
41 | 317 |
self.file.seek(offset) |
318 |
||
48 | 319 |
# read the data we want |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
320 |
block = self.file.read(read_size) |
41 | 321 |
|
48 | 322 |
# sanity check |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
323 |
assert len(block) == read_size |
41 | 324 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
325 |
# yield |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
326 |
yield block |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
327 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
328 |
def _read_lines_reverse (self) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
329 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
330 |
Yields decoded lines from the end of the file, in reverse order. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
331 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
332 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
333 |
# partial lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
334 |
buf = '' |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
335 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
336 |
# read from end of file, a block at a time |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
337 |
for block in self._read_blocks_reverse() : |
41 | 338 |
# add in our previous buf |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
339 |
buf = block + buf |
41 | 340 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
341 |
# split up lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
342 |
lines = buf.split(self.sep) |
41 | 343 |
|
344 |
# keep the first one as our buffer, as it's incomplete |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
345 |
buf = lines[0] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
346 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
347 |
# yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
348 |
# XXX: use something like islice, this has to build a slice object |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
349 |
for line in lines[:0:-1] : |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
350 |
yield self.decoder.decode(line) |
41 | 351 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
352 |
def read_latest (self, count) : |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
353 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
354 |
Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
355 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
356 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
357 |
# the list of lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
358 |
lines = [] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
359 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
360 |
# start reading lines into lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
361 |
for line in self._read_lines_reverse() : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
362 |
# append |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
363 |
lines.append(line) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
364 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
365 |
# done? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
366 |
if len(lines) >= count : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
367 |
break |
48 | 368 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
369 |
# decode in reverse order, using our starting date.... |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
370 |
# XXX: use lines[::-1] or reversed? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
371 |
# XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
372 |
return self.parser.parse_lines(self.channel, reversed(lines), self.start_date) |
41 | 373 |
|
374 |
class LogDirectory (LogSource) : |
|
375 |
""" |
|
376 |
A directory containing a series of timestamped LogFiles |
|
377 |
""" |
|
378 |
||
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
379 |
def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) : |
41 | 380 |
""" |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
381 |
Load the logfiles at the given path, which are for the given LogChannel |
41 | 382 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
383 |
Decode the file lines using the given decoder, the files are named according the the date in the given |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
384 |
timezone and date format, and will be parsed using the given parser. |
41 | 385 |
""" |
386 |
||
387 |
# store |
|
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
388 |
self.channel = channel |
41 | 389 |
self.path = path |
390 |
self.tz = tz |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
391 |
self.parser = parser |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
81
diff
changeset
|
392 |
self.decoder = decoder |
41 | 393 |
self.filename_fmt = filename_fmt |
394 |
||
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
395 |
def _get_logfile_date (self, d, load=True, mtime=False, ignore_missing=False) : |
41 | 396 |
""" |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
397 |
Get the logfile corresponding to the given naive date in our timezone. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
398 |
|
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
399 |
If load is False, only test for the presence of the logfile, do not actually open it. If mtime is given, |
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
400 |
then this returns the file's mtime |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
401 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
402 |
Returns None if the logfile does not exist, unless ignore_missing is given as False. |
41 | 403 |
""" |
404 |
||
405 |
# format filename |
|
406 |
filename = d.strftime(self.filename_fmt) |
|
407 |
||
408 |
# build path |
|
409 |
path = os.path.join(self.path, filename) |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
410 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
411 |
try : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
412 |
if load : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
413 |
# open+return the LogFile |
86
645cf9c4441e
implement full parser+formatter for irssi
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
414 |
return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel) |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
415 |
|
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
416 |
elif mtime : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
417 |
# stat |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
418 |
return utils.mtime(path) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
419 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
420 |
else : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
421 |
# test |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
422 |
return os.path.exists(path) |
41 | 423 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
424 |
# XXX: move to LogFile |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
425 |
except IOError, e : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
426 |
# return None for missing files |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
427 |
if e.errno == errno.ENOENT and ignore_missing : |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
428 |
return None |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
429 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
430 |
else : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
431 |
raise |
41 | 432 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
433 |
def _iter_logfile_dates (self, after=None, until=None, reverse=False) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
434 |
""" |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
435 |
Yields a series of naive datetime objects representing the logfiles that are available, in time order. |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
436 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
437 |
Parameters : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
438 |
after only dates from said date onwards will be returned |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
439 |
until only dates up to and including said date will be returned |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
440 |
reverse the dates are returned in reverse order instead. Note that the meaning of after/until doesn't change |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
441 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
442 |
|
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
443 |
# convert timestamps to our timezone's dates |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
444 |
if after : |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
445 |
after = after.astimezone(self.tz).date() |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
446 |
|
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
447 |
if until : |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
448 |
until = until.astimezone(self.tz).date() |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
449 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
450 |
# listdir |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
451 |
filenames = os.listdir(self.path) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
452 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
453 |
# sort |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
454 |
filenames.sort(reverse=reverse) |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
455 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
456 |
# iter files |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
457 |
for filename in filenames : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
458 |
try : |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
459 |
# parse date |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
460 |
dt = self.tz.localize(datetime.datetime.strptime(filename, self.filename_fmt)) |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
461 |
date = dt.date() |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
462 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
463 |
except : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
464 |
# ignore |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
465 |
continue |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
466 |
|
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
467 |
else : |
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
468 |
if (after and date < after) or (until and date > until) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
469 |
# ignore |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
470 |
continue |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
471 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
472 |
else : |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
473 |
# yield |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
474 |
yield dt |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
475 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
476 |
def _iter_date_reverse (self, dt=None) : |
41 | 477 |
""" |
478 |
Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the |
|
479 |
given *datetime*, or the the current date, if none given |
|
480 |
""" |
|
481 |
||
482 |
# default to now |
|
483 |
if not dt : |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
484 |
dtz = self.tz.localize(datetime.datetime.now()) |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
485 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
486 |
else : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
487 |
# convert to target timezone |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
488 |
dtz = dt.astimezone(self.tz) |
41 | 489 |
|
490 |
# iterate unto infinity |
|
491 |
while True : |
|
492 |
# yield |
|
493 |
yield dtz.date() |
|
494 |
||
495 |
# one day sdrawkcab |
|
496 |
dtz -= ONE_DAY |
|
497 |
||
63 | 498 |
def _iter_logfile_reverse (self, dt=None, max_files=100) : |
41 | 499 |
""" |
63 | 500 |
Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the |
501 |
current date, if none given. |
|
502 |
||
503 |
Reads/probes at most max_files files. |
|
41 | 504 |
""" |
505 |
||
63 | 506 |
# start counting at zero... |
507 |
file_count = 0 |
|
48 | 508 |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
509 |
# have we found any files at all so far? |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
510 |
have_found = False |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
511 |
|
63 | 512 |
# iterate backwards over days |
513 |
for day in self._iter_date_reverse(dt) : |
|
514 |
# stop if we've handled enough files by now |
|
515 |
if file_count > max_files : |
|
516 |
break |
|
517 |
||
518 |
# try and open the next logfile |
|
41 | 519 |
logfile = None |
520 |
||
63 | 521 |
file_count += 1 |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
522 |
logfile = self._get_logfile_date(day, ignore_missing=True) |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
523 |
|
63 | 524 |
# no logfile there? |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
525 |
if not logfile : |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
526 |
# hit our limit? |
63 | 527 |
if file_count > max_files : |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
528 |
# if we didn't find any logfiles at all, terminate rudely |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
529 |
if not have_found : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
530 |
raise Exception("No recent logfiles found") |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
531 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
532 |
else : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
533 |
# stop looking, deal with what we've got |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
534 |
return |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
535 |
|
41 | 536 |
else : |
537 |
# skip to next day |
|
538 |
continue |
|
48 | 539 |
|
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
540 |
# mark have_found |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
541 |
have_found = True |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
542 |
|
63 | 543 |
# yield it |
544 |
yield logfile |
|
545 |
||
546 |
def get_latest (self, count) : |
|
547 |
""" |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
548 |
Uses _logfile_reverse to read the yield the given lines from as many logfiles as needed |
63 | 549 |
""" |
550 |
||
551 |
# read the events into here |
|
552 |
lines = [] |
|
553 |
||
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
554 |
# start reading in those logfiles |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
555 |
for logfile in self._iter_logfile_reverse() : |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
556 |
# read the events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
557 |
# XXX: use a queue |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
558 |
lines = list(logfile.read_latest(count)) + lines |
81
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
559 |
|
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
560 |
# done? |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
561 |
if len(lines) >= count : |
745032a57803
add #test logfile, improve handling of low-logfile situations
Tero Marttila <terom@fixme.fi>
parents:
78
diff
changeset
|
562 |
break |
48 | 563 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
564 |
# return the events |
48 | 565 |
return lines |
41 | 566 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
567 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
568 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
569 |
A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
570 |
differs from our native datetime, this may involve lines from more than one logfile. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
571 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
572 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
573 |
# begin/end of 24h period, in target timezone |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
574 |
dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
575 |
dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
576 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
577 |
# as dates |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
578 |
d_begin = dtz_begin.date() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
579 |
d_end = dtz_end.date() |
72
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
580 |
|
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
581 |
|
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
582 |
# print "LogDirectory.get_date - %s" % dt |
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
583 |
# print "\t %s %s" % (d_begin, dtz_begin) |
5ade0288f2ec
implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
584 |
# print "\t-> %s %s" % (d_end, dtz_end) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
585 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
586 |
# if they're the same, just pull the full log for that date |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
587 |
if d_begin == d_end : |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
588 |
# open that log |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
589 |
logfile = self._get_logfile_date(d_begin) |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
590 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
591 |
# return the full data |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
63
diff
changeset
|
592 |
return logfile.read_full() |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
593 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
594 |
# otherwise, we need to pull two partial logs |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
595 |
else : |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
596 |
# open both of them, but it's okay if we don't have the second one |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
597 |
f_begin = self._get_logfile_date(d_begin) |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
598 |
f_end = self._get_logfile_date(d_end, ignore_missing=True) |
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
599 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
600 |
# chain together the two sources |
55 | 601 |
return itertools.chain( |
602 |
f_begin.read_from(dtz_begin), |
|
603 |
f_end.read_until(dtz_end) if f_end else [] |
|
604 |
) |
|
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
605 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
606 |
def _iter_month_days (self, month) : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
607 |
""" |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
608 |
Iterates over the days of a month as dt objects with time=0 |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
609 |
""" |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
610 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
611 |
# there's at most 31 days in a month... |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
612 |
for day in xrange(1, 32) : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
613 |
try : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
614 |
# try and build the datetime |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
615 |
dt = datetime.datetime(month.year, month.month, day) |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
616 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
617 |
except : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
618 |
# stop |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
619 |
return |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
620 |
|
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
621 |
else : |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
622 |
# fix timezones + yield |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
623 |
yield month.tzinfo.localize(dt) |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
624 |
|
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
625 |
def get_month_days (self, month) : |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
626 |
""" |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
627 |
Returns a set of dates for which logfiles are available in the given datetime's month |
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
628 |
""" |
112
090192b64d7e
add three calendars to the channel_calendar view
Tero Marttila <terom@fixme.fi>
parents:
111
diff
changeset
|
629 |
|
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
630 |
# iterate over month's days |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
631 |
for dt in self._iter_month_days(month) : |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
632 |
# date in our target timezone |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
633 |
log_date = dt.astimezone(self.tz).date() |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
634 |
|
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
635 |
# test for it |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
636 |
if self._get_logfile_date(log_date, load=False, ignore_missing=True) : |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
637 |
# valid |
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
638 |
yield dt.date() |
54
b65a95eb9f6b
implement browse-by-date to show a nice calendar
Tero Marttila <terom@fixme.fi>
parents:
50
diff
changeset
|
639 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
640 |
def get_modified (self, dt=None, after=None, until=None) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
641 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
642 |
Returns the contents off all logfiles with mtimes past the given date |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
643 |
""" |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
644 |
|
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
645 |
# iterate through all available logfiles in date order, as datetimes, from the given date on |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
102
diff
changeset
|
646 |
for log_date in self._iter_logfile_dates(after, until) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
647 |
# compare against dt? |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
648 |
if dt : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
649 |
# stat |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
650 |
mtime = self._get_logfile_date(log_date, load=False, mtime=True, ignore_missing=True) |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
651 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
652 |
# not modified? |
102
e396613bc873
have LogDirectory use utils.mtime instead of os.stat
Tero Marttila <terom@fixme.fi>
parents:
97
diff
changeset
|
653 |
if mtime < dt : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
654 |
# skip |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
655 |
continue |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
656 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
657 |
# open |
115
751e3fcd11d2
have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents:
112
diff
changeset
|
658 |
logfile = self._get_logfile_date(log_date) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
659 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
660 |
# yield all lines |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
661 |
for line in logfile.read_full() : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
662 |
yield line |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
86
diff
changeset
|
663 |
|
111
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
664 |
def get_prev_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
665 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
666 |
Just use _iter_logfile_dates |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
667 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
668 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
669 |
# use for to "iter" once |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
670 |
for log_date in self._iter_logfile_dates(until=dt - ONE_DAY, reverse=True) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
671 |
return log_date |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
672 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
673 |
else : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
674 |
return None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
675 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
676 |
def get_next_date (self, dt) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
677 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
678 |
Just use _iter_logfile_dates |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
679 |
""" |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
680 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
681 |
# use for to "iter" once |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
682 |
for log_date in self._iter_logfile_dates(after=dt + ONE_DAY) : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
683 |
return log_date |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
684 |
|
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
685 |
else : |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
686 |
return None |
95c0c49d76aa
implement prev/next_date in LogSource
Tero Marttila <terom@fixme.fi>
parents:
106
diff
changeset
|
687 |