author | Tero Marttila <terom@fixme.fi> |
Mon, 09 Feb 2009 03:05:43 +0200 | |
changeset 53 | 8103d18907a0 |
parent 50 | f13cf27a360b |
child 54 | b65a95eb9f6b |
permissions | -rw-r--r-- |
41 | 1 |
""" |
2 |
A source of IRC log files |
|
3 |
""" |
|
4 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
5 |
import datetime, itertools |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
6 |
import os, errno |
41 | 7 |
import pytz |
8 |
||
9 |
class LogSource (object) : |
|
10 |
""" |
|
11 |
A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events |
|
12 |
""" |
|
13 |
||
14 |
def get_latest (self, count) : |
|
15 |
""" |
|
16 |
Yield the latest events, up to `count` of them. |
|
17 |
""" |
|
18 |
||
19 |
abstract |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
20 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
21 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
22 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
23 |
Get logs for the given date (as a datetime) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
24 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
25 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
26 |
abstract |
41 | 27 |
|
28 |
class LogFile (LogSource) : |
|
29 |
""" |
|
30 |
A file containing LogEvents |
|
31 |
""" |
|
32 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
33 |
def __init__ (self, path, parser, start_date=None, charset='utf-8', sep='\n') : |
41 | 34 |
""" |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
35 |
Open the file at the given path, which contains data with the given charset, as lines separated by the |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
36 |
given separator. Lines are parsed using the given parser, using the given date as an initial date, see |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
37 |
LogParser for more info. XXX: currently we assume start_date also for the end of the file |
41 | 38 |
""" |
39 |
||
40 |
# store |
|
41 |
self.path = path |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
42 |
self.parser = parser |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
43 |
self.start_date = start_date |
41 | 44 |
self.charset = charset |
45 |
self.sep = sep |
|
46 |
||
47 |
# open |
|
48 | 48 |
self.file = open(path, 'rb') |
41 | 49 |
|
50 |
def __iter__ (self) : |
|
51 |
""" |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
52 |
Yields a series of unicode lines, as read from the top of the file |
41 | 53 |
""" |
54 |
||
55 |
# seek to beginning |
|
56 |
self.file.seek(0) |
|
57 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
58 |
# iterate over lines, decoding them as well |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
59 |
return (line.decode(self.charset) for line in self.file) |
41 | 60 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
61 |
def read_full (self) : |
41 | 62 |
""" |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
63 |
Reads all LogLines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
64 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
65 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
66 |
# just use our __iter__ |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
67 |
return self.parser.parse_lines(self, self.start_date) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
68 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
69 |
def read_from (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
70 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
71 |
Reads all LogLines from the given naive timestamp onwards |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
72 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
73 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
74 |
# start reading at beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
75 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
76 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
77 |
# skip unwanted events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
78 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
79 |
if event.timestamp < dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
80 |
continue |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
81 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
82 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
83 |
# include this line as well |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
84 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
85 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
86 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
87 |
# yield the rest as-is |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
88 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
89 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
90 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
91 |
def read_until (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
92 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
93 |
Reads all LogLines up until the given naive timestamp |
41 | 94 |
""" |
95 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
96 |
# start reading events at the beginning |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
97 |
events = self.read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
98 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
99 |
# yield events until we hit the given timestamp |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
100 |
for event in events : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
101 |
if event.timestamp <= dt : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
102 |
yield event |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
103 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
104 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
105 |
break |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
106 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
107 |
# ignore the rest |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
108 |
return |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
109 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
110 |
def _read_blocks_reverse (self, blocksize=1024) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
111 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
112 |
Yields blocks of file data in reverse order, starting at the end of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
113 |
""" |
41 | 114 |
|
115 |
# seek to end of file |
|
116 |
self.file.seek(0, os.SEEK_END) |
|
117 |
||
118 |
# read offset |
|
48 | 119 |
# XXX: hack -1 to get rid of trailing newline |
120 |
size = offset = self.file.tell() - 1 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
121 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
122 |
# do not try to read past the beginning of the file |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
123 |
while offset > 0: |
48 | 124 |
# calc new offset + size |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
125 |
if offset > blocksize : |
48 | 126 |
# full block |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
127 |
offset -= blocksize |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
128 |
read_size = blocksize |
41 | 129 |
|
48 | 130 |
else : |
131 |
# partial block |
|
132 |
read_size = offset |
|
41 | 133 |
offset = 0 |
134 |
||
43
fc11c4e86a82
implement channel_view count, the query stuff, css, layout all need some cleanup :(
Tero Marttila <terom@fixme.fi>
parents:
41
diff
changeset
|
135 |
# seek to offset |
41 | 136 |
self.file.seek(offset) |
137 |
||
48 | 138 |
# read the data we want |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
139 |
block = self.file.read(read_size) |
41 | 140 |
|
48 | 141 |
# sanity check |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
142 |
assert len(block) == read_size |
41 | 143 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
144 |
# yield |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
145 |
yield block |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
146 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
147 |
def _read_lines_reverse (self) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
148 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
149 |
Yields decoded lines from the end of the file, in reverse order. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
150 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
151 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
152 |
# partial lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
153 |
buf = '' |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
154 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
155 |
# read from end of file, a block at a time |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
156 |
for block in self._read_blocks_reverse() : |
41 | 157 |
# add in our previous buf |
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
158 |
buf = block + buf |
41 | 159 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
160 |
# split up lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
161 |
lines = buf.split(self.sep) |
41 | 162 |
|
163 |
# keep the first one as our buffer, as it's incomplete |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
164 |
buf = lines[0] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
165 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
166 |
# yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
167 |
# XXX: use something like islice, this has to build a slice object |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
168 |
for line in lines[:0:-1] : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
169 |
yield line.decode(self.charset) |
41 | 170 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
171 |
def get_latest (self, count) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
172 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
173 |
Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
174 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
175 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
176 |
# the list of lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
177 |
lines = [] |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
178 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
179 |
# start reading lines into lines |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
180 |
for line in self._read_lines_reverse() : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
181 |
# append |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
182 |
lines.append(line) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
183 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
184 |
# done? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
185 |
if len(lines) >= count : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
186 |
break |
48 | 187 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
188 |
# decode in reverse order, using our starting date.... |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
189 |
# XXX: use lines[::-1] or reversed? |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
190 |
# XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
191 |
return self.parser.parse_lines(reversed(lines), self.start_date) |
41 | 192 |
|
193 |
class LogDirectory (LogSource) : |
|
194 |
""" |
|
195 |
A directory containing a series of timestamped LogFiles |
|
196 |
""" |
|
197 |
||
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
198 |
def __init__ (self, path, tz, parser, charset='utf-8', filename_fmt='%Y-%m-%d') : |
41 | 199 |
""" |
200 |
Load the logfiles at the given path. |
|
201 |
||
202 |
The files contain data in the given charset, and are named according the the date in the given timezone and |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
203 |
date format, and will be parsed using the given parser. |
41 | 204 |
""" |
205 |
||
206 |
# store |
|
207 |
self.path = path |
|
208 |
self.tz = tz |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
209 |
self.parser = parser |
41 | 210 |
self.charset = charset |
211 |
self.filename_fmt = filename_fmt |
|
212 |
||
213 |
def _get_logfile_datetime (self, dt) : |
|
214 |
""" |
|
215 |
Get the logfile corresponding to the given datetime |
|
216 |
""" |
|
217 |
||
218 |
# convert to target timezone |
|
219 |
dtz = dt.astimezone(self.tz) |
|
220 |
||
221 |
# convert to date and use that |
|
222 |
return self._get_logfile_date(dtz.date()) |
|
223 |
||
224 |
def _get_logfile_date (self, d) : |
|
225 |
""" |
|
226 |
Get the logfile corresponding to the given naive date in our timezone |
|
227 |
""" |
|
228 |
||
229 |
# format filename |
|
230 |
filename = d.strftime(self.filename_fmt) |
|
231 |
||
232 |
# build path |
|
233 |
path = os.path.join(self.path, filename) |
|
234 |
||
235 |
# return the LogFile |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
236 |
return LogFile(path, self.parser, d, self.charset) |
41 | 237 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
238 |
def _iter_date_reverse (self, dt=None) : |
41 | 239 |
""" |
240 |
Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the |
|
241 |
given *datetime*, or the the current date, if none given |
|
242 |
""" |
|
243 |
||
244 |
# default to now |
|
245 |
if not dt : |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
246 |
dt = datetime.datetime.now(pytz.utc) |
41 | 247 |
|
248 |
# convert to target timezone |
|
249 |
dtz = dt.astimezone(self.tz) |
|
250 |
||
251 |
# our timedelta |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
252 |
ONE_DAY = datetime.timedelta(1) |
41 | 253 |
|
254 |
# iterate unto infinity |
|
255 |
while True : |
|
256 |
# yield |
|
257 |
yield dtz.date() |
|
258 |
||
259 |
# one day sdrawkcab |
|
260 |
dtz -= ONE_DAY |
|
261 |
||
262 |
def get_latest (self, count) : |
|
263 |
""" |
|
264 |
Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed |
|
265 |
""" |
|
266 |
||
267 |
# iterate backwards from now |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
268 |
day_iter = self._iter_date_reverse() |
41 | 269 |
|
270 |
# number of files read |
|
271 |
files = 0 |
|
272 |
||
273 |
# only read up to 100 files or so |
|
274 |
MAX_FILES = 100 |
|
48 | 275 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
276 |
# read the events into here |
48 | 277 |
lines = [] |
41 | 278 |
|
279 |
# loop until done |
|
48 | 280 |
while len(lines) < count : |
41 | 281 |
logfile = None |
282 |
||
283 |
try : |
|
284 |
# get next logfile |
|
285 |
files += 1 |
|
286 |
||
287 |
# open |
|
288 |
logfile = self._get_logfile_date(day_iter.next()) |
|
289 |
||
290 |
except IOError, e : |
|
291 |
# skip nonexistant days if we haven't found any logs yet |
|
292 |
if e.errno != errno.ENOENT : |
|
293 |
raise |
|
294 |
||
295 |
if files > MAX_FILES : |
|
296 |
raise Exception("No recent logfiles found") |
|
297 |
||
298 |
else : |
|
299 |
# skip to next day |
|
300 |
continue |
|
48 | 301 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
302 |
# read the events |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
303 |
# XXX: use a queue |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
304 |
lines = list(logfile.get_latest(count)) + lines |
48 | 305 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
306 |
# return the events |
48 | 307 |
return lines |
41 | 308 |
|
50
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
309 |
def get_date (self, dt) : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
310 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
311 |
A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
312 |
differs from our native datetime, this may involve lines from more than one logfile. |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
313 |
""" |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
314 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
315 |
# begin/end of 24h period, in target timezone |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
316 |
dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
317 |
dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
318 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
319 |
# as dates |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
320 |
d_begin = dtz_begin.date() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
321 |
d_end = dtz_end.date() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
322 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
323 |
# if they're the same, just pull the full log for that date |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
324 |
if d_begin == d_end : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
325 |
return self._get_logfile_date(d_begin).read_full() |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
326 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
327 |
# otherwise, we need to pull two partial logs |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
328 |
else : |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
329 |
# open both of them |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
330 |
f_begin = self._get_logfile_date(d_begin) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
331 |
f_end = self._get_logfile_date(d_end) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
332 |
|
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
333 |
# chain together the two sources |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
334 |
return itertools.chain(f_begin.read_from(dtz_begin), f_end.read_until(dtz_end)) |
f13cf27a360b
implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
48
diff
changeset
|
335 |