5 """ |
5 """ |
6 |
6 |
7 # XXX: fix path |
7 # XXX: fix path |
8 import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..') |
8 import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..') |
9 |
9 |
|
10 import os, os.path |
10 import datetime, pytz |
11 import datetime, pytz |
11 |
12 |
12 # configuration and the LogSearchIndex module |
13 # configuration and the LogSearchIndex module |
13 import config, log_search, channels |
14 import config, utils, log_search, channels |
14 |
15 |
15 def _open_index (options, open_mode) : |
16 def _open_index (options, open_mode) : |
16 """ |
17 """ |
17 Opens the LogSearchIndex |
18 Opens the LogSearchIndex |
18 """ |
19 """ |
32 channel = config.LOG_CHANNELS.lookup(channel_name) |
33 channel = config.LOG_CHANNELS.lookup(channel_name) |
33 |
34 |
34 # return |
35 # return |
35 return index, channel |
36 return index, channel |
36 |
37 |
|
38 def _insert_lines (index, options, channel, lines) : |
|
39 """ |
|
40 Insert the given lines into the index. |
|
41 |
|
42 Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines |
|
43 """ |
|
44 |
|
45 # last date |
|
46 date = None |
|
47 |
|
48 # count |
|
49 count = 0 |
|
50 |
|
51 # iter lines |
|
52 for line in lines : |
|
53 # output new date header? |
|
54 if not options.quiet and (not date or line.timestamp.date() != date) : |
|
55 # previous date's line count? |
|
56 if date : |
|
57 print "OK: %d lines" % count |
|
58 |
|
59 # reset count |
|
60 count = 0 |
|
61 |
|
62 # timestamp's date |
|
63 date = line.timestamp.date() |
|
64 |
|
65 # status header |
|
66 print "%s:" % (date.strftime('%Y-%m-%d'), ), |
|
67 |
|
68 # insert |
|
69 index.insert_line(channel, line) |
|
70 |
|
71 # count |
|
72 count += 1 |
|
73 |
|
74 # final count line |
|
75 if not options.quiet and date : |
|
76 print "OK: %d lines" % count |
|
77 |
37 def _load_channel_date (index, options, channel, date) : |
78 def _load_channel_date (index, options, channel, date) : |
38 """ |
79 """ |
39 Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex |
80 Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex |
40 """ |
81 """ |
41 |
82 |
42 if not options.quiet : |
83 if not options.quiet : |
43 print "%s %s..." % (channel.id, date.strftime(channel.source.filename_fmt)), |
84 print "Loading date for channel %s" % channel.id |
44 |
85 |
45 try : |
86 try : |
46 # load lines for date |
87 # load lines for date |
47 lines = channel.source.get_date(date) |
88 lines = channel.source.get_date(date) |
48 |
89 |
49 except Exception, e : |
90 except Exception, e : |
50 if not options.skip_missing : |
91 if not options.skip_missing : |
51 raise |
92 raise |
52 |
93 |
53 if not options.quiet : |
94 if not options.quiet : |
54 print "Skipped: %s" % (e, ) |
95 print "\tSkipped: %s" % (e, ) |
55 |
96 |
56 else : |
97 else : |
57 # insert -> count |
98 # insert |
58 count = index.insert(channel, lines) |
99 _insert_lines(index, options, channel, lines) |
59 |
|
60 if not options.quiet : |
|
61 print "OK: %d lines" % count |
|
62 |
|
63 |
100 |
64 def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') : |
101 def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') : |
65 """ |
102 """ |
66 Parse the given datetime, using the given timezone(defaults to options.tz) and format |
103 Parse the given datetime, using the given timezone(defaults to options.tz) and format |
67 """ |
104 """ |
206 lines = index.list(channel, date) |
243 lines = index.list(channel, date) |
207 |
244 |
208 # display |
245 # display |
209 _output_lines(options, lines) |
246 _output_lines(options, lines) |
210 |
247 |
|
248 def cmd_autoload (options, *channel_names) : |
|
249 """ |
|
250 Automatically loads all channel logs that have not been indexed yet (by logfile mtime) |
|
251 """ |
|
252 |
|
253 # open index |
|
254 index = _open_index(options, 'c' if options.create else 'a') |
|
255 |
|
256 # default to all channels |
|
257 if not channel_names : |
|
258 channels = config.LOG_CHANNELS |
|
259 |
|
260 else : |
|
261 channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names] |
|
262 |
|
263 # iterate channels |
|
264 for channel in channels : |
|
265 if not options.quiet : |
|
266 print "Channel %s:" % channel.id, |
|
267 |
|
268 # path to our state file |
|
269 statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id) |
|
270 |
|
271 # override? |
|
272 if options.reload : |
|
273 # load all |
|
274 mtime = None |
|
275 |
|
276 if not options.quiet : |
|
277 print "reloading all:", |
|
278 |
|
279 # stat for mtime |
|
280 # XXX: replace with single utils.mtime() |
|
281 elif os.path.exists(statefile_path) : |
|
282 # get last update date for channel |
|
283 mtime = utils.from_utc_timestamp(os.stat(statefile_path).st_mtime) |
|
284 |
|
285 if not options.quiet : |
|
286 print "last load=%s:" % mtime, |
|
287 |
|
288 else : |
|
289 # unknown, load all |
|
290 mtime = None |
|
291 |
|
292 if not options.quiet : |
|
293 print "no previous load state:", |
|
294 |
|
295 # get lines |
|
296 lines = channel.source.get_modified(mtime) |
|
297 |
|
298 # insert |
|
299 if not options.quiet : |
|
300 print "inserting..." |
|
301 |
|
302 _insert_lines(index, options, channel, lines) |
|
303 |
|
304 # write autoload state |
|
305 open(statefile_path, 'w').close() |
|
306 |
211 def cmd_help (options, *args) : |
307 def cmd_help (options, *args) : |
212 """ |
308 """ |
213 Help about commands |
309 Help about commands |
214 """ |
310 """ |
215 |
311 |
269 parser.add_option('-h', "--help", dest="help", help="Show this help message and exit", action="store_true") |
365 parser.add_option('-h', "--help", dest="help", help="Show this help message and exit", action="store_true") |
270 parser.add_option('-F', "--formatter", dest="formatter_name", help="LogFormatter to use", metavar="FMT", type="choice", default="irssi", |
366 parser.add_option('-F', "--formatter", dest="formatter_name", help="LogFormatter to use", metavar="FMT", type="choice", default="irssi", |
271 choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()]) |
367 choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()]) |
272 |
368 |
273 parser.add_option('-I', "--index", dest="index_path", help="Index database path", metavar="PATH", default="logs/index") |
369 parser.add_option('-I', "--index", dest="index_path", help="Index database path", metavar="PATH", default="logs/index") |
|
370 parser.add_option( "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir", metavar="PATH", default="logs/autoload-state") |
274 parser.add_option('-Z', "--timezone", dest="tz_name", help="Timezone for output", metavar="TZ", default="UTC") |
371 parser.add_option('-Z', "--timezone", dest="tz_name", help="Timezone for output", metavar="TZ", default="UTC") |
275 parser.add_option('-f', "--force", dest="force", help="Force dangerous operation", action="store_true") |
372 parser.add_option('-f', "--force", dest="force", help="Force dangerous operation", action="store_true") |
276 parser.add_option( "--create", dest="create", help="Create index database", action="store_true") |
373 parser.add_option( "--create", dest="create", help="Create index database", action="store_true") |
277 parser.add_option( "--skip-missing", dest="skip_missing", help="Skip missing logfiles", action="store_true") |
374 parser.add_option( "--skip-missing", dest="skip_missing", help="Skip missing logfiles", action="store_true") |
|
375 parser.add_option( "--reload", dest="reload", help="Force reload lines", action="store_true") |
278 parser.add_option( "--quiet", dest="quiet", help="Supress status messages", action="store_true") |
376 parser.add_option( "--quiet", dest="quiet", help="Supress status messages", action="store_true") |
279 |
377 |
280 # parse |
378 # parse |
281 options, args = parser.parse_args(argv[1:]) |
379 options, args = parser.parse_args(argv[1:]) |
282 |
380 |