author | Tero Marttila <terom@fixme.fi> |
Tue, 10 Feb 2009 23:59:56 +0200 | |
changeset 88 | 0b8e2ba5f76f |
parent 83 | scripts/search-index.py@a34e9f56ddda |
child 89 | 2dc6de43f317 |
permissions | -rwxr-xr-x |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
1 |
#!/usr/bin/env python2.5 |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
2 |
|
65 | 3 |
""" |
4 |
Tool for accessing the search index |
|
5 |
""" |
|
6 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
7 |
# XXX: fix path |
65 | 8 |
import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..') |
9 |
||
10 |
import datetime, pytz |
|
11 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
12 |
# configuration and the LogSearchIndex module |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
13 |
import config, log_search, channels |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
14 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
15 |
def _open_index (options, open_mode) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
16 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
17 |
Opens the LogSearchIndex |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
18 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
19 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
20 |
return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
21 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
22 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
23 |
def _open_index_and_channel (options, channel_name, open_mode) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
24 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
25 |
Opens+returns a LogSearchIndex and a LogChannel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
26 |
""" |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
27 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
28 |
# open the LogSearchIndex |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
29 |
index = _open_index(options, open_mode) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
30 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
31 |
# open the channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
32 |
channel = config.LOG_CHANNELS.lookup(channel_name) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
33 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
34 |
# return |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
35 |
return index, channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
36 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
37 |
def _load_channel_date (index, options, channel, date) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
38 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
39 |
Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
40 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
41 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
42 |
if not options.quiet : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
43 |
print "%s %s..." % (channel.id, date.strftime(channel.source.filename_fmt)), |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
44 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
45 |
try : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
46 |
# load lines for date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
47 |
lines = channel.source.get_date(date) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
48 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
49 |
except Exception, e : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
50 |
if not options.skip_missing : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
51 |
raise |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
52 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
53 |
if not options.quiet : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
54 |
print "Skipped: %s" % (e, ) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
55 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
56 |
else : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
57 |
# insert -> count |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
58 |
count = index.insert(channel, lines) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
59 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
60 |
if not options.quiet : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
61 |
print "OK: %d lines" % count |
65 | 62 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
63 |
class CommandError (Exception) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
64 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
65 |
Error with command-line arguments |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
66 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
67 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
68 |
pass |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
69 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
70 |
def cmd_create (options) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
71 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
72 |
Creates a new index |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
73 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
74 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
75 |
# open index |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
76 |
index = _open_index(options, 'c' if not options.force else '*') |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
77 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
78 |
# that's all |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
79 |
pass |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
80 |
|
65 | 81 |
def cmd_load (options, channel_name, *dates) : |
82 |
""" |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
83 |
Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index |
65 | 84 |
""" |
85 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
86 |
# open index/channel |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
87 |
index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a') |
65 | 88 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
89 |
# handle each date |
65 | 90 |
for date_name in dates : |
68 | 91 |
try : |
92 |
# parse date |
|
93 |
date = datetime.datetime.strptime(date_name, '%Y-%m-%d').replace(tzinfo=channel.source.tz) |
|
65 | 94 |
|
68 | 95 |
except Exception, e : |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
96 |
print "[ERROR] Invalid date: %s: %s" % (date_name, e) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
97 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
98 |
if options.skip_missing : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
99 |
continue |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
100 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
101 |
else : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
102 |
raise |
68 | 103 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
104 |
# load |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
105 |
_load_channel_date(index, options, channel, date) |
65 | 106 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
107 |
def cmd_load_month (options, channel_name, *months) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
108 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
109 |
Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
110 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
111 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
112 |
# open index/channel |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
113 |
index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a') |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
114 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
115 |
# handle each date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
116 |
for month_name in months : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
117 |
try : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
118 |
# parse date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
119 |
month = datetime.datetime.strptime(month_name, '%Y-%m').replace(tzinfo=channel.source.tz) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
120 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
121 |
except Exception, e : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
122 |
print "[ERROR] Invalid date: %s: %s" % (month_name, e) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
123 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
124 |
if options.skip_missing : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
125 |
continue |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
126 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
127 |
else : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
128 |
raise |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
129 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
130 |
# get the set of days |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
131 |
days = list(channel.source.get_month_days(month)) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
132 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
133 |
print "Loading %d days of logs:" % (len(days)) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
134 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
135 |
# load each day |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
136 |
for date in days : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
137 |
# convert to datetime |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
138 |
dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
139 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
140 |
# load |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
141 |
_load_channel_date(index, options, channel, dt) |
65 | 142 |
|
143 |
def cmd_search (options, channel_name, query) : |
|
144 |
""" |
|
145 |
Search the index for events on a specific channel with the given query |
|
146 |
""" |
|
147 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
148 |
# sanity-check |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
149 |
if options.create_index : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
150 |
raise Exception("--create doesn't make sense for 'search'") |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
151 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
152 |
# open index/channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
153 |
index, channel = _open_index_and_channel(options, channel_name, 'r') |
65 | 154 |
|
155 |
# search |
|
156 |
lines = index.search_simple(channel, query) |
|
157 |
||
158 |
# display as plaintext |
|
159 |
for line in options.formatter.format_txt(lines) : |
|
160 |
print line |
|
161 |
||
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
162 |
def cmd_help (options, *args) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
163 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
164 |
Help about commands |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
165 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
166 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
167 |
import inspect |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
168 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
169 |
# general help stuff |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
170 |
options._parser.print_help() |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
171 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
172 |
# specific command? |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
173 |
if args : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
174 |
# the command name |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
175 |
command, = args |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
176 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
177 |
# XXX: display info about specific command |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
178 |
xxx |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
179 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
180 |
# general |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
181 |
else : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
182 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
183 |
print "Available commands:" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
184 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
185 |
# build list of all cmd_* objects |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
186 |
cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
187 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
188 |
# sort alphabetically |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
189 |
cmd_objects.sort() |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
190 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
191 |
# iterate through all cmd_* objects |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
192 |
for cmd_func_name, cmd_func in cmd_objects : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
193 |
# remove cmd_ prefix |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
194 |
cmd_name = cmd_func_name[4:] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
195 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
196 |
# inspect |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
197 |
cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
198 |
cmd_doc = inspect.getdoc(cmd_func) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
199 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
200 |
# remove the "options" arg |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
201 |
cmd_args = cmd_args[1:] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
202 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
203 |
# display |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
204 |
print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
205 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
206 |
def main (argv) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
207 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
208 |
Command-line main, with given argv |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
209 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
210 |
|
65 | 211 |
from optparse import OptionParser |
212 |
||
213 |
# define parser |
|
214 |
parser = OptionParser( |
|
215 |
usage = "%prog [options] <command> [ ... ]", |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
216 |
add_help_option = False, |
65 | 217 |
) |
218 |
||
219 |
# define command-line arguments |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
220 |
parser.add_option('-h', "--help", dest="help", help="Show this help message and exit", action="store_true") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
221 |
parser.add_option('-F', "--formatter", dest="formatter_name", help="LogFormatter to use", metavar="FMT", type="choice", default="irssi", |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
222 |
choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()]) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
223 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
224 |
parser.add_option('-I', "--index", dest="index_path", help="Index database path", metavar="PATH", default="logs/index") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
225 |
parser.add_option('-Z', "--timezone", dest="tz_name", help="Timezone for output", metavar="TZ", default="UTC") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
226 |
parser.add_option('-f', "--force", dest="force", help="Force dangerous operation", action="store_true") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
227 |
parser.add_option( "--create", dest="create", help="Create index database", action="store_true") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
228 |
parser.add_option( "--skip-missing", dest="skip_missing", help="Skip missing logfiles", action="store_true") |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
229 |
parser.add_option( "--quiet", dest="quiet", help="Supress status messages", action="store_true") |
65 | 230 |
|
231 |
# parse |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
232 |
options, args = parser.parse_args(argv[1:]) |
65 | 233 |
|
234 |
# postprocess stuff |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
235 |
options._parser = parser |
65 | 236 |
options.tz = pytz.timezone(options.tz_name) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
237 |
options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None) |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
238 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
239 |
# special-case --help |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
240 |
if options.help : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
241 |
return cmd_help(options, *args) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
242 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
243 |
# must have at least the command argument |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
244 |
if not args : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
245 |
raise CommandError("Missing command") |
65 | 246 |
|
247 |
# pop command |
|
248 |
command = args.pop(0) |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
249 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
250 |
# get func |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
251 |
func = globals().get('cmd_%s' % command) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
252 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
253 |
# unknown command? |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
254 |
if not func : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
255 |
raise CommandError("Unknown command: %s" % command) |
65 | 256 |
|
257 |
# call |
|
258 |
func(options, *args) |
|
259 |
||
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
260 |
if __name__ == '__main__' : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
261 |
try : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
262 |
main(sys.argv) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
263 |
sys.exit(0) |
65 | 264 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
265 |
except CommandError, e : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
266 |
print e |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
267 |
sys.exit(1) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
268 |