author | Tero Marttila <terom@fixme.fi> |
Thu, 12 Feb 2009 22:34:54 +0200 | |
changeset 119 | df859bfdd3be |
parent 106 | 0690d715385d |
child 121 | 86aebc9cb60b |
permissions | -rwxr-xr-x |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
1 |
#!/usr/bin/env python2.5 |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
2 |
|
65 | 3 |
""" |
4 |
Tool for accessing the search index |
|
5 |
""" |
|
6 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
7 |
# XXX: fix path |
65 | 8 |
import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..') |
9 |
||
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
10 |
import os, os.path, fcntl |
65 | 11 |
import datetime, pytz |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
12 |
import optparse |
65 | 13 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
14 |
# configuration and the LogSearchIndex module |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
15 |
import config, utils, log_search, channels |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
16 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
17 |
def _open_index (options, open_mode) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
18 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
19 |
Opens the LogSearchIndex |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
20 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
21 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
22 |
return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
23 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
24 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
25 |
def _open_index_and_channel (options, channel_name, open_mode) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
26 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
27 |
Opens+returns a LogSearchIndex and a LogChannel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
28 |
""" |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
29 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
30 |
# open the LogSearchIndex |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
31 |
index = _open_index(options, open_mode) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
32 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
33 |
# open the channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
34 |
channel = config.LOG_CHANNELS.lookup(channel_name) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
35 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
36 |
# return |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
37 |
return index, channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
38 |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
39 |
def _iter_insert_stats (index, channel, lines) : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
40 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
41 |
Insert the given lines into the index. |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
42 |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
43 |
Assumes the lines will be in time-order, and yields a series of (date, count) tuples for every date that lines |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
44 |
are inserted for |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
45 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
46 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
47 |
# last date |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
48 |
date = None |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
49 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
50 |
# count |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
51 |
count = 0 |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
52 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
53 |
# iter lines |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
54 |
for line in lines : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
55 |
# next day? |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
56 |
if not date or line.timestamp.date() != date : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
57 |
if date : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
58 |
# yield stats |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
59 |
yield date, count |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
60 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
61 |
# reset count |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
62 |
count = 0 |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
63 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
64 |
# timestamp's date |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
65 |
date = line.timestamp.date() |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
66 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
67 |
# insert |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
68 |
index.insert_line(channel, line) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
69 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
70 |
# count |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
71 |
count += 1 |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
72 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
73 |
# final count? |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
74 |
if date and count : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
75 |
yield date, count |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
76 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
77 |
def _insert_lines (index, options, channel, lines) : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
78 |
""" |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
79 |
Insert the given lines into the index. |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
80 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
81 |
Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
82 |
""" |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
83 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
84 |
# iterate insert stats |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
85 |
for date, count in _iter_insert_stats(index, channel, lines) : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
86 |
# output date header? |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
87 |
if not options.quiet : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
88 |
print "%s: %s" % (date.strftime('%Y-%m-%d'), count), |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
89 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
90 |
def _load_channel_date (index, options, channel, date) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
91 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
92 |
Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
93 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
94 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
95 |
if not options.quiet : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
96 |
print "Loading date for channel %s" % channel.id |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
97 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
98 |
try : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
99 |
# load lines for date |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
100 |
lines = channel.source.get_date(date) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
101 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
102 |
except Exception, e : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
103 |
if not options.skip_missing : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
104 |
raise |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
105 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
106 |
if not options.quiet : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
107 |
print "\tSkipped: %s" % (e, ) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
108 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
109 |
else : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
110 |
# insert |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
111 |
_insert_lines(index, options, channel, lines) |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
112 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
113 |
def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
114 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
115 |
Parse the given datetime, using the given timezone(defaults to options.tz) and format |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
116 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
117 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
118 |
# default tz |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
119 |
if not tz : |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
120 |
tz = options.timezone |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
121 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
122 |
try : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
123 |
# parse |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
124 |
return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
125 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
126 |
except Exception, e : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
127 |
raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e)) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
128 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
129 |
def _output_lines (options, lines) : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
130 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
131 |
Display the formatted LogLines |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
132 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
133 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
134 |
# display as plaintext |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
135 |
for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
136 |
print txt_data |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
137 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
138 |
class CommandError (Exception) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
139 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
140 |
Error with command-line arguments |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
141 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
142 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
143 |
pass |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
144 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
145 |
def cmd_create (options) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
146 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
147 |
Creates a new index |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
148 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
149 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
150 |
# open index |
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
151 |
index = _open_index(options, 'ctrunc' if options.force else 'c') |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
152 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
153 |
# that's all |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
154 |
pass |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
155 |
|
65 | 156 |
def cmd_load (options, channel_name, *dates) : |
157 |
""" |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
158 |
Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index |
65 | 159 |
""" |
160 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
161 |
# open index/channel |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
162 |
index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a') |
65 | 163 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
164 |
# handle each date |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
165 |
for date_str in dates : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
166 |
# prase date |
68 | 167 |
try : |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
168 |
date = _parse_date(options, date_str, channel.source.tz) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
169 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
170 |
# handle errors |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
171 |
except CommandError, e : |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
172 |
if options.skip_missing : |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
173 |
print "[ERROR] %s" % (date_name, e) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
174 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
175 |
else : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
176 |
raise |
68 | 177 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
178 |
# otherwise, load |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
179 |
else : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
180 |
_load_channel_date(index, options, channel, date) |
65 | 181 |
|
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
182 |
def cmd_load_month (options, channel_name, *months) : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
183 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
184 |
Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
185 |
""" |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
186 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
187 |
# open index/channel |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
188 |
index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a') |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
189 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
190 |
# handle each date |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
191 |
for month_str in months : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
192 |
# prase date |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
193 |
try : |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
194 |
month = _parse_date(options, month_str, channel.source.tz, '%Y-%m') |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
195 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
196 |
# handle errors |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
197 |
except CommandError, e : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
198 |
# skip? |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
199 |
if options.skip_missing : |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
200 |
print "[ERROR] %s" % (date_name, e) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
201 |
continue |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
202 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
203 |
else : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
204 |
raise |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
205 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
206 |
# get the set of days |
83
a34e9f56ddda
improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents:
82
diff
changeset
|
207 |
days = list(channel.source.get_month_days(month)) |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
208 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
209 |
print "Loading %d days of logs:" % (len(days)) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
210 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
211 |
# load each day |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
212 |
for date in days : |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
213 |
# convert to datetime |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
214 |
dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz) |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
215 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
216 |
# load |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
217 |
_load_channel_date(index, options, channel, dt) |
65 | 218 |
|
219 |
def cmd_search (options, channel_name, query) : |
|
220 |
""" |
|
221 |
Search the index for events on a specific channel with the given query |
|
222 |
""" |
|
223 |
||
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
224 |
# sanity-check |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
225 |
if options.create : |
82
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
226 |
raise Exception("--create doesn't make sense for 'search'") |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
227 |
|
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
228 |
# open index/channel |
afd3120ec71e
add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
229 |
index, channel = _open_index_and_channel(options, channel_name, 'r') |
65 | 230 |
|
231 |
# search |
|
232 |
lines = index.search_simple(channel, query) |
|
233 |
||
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
234 |
# display |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
235 |
_output_lines(options, lines) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
236 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
237 |
def cmd_list (options, channel_name, *dates) : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
238 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
239 |
List the indexed events for a specific date |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
240 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
241 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
242 |
# sanity-check |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
243 |
if options.create : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
244 |
raise Exception("--create doesn't make sense for 'search'") |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
245 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
246 |
# open index/channel |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
247 |
index, channel = _open_index_and_channel(options, channel_name, 'r') |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
248 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
249 |
# ...for each date |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
250 |
for date_str in dates : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
251 |
# parse date |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
252 |
date = _parse_date(options, date_str) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
253 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
254 |
# list |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
255 |
lines = index.list(channel, date) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
256 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
257 |
# display |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
88
diff
changeset
|
258 |
_output_lines(options, lines) |
65 | 259 |
|
101 | 260 |
def _autoload_reset (options, channels) : |
261 |
""" |
|
262 |
Reset old autoload state |
|
263 |
""" |
|
264 |
||
265 |
# warn |
|
266 |
if not options.quiet : |
|
267 |
print "[WARN] Resetting autoload state for: %s" % ', '.join(channel.id for channel in channels) |
|
268 |
||
269 |
# iter |
|
270 |
for channel in channels : |
|
271 |
# statefile path |
|
272 |
statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id) |
|
273 |
||
274 |
# is it present? |
|
275 |
if not os.path.exists(statefile_path) : |
|
276 |
if not options.quiet : |
|
277 |
print "[WARN] No statefile found at %s" % statefile_path |
|
278 |
||
279 |
else : |
|
280 |
if not options.quiet : |
|
281 |
print "\t%s: " % channel.id, |
|
282 |
||
283 |
# remove the statefile |
|
284 |
os.remove(statefile_path) |
|
285 |
||
286 |
if not options.quiet : |
|
287 |
print "OK" |
|
288 |
||
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
289 |
def cmd_autoload (options, *channel_names) : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
290 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
291 |
Automatically loads all channel logs that have not been indexed yet (by logfile mtime) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
292 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
293 |
|
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
294 |
# open index, nonblocking |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
295 |
index = _open_index(options, 'c?' if options.create else 'a?') |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
296 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
297 |
# default to all channels |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
298 |
if not channel_names : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
299 |
channels = config.LOG_CHANNELS |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
300 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
301 |
else : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
302 |
channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names] |
101 | 303 |
|
304 |
# reset autoload state? |
|
305 |
if options.reset : |
|
306 |
_autoload_reset(options, channels) |
|
307 |
||
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
308 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
309 |
# iterate channels |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
310 |
for channel in channels : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
311 |
if not options.quiet : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
312 |
print "Channel %s:" % channel.id |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
313 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
314 |
# no 'from' by default |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
315 |
after = None |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
316 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
317 |
# path to our state file |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
318 |
statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id) |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
319 |
statefile_tmppath = statefile_path + '.tmp' |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
320 |
|
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
321 |
# does it exist? |
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
322 |
have_tmpfile = os.path.exists(statefile_tmppath) |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
323 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
324 |
# do we have a tempfile from a previous crash? |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
325 |
if have_tmpfile and not options.ignore_resume : |
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
326 |
# first, open it... |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
327 |
statefile_tmp = open(statefile_tmppath, 'r+') |
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
328 |
|
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
329 |
# ... then lock it |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
330 |
fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB) |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
331 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
332 |
# read after timestamp |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
333 |
after_str = statefile_tmp.read().rstrip() |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
334 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
335 |
if after_str : |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
336 |
# parse timestamp |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
337 |
after = utils.from_utc_timestamp(int(after_str)) |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
338 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
339 |
if not options.quiet : |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
340 |
print "\tContinuing earlier progress from %s" % after |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
341 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
342 |
else : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
343 |
# ignore |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
344 |
if not options.quiet : |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
345 |
print "\t[WARN] Ignoring empty temporary statefile" |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
346 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
347 |
else : |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
348 |
# warn about old tmpfile that was ignored |
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
349 |
if have_tmpfile and not options.quiet : |
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
350 |
print "\t[WARN] Ignoring old tmpfile state" |
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
351 |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
352 |
# open new tempfile |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
353 |
statefile_tmp = open(statefile_tmppath, 'w') |
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
354 |
|
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
355 |
# lock |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
98
diff
changeset
|
356 |
fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB) |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
357 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
358 |
# override? |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
359 |
if options.reload : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
360 |
# load all |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
361 |
mtime = None |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
362 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
363 |
if not options.quiet : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
364 |
print "\tForcing reload!" |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
365 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
366 |
# stat for mtime |
95 | 367 |
else : |
368 |
# stat for mtime, None if unknown |
|
369 |
mtime = utils.mtime(statefile_path, ignore_missing=True) |
|
370 |
||
371 |
if mtime and not options.quiet : |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
372 |
print "\tLast load time was %s" % mtime |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
373 |
|
95 | 374 |
elif not options.quiet : |
100
147be2910b02
implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
375 |
print "\t[WARN] No previous load state! Loading full logs" |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
376 |
|
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
377 |
# only after some specific date? |
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
378 |
if options.after : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
379 |
# use unless read from tempfile |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
380 |
if not after : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
381 |
after = options.after |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
382 |
|
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
383 |
if not options.quiet : |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
384 |
print "\tOnly including dates from %s onwards" % after |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
385 |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
386 |
else : |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
387 |
if not options.quiet : |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
388 |
print "\t[WARN] Ignoring --from because we found a tempfile" |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
389 |
|
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
390 |
# only up to some specific date? |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
391 |
if options.until : |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
392 |
until = options.until |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
393 |
|
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
394 |
if not options.quiet : |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
395 |
print "\tOnly including dates up to (and including) %s" % until |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
396 |
else : |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
397 |
# default to now |
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
398 |
until = None |
94
6673de9bc911
implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
399 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
400 |
# get lines |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
401 |
lines = channel.source.get_modified(mtime, after, until) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
402 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
403 |
# insert |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
404 |
if not options.quiet : |
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
405 |
print "\tLoading and inserting..." |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
406 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
407 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
408 |
# iterate insert() per day to display info and update progress |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
409 |
for date, count in _iter_insert_stats(index, channel, lines) : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
410 |
# output date header? |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
411 |
if not options.quiet : |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
412 |
print "\t%10s: %d" % (date.strftime('%Y-%m-%d'), count) |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
413 |
|
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
414 |
# write temp state |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
415 |
statefile_tmp.seek(0) |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
416 |
statefile_tmp.write(str(utils.to_utc_timestamp(datetime.datetime.combine(date, datetime.time(0))))) |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
417 |
statefile_tmp.flush() |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
418 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
419 |
# write autoload state |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
420 |
open(statefile_path, 'w').close() |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
421 |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
422 |
# close+delete tempfile |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
423 |
statefile_tmp.close() |
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
424 |
os.remove(statefile_tmppath) |
101 | 425 |
|
426 |
||
427 |
||
428 |
# done |
|
429 |
return |
|
98
8c6e36849f9a
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents:
95
diff
changeset
|
430 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
431 |
def cmd_help (options, *args) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
432 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
433 |
Help about commands |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
434 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
435 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
436 |
import inspect |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
437 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
438 |
# general help stuff |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
439 |
options._parser.print_help() |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
440 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
441 |
# specific command? |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
442 |
if args : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
443 |
# the command name |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
444 |
command, = args |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
445 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
446 |
# XXX: display info about specific command |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
447 |
xxx |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
448 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
449 |
# general |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
450 |
else : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
451 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
452 |
print "Available commands:" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
453 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
454 |
# build list of all cmd_* objects |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
455 |
cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
456 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
457 |
# sort alphabetically |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
458 |
cmd_objects.sort() |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
459 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
460 |
# iterate through all cmd_* objects |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
461 |
for cmd_func_name, cmd_func in cmd_objects : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
462 |
# remove cmd_ prefix |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
463 |
cmd_name = cmd_func_name[4:] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
464 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
465 |
# inspect |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
466 |
cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
467 |
cmd_doc = inspect.getdoc(cmd_func) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
468 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
469 |
# remove the "options" arg |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
470 |
cmd_args = cmd_args[1:] |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
471 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
472 |
# display |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
473 |
print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
474 |
|
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
475 |
class MyOption (optparse.Option) : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
476 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
477 |
Our custom types for optparse |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
478 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
479 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
480 |
def check_date (option, opt, value) : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
481 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
482 |
Parse a date |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
483 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
484 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
485 |
try : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
486 |
# parse |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
487 |
return datetime.datetime.strptime(value, '%Y-%m-%d') |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
488 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
489 |
# trap -> OptionValueError |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
490 |
except Exception, e : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
491 |
raise optparse.OptionValueError("option %s: invalid date value: %r" % (opt, value)) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
492 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
493 |
def check_timezone (option, opt, value) : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
494 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
495 |
Parse a timezone |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
496 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
497 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
498 |
try : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
499 |
# parse |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
500 |
return pytz.timezone(value) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
501 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
502 |
# trap -> OptionValueError |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
503 |
except Exception, e : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
504 |
raise optparse.OptionValueError("option %s: invalid timezone: %r" % (opt, value)) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
505 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
506 |
def take_action (self, action, dest, opt, value, values, parser) : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
507 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
508 |
Override take_action to handle date |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
509 |
""" |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
510 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
511 |
if action == "parse_date" : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
512 |
# get timezone |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
513 |
tz = values.timezone |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
514 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
515 |
# set timezone |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
516 |
value = value.replace(tzinfo=tz) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
517 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
518 |
# store |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
519 |
return optparse.Option.take_action(self, 'store', dest, opt, value, values, parser) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
520 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
521 |
else : |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
522 |
# default |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
523 |
return optparse.Option.take_action(self, action, dest, opt, value, values, parser) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
524 |
|
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
525 |
TYPES = optparse.Option.TYPES + ('date', 'timezone') |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
526 |
TYPE_CHECKER = optparse.Option.TYPE_CHECKER.copy() |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
527 |
TYPE_CHECKER['date'] = check_date |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
528 |
TYPE_CHECKER['timezone'] = check_timezone |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
529 |
ACTIONS = optparse.Option.ACTIONS + ('parse_date', ) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
530 |
STORE_ACTIONS = optparse.Option.STORE_ACTIONS + ('parse_date', ) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
531 |
TYPED_ACTIONS = optparse.Option.TYPED_ACTIONS + ('parse_date', ) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
532 |
ACTIONS = optparse.Option.ACTIONS + ('parse_date', ) |
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
533 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
534 |
def main (argv) : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
535 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
536 |
Command-line main, with given argv |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
537 |
""" |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
538 |
|
65 | 539 |
# define parser |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
540 |
parser = optparse.OptionParser( |
65 | 541 |
usage = "%prog [options] <command> [ ... ]", |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
542 |
add_help_option = False, |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
543 |
option_class = MyOption, |
65 | 544 |
) |
545 |
||
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
546 |
# general options # # # # |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
547 |
general = optparse.OptionGroup(parser, "General Options") |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
548 |
general.add_option('-h', "--help", dest="help", help="Show this help message and exit", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
549 |
action="store_true" ) |
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
550 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
551 |
general.add_option( "--formatter", dest="formatter_name", help="LogFormatter to use", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
552 |
metavar="FMT", type="choice", default="irssi", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
553 |
choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()] ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
554 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
555 |
general.add_option( "--index", dest="index_path", help="Index database path", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
556 |
metavar="PATH", default="logs/index" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
557 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
558 |
general.add_option( "--timezone", dest="timezone", help="Timezone for output", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
559 |
metavar="TZ", type="timezone", default=pytz.utc ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
560 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
561 |
general.add_option( "--force", dest="force", help="Force dangerous operation", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
562 |
action="store_true" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
563 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
564 |
general.add_option( "--quiet", dest="quiet", help="Supress status messages", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
565 |
action="store_true" ) |
104
34c65a8c8b94
split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
566 |
parser.add_option_group(general) |
34c65a8c8b94
split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
567 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
568 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
569 |
# cmd_load options # # # # |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
570 |
load = optparse.OptionGroup(parser, "Load Options") |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
571 |
load.add_option( "--skip-missing", dest="skip_missing", help="Skip missing logfiles", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
572 |
action="store_true" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
573 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
574 |
load.add_option( "--create", dest="create", help="Create index database", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
575 |
action="store_true" ) |
104
34c65a8c8b94
split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
576 |
parser.add_option_group(load) |
34c65a8c8b94
split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
577 |
|
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
578 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
579 |
# cmd_autoload options # # # # |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
580 |
autoload = optparse.OptionGroup(parser, "Autoload Options") |
106
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
581 |
autoload.add_option( "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
582 |
metavar="PATH", default="logs/autoload-state" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
583 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
584 |
autoload.add_option( "--from", dest="after", help="Only autoload logfiles from the given date on", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
585 |
metavar="DATE", type="date", action="parse_date", default=None ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
586 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
587 |
autoload.add_option( "--until", dest="until", help="Only autoload logfiles up to (and including) the given date", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
588 |
metavar="DATE", type="date", action="parse_date", default=None ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
589 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
590 |
autoload.add_option( "--reload", dest="reload", help="Force reload lines", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
591 |
action="store_true" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
592 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
593 |
autoload.add_option( "--reset", dest="reset", help="Reset old autload state", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
594 |
action="store_true" ) |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
595 |
|
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
596 |
autoload.add_option( "--ignore-resume", dest="ignore_resume", help="Do not try and resume interrupted autoload", |
0690d715385d
reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents:
105
diff
changeset
|
597 |
action="store_true" ) |
104
34c65a8c8b94
split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents:
103
diff
changeset
|
598 |
parser.add_option_group(autoload) |
65 | 599 |
|
600 |
# parse |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
601 |
options, args = parser.parse_args(argv[1:]) |
65 | 602 |
|
603 |
# postprocess stuff |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
604 |
options._parser = parser |
105
e24da9a94ffb
implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents:
104
diff
changeset
|
605 |
options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.timezone, "%H:%M:%S", None, None) |
103
0e829e6275dc
implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents:
101
diff
changeset
|
606 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
607 |
# special-case --help |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
608 |
if options.help : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
609 |
return cmd_help(options, *args) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
610 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
611 |
# must have at least the command argument |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
612 |
if not args : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
613 |
raise CommandError("Missing command") |
65 | 614 |
|
615 |
# pop command |
|
616 |
command = args.pop(0) |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
617 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
618 |
# get func |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
619 |
func = globals().get('cmd_%s' % command) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
620 |
|
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
621 |
# unknown command? |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
622 |
if not func : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
623 |
raise CommandError("Unknown command: %s" % command) |
65 | 624 |
|
625 |
# call |
|
626 |
func(options, *args) |
|
627 |
||
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
628 |
if __name__ == '__main__' : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
629 |
try : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
630 |
main(sys.argv) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
631 |
sys.exit(0) |
65 | 632 |
|
88
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
633 |
except CommandError, e : |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
634 |
print e |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
635 |
sys.exit(1) |
0b8e2ba5f76f
improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents:
83
diff
changeset
|
636 |