author | Tero Marttila <terom@fixme.fi> |
Sun, 13 Sep 2009 20:11:00 +0300 | |
changeset 144 | 35c4c56f1376 |
parent 141 | 65c98c9e1716 |
permissions | -rw-r--r-- |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
1 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
2 |
Full-text searching of logs |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
3 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
4 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
5 |
import datetime, calendar, pytz |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
6 |
import os.path |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
7 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
8 |
import HyperEstraier as hype |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
9 |
|
96
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
10 |
import log_line, utils, config |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
11 |
|
74
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
12 |
class LogSearchError (Exception) : |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
13 |
""" |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
14 |
General search error |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
15 |
""" |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
16 |
|
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
17 |
pass |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
18 |
|
127
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
19 |
class SearchIndexError (LogSearchError) : |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
20 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
21 |
Error manipulating the index |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
22 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
23 |
|
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
24 |
def __init__ (self, msg, db) : |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
25 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
26 |
Build the error from the given message + HyperEstraier.Database |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
27 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
28 |
|
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
29 |
super(SearchIndexError, self).__init__("%s: %s" % (msg, db.err_msg(db.error()))) |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
30 |
|
74
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
31 |
class NoResultsFound (LogSearchError) : |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
32 |
""" |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
33 |
No results found |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
34 |
""" |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
35 |
|
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
36 |
pass |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
37 |
|
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
38 |
class LogSearchIndex (object) : |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
39 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
40 |
An index on the logs for a group of channels. |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
41 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
42 |
This uses Hyper Estraier to handle searching, whereby each log line is a document (yes, I have a powerful server). |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
43 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
44 |
These log documents have the following attributes: |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
45 |
@uri - channel/date/line |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
46 |
channel - channel code |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
47 |
type - the LogType id |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
48 |
timestamp - UTC timestamp |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
49 |
source_nickname - source nickname |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
50 |
source_username - source username |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
51 |
source_hostname - source hostname |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
52 |
source_chanflags - source channel flags |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
53 |
target_nickname - target nickname |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
54 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
55 |
Each document then has a single line of data, which is the log data message |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
56 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
57 |
|
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
58 |
def __init__ (self, channels, path, mode='r') : |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
59 |
""" |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
60 |
Open the database at the given path, with the given mode: |
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
61 |
First char: |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
62 |
r - read, error if not exists |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
63 |
w - write, create if not exists |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
64 |
a - write, error if not exists |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
65 |
c - create, error if exists |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
66 |
|
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
67 |
Additional chars: |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
68 |
trunc - truncate if exists |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
69 |
+ - read as well as write |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
70 |
? - non-blocking lock open, i.e. it fails if already open |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
71 |
|
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
72 |
Channels is the ChannelList. |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
73 |
""" |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
74 |
|
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
75 |
# store |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
76 |
self.channels = channels |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
77 |
self.path = path |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
78 |
self.mode = mode |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
79 |
|
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
80 |
# check it does not already exist? |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
81 |
if mode in 'c' and os.path.exists(path) : |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
82 |
raise LogSearchError("Index already exists: %s" % (path, )) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
83 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
84 |
# mapping of { mode -> flags } |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
85 |
mode_to_flag = { |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
86 |
'r': hype.Database.DBREADER, |
67
13975aa16b4c
fix LogSearchIndex open permissions
Tero Marttila <terom@fixme.fi>
parents:
66
diff
changeset
|
87 |
'w': hype.Database.DBWRITER | hype.Database.DBCREAT, |
13975aa16b4c
fix LogSearchIndex open permissions
Tero Marttila <terom@fixme.fi>
parents:
66
diff
changeset
|
88 |
'a': hype.Database.DBWRITER, |
121
86aebc9cb60b
some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents:
118
diff
changeset
|
89 |
'c': hype.Database.DBWRITER | hype.Database.DBCREAT, |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
90 |
} |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
91 |
|
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
92 |
# flags to use, standard modes |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
93 |
flags = mode_to_flag[mode[0]] |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
94 |
|
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
95 |
# mode-flags |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
96 |
if '?' in mode : |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
97 |
# non-blocking locking |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
98 |
flags |= hype.Database.DBLCKNB |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
99 |
|
99
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
100 |
elif '+' in mode : |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
101 |
# read |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
102 |
flags |= hype.Database.DBREADER |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
103 |
|
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
104 |
elif 'trunc' in mode : |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
105 |
# truncate. Dangerous! |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
106 |
flags |= hype.Database.DBTRUNC |
8719ac564b22
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents:
96
diff
changeset
|
107 |
|
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
108 |
# make instance |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
109 |
self.db = hype.Database() |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
110 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
111 |
# open |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
112 |
if not self.db.open(path, flags) : |
127
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
113 |
raise SearchIndexError("Index open failed: %s, mode=%s, flags=%#06x" % (path, mode, flags), self.db) |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
114 |
|
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
115 |
def close (self) : |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
116 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
117 |
Explicitly close the index, this is done automatically on del |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
118 |
""" |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
119 |
|
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
120 |
if not self.db.close() : |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
121 |
raise SearchIndexError("Index close failed", self.db) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
122 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
123 |
def insert (self, channel, lines) : |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
124 |
""" |
68 | 125 |
Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
126 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
127 |
|
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
128 |
# count from zero |
68 | 129 |
count = 0 |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
130 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
131 |
# iterate |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
132 |
for line in lines : |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
133 |
# insert |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
134 |
self.insert_line(channel, line) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
135 |
|
68 | 136 |
# count |
137 |
count += 1 |
|
138 |
||
139 |
# return |
|
140 |
return count |
|
141 |
||
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
142 |
def insert_line (self, channel, line) : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
143 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
144 |
Adds a single LogLine for the given LogChannel to the index |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
145 |
""" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
146 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
147 |
# validate the LogChannel |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
148 |
assert channel.id |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
149 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
150 |
# validate the LogLine |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
151 |
assert line.offset |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
152 |
assert line.timestamp |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
153 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
154 |
# create new document |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
155 |
doc = hype.Document() |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
156 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
157 |
# line date |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
158 |
date = line.timestamp.date() |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
159 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
160 |
# ensure that it's not 1900 |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
161 |
assert date.year != 1900 |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
162 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
163 |
# add URI |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
164 |
doc.add_attr('@uri', "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset)) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
165 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
166 |
# add channel id |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
167 |
doc.add_attr('channel', channel.id) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
168 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
169 |
# add type |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
170 |
doc.add_attr('type', str(line.type)) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
171 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
172 |
# add UTC timestamp |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
173 |
doc.add_attr('timestamp', str(utils.to_utc_timestamp(line.timestamp))) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
174 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
175 |
# add source attribute? |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
176 |
if line.source : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
177 |
source_nickname, source_username, source_hostname, source_chanflags = line.source |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
178 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
179 |
if source_nickname : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
180 |
doc.add_attr('source_nickname', source_nickname.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
181 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
182 |
if source_username : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
183 |
doc.add_attr('source_username', source_username.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
184 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
185 |
if source_hostname : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
186 |
doc.add_attr('source_hostname', source_hostname.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
187 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
188 |
if source_chanflags : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
189 |
doc.add_attr('source_chanflags', source_chanflags.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
190 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
191 |
# add target attributes? |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
192 |
if line.target : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
193 |
target_nickname = line.target |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
194 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
195 |
if target_nickname : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
196 |
doc.add_attr('target_nickname', target_nickname.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
197 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
198 |
# add data |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
199 |
if line.data : |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
200 |
doc.add_text(line.data.encode('utf8')) |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
201 |
|
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
202 |
# put, "clean up dispensable regions of the overwritten document" |
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
203 |
if not self.db.put_doc(doc, hype.Database.PDCLEAN) : |
127
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
204 |
raise SearchIndexError("put_doc", self.db) |
93
48fca00689e3
implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents:
89
diff
changeset
|
205 |
|
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
206 |
def search_cond (self, cond) : |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
207 |
""" |
74
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
208 |
Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
209 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
210 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
211 |
# execute search, unused 'flags' arg stays zero |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
212 |
results = self.db.search(cond, 0) |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
213 |
|
74
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
214 |
# no results? |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
215 |
if not results : |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
216 |
raise NoResultsFound() |
1ab95857d584
handle the 'no search results' case
Tero Marttila <terom@fixme.fi>
parents:
68
diff
changeset
|
217 |
|
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
218 |
# iterate over the document IDs |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
219 |
for doc_id in results : |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
220 |
# load document, this throws an exception... |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
221 |
# option constants are hype.Database.GDNOATTR/GDNOTEXT |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
222 |
doc = self.db.get_doc(doc_id, 0) |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
223 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
224 |
# load the attributes/text |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
225 |
channel = self.channels.lookup(doc.attr('channel')) |
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
226 |
type = int(doc.attr('type')) |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
227 |
timestamp = utils.from_utc_timestamp(int(doc.attr('timestamp'))) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
228 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
229 |
# source |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
230 |
source = (doc.attr('source_nickname'), doc.attr('source_username'), doc.attr('source_hostname'), doc.attr('source_chanflags')) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
231 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
232 |
# target |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
233 |
target = doc.attr('target_nickname') |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
234 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
235 |
# message text |
87
39915772f090
update LogSearchIndex to use new LogLine fields
Tero Marttila <terom@fixme.fi>
parents:
74
diff
changeset
|
236 |
message = doc.cat_texts().decode('utf8') |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
237 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
238 |
# build+yield to as LogLine |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
239 |
yield log_line.LogLine(channel, None, type, timestamp, source, target, message) |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
240 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
241 |
def search (self, options=None, channel=None, attrs=None, phrase=None, order=None, max=None, skip=None) : |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
242 |
""" |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
243 |
Search with flexible parameters |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
244 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
245 |
options - bitmask of hype.Condition.* |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
246 |
channel - LogChannel object |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
247 |
attrs - raw attribute expressions |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
248 |
phrase - the search query phrase |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
249 |
order - order attribute expression |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
250 |
max - number of results to return |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
251 |
skip - number of results to skip |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
252 |
""" |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
253 |
|
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
254 |
# build condition |
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
255 |
cond = hype.Condition() |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
256 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
257 |
if options : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
258 |
# set options |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
259 |
cond.set_options(options) |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
260 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
261 |
if channel : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
262 |
# add channel attribute |
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
263 |
cond.add_attr(("channel STREQ %s" % channel.id).encode('utf8')) |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
264 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
265 |
if attrs : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
266 |
# add attributes |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
267 |
for attr in attrs : |
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
268 |
cond.add_attr(attr.encode('utf8')) |
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
269 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
270 |
if phrase : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
271 |
# add phrase |
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
272 |
cond.set_phrase(phrase.encode('utf8')) |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
273 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
274 |
if order : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
275 |
# set order |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
276 |
cond.set_order(order) |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
277 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
278 |
if max : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
279 |
# set max |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
280 |
cond.set_max(max) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
281 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
282 |
if skip : |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
283 |
# set skip |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
284 |
cond.set_skip(skip) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
285 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
286 |
# execute |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
287 |
return self.search_cond(cond) |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
288 |
|
141
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
289 |
def search_simple (self, channel, query, count=None, offset=None) : |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
290 |
""" |
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
291 |
Search for lines from the given channel for the given simple query. |
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
292 |
|
141
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
293 |
The given text is searched for in the text of the given channel's entries, and the list of results in |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
294 |
reverse time order is returned. |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
295 |
""" |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
296 |
|
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
297 |
# search attributes |
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
298 |
attrs = [] |
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
299 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
300 |
# use search(), backwards |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
301 |
results = list(self.search( |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
302 |
# simplified phrase |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
303 |
options = hype.Condition.SIMPLE, |
64
cdb6403c2498
beginnings of a LogSearchIndex class
Tero Marttila <terom@fixme.fi>
parents:
diff
changeset
|
304 |
|
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
305 |
# specific channel |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
306 |
channel = channel, |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
307 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
308 |
# given phrase |
141
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
309 |
phrase = query, |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
310 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
311 |
# attributes defined above |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
312 |
attrs = attrs, |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
313 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
314 |
# order by timestamp, descending (backwards) |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
315 |
order = "timestamp NUMD", |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
316 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
317 |
# count/offset |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
318 |
max = count, |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
319 |
skip = offset, |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
320 |
)) |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
321 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
322 |
# reverse |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
323 |
return reversed(results) |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
324 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
325 |
def search_advanced (self, channel, phrase=None, nick_query=None, count=None, offset=None) : |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
326 |
""" |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
327 |
Search for lines from the given channel for the given full-featured query. |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
328 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
329 |
The given phrase is used to build the condition, or alternatively, the given extra *_query parameters can |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
330 |
be used to specific additional attributes to search. |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
331 |
""" |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
332 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
333 |
attrs = [] |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
334 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
335 |
if nick_query : |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
336 |
# search for messages from specific nickname |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
337 |
attrs.append("source_nickname STRINC %s" % nick_query) |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
338 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
339 |
# use search(), backwards |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
340 |
results = list(self.search( |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
341 |
# specific channel |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
342 |
channel = channel, |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
343 |
|
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
344 |
# given phrase |
65c98c9e1716
improved search - separate q=/nick= fields
Tero Marttila <terom@fixme.fi>
parents:
140
diff
changeset
|
345 |
phrase = phrase, |
118
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
346 |
|
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
347 |
# attributes defined above |
f530c158aa07
implement some basic search-targets for message and nickname
Tero Marttila <terom@fixme.fi>
parents:
99
diff
changeset
|
348 |
attrs = attrs, |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
349 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
350 |
# order by timestamp, descending (backwards) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
351 |
order = "timestamp NUMD", |
66
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
352 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
353 |
# count/offset |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
354 |
max = count, |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
355 |
skip = offset, |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
356 |
)) |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
357 |
|
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
358 |
# reverse |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
359 |
return reversed(results) |
090ed78ec8fa
add count/skip to search results, requires modifications to the swig bindings for HyperEstraier...
Tero Marttila <terom@fixme.fi>
parents:
65
diff
changeset
|
360 |
|
89
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
361 |
def list (self, channel, date, count=None, skip=None) : |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
362 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
363 |
List all indexed log items for the given UTC date |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
364 |
""" |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
365 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
366 |
# start/end dates |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
367 |
dt_start = datetime.datetime(date.year, date.month, date.day, 0, 0, 0, 0) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
368 |
dt_end = datetime.datetime(date.year, date.month, date.day, 23, 23, 59, 999999) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
369 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
370 |
# search |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
371 |
return self.search( |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
372 |
# specific channel |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
373 |
channel = channel, |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
374 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
375 |
# specific date range |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
376 |
attrs = [ |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
377 |
"timestamp NUMBT %d %d" % (utils.to_utc_timestamp(dt_start), utils.to_utc_timestamp(dt_end)) |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
378 |
], |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
379 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
380 |
# order correctly |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
381 |
order = "timestamp NUMA", |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
382 |
|
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
383 |
# max count/offset |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
384 |
max = count, |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
385 |
skip = skip |
2dc6de43f317
add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents:
87
diff
changeset
|
386 |
) |
96
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
387 |
|
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
388 |
def get_index () : |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
389 |
""" |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
390 |
Returns the default read-only index, suitable for searching |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
391 |
""" |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
392 |
|
127
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
393 |
# XXX: no caching, just open it every time |
5746705a2719
improve LogSearchIndex error handling, add explicit close() method, and modify get_index to not keep the index open persistently
Tero Marttila <terom@fixme.fi>
parents:
121
diff
changeset
|
394 |
_index = LogSearchIndex(config.LOG_CHANNELS, config.SEARCH_INDEX_PATH, 'r') |
96
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
395 |
|
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
396 |
# return |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
397 |
return _index |
d30c88e89a7e
move the LogSearchIndex open from handlers to log_search, and make it lazy
Tero Marttila <terom@fixme.fi>
parents:
93
diff
changeset
|
398 |