44 """ |
44 """ |
45 |
45 |
46 def __init__ (self, channels, path, mode='r') : |
46 def __init__ (self, channels, path, mode='r') : |
47 """ |
47 """ |
48 Open the database at the given path, with the given mode: |
48 Open the database at the given path, with the given mode: |
49 r - read-only |
49 r - read, error if not exists |
50 w - write, create if not exists |
50 w - write, create if not exists |
51 a - write, error if not exists |
51 a - write, error if not exists |
52 c - write, create, error if exists |
52 c - write, create, error if exists |
53 * - write, create, truncate if exists |
53 * - write, create, truncate if exists |
54 |
54 |
86 def insert (self, channel, lines) : |
86 def insert (self, channel, lines) : |
87 """ |
87 """ |
88 Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items |
88 Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items |
89 """ |
89 """ |
90 |
90 |
91 # validate the LogChannel |
91 # count from zero |
92 assert channel.name |
|
93 |
|
94 count = 0 |
92 count = 0 |
95 |
93 |
96 # iterate |
94 # iterate |
97 for line in lines : |
95 for line in lines : |
98 # validate the LogLine |
96 # insert |
99 assert line.offset |
97 self.insert_line(channel, line) |
100 assert line.timestamp |
98 |
101 |
|
102 # create new document |
|
103 doc = hype.Document() |
|
104 |
|
105 # line date |
|
106 date = line.timestamp.date() |
|
107 |
|
108 # ensure that it's not 1900 |
|
109 assert date.year != 1900 |
|
110 |
|
111 # add URI |
|
112 doc.add_attr('@uri', "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset)) |
|
113 |
|
114 # add channel id |
|
115 doc.add_attr('channel', channel.id) |
|
116 |
|
117 # add type |
|
118 doc.add_attr('type', str(line.type)) |
|
119 |
|
120 # add UTC timestamp |
|
121 doc.add_attr('timestamp', str(utils.to_utc_timestamp(line.timestamp))) |
|
122 |
|
123 # add source attribute? |
|
124 if line.source : |
|
125 source_nickname, source_username, source_hostname, source_chanflags = line.source |
|
126 |
|
127 if source_nickname : |
|
128 doc.add_attr('source_nickname', source_nickname.encode('utf8')) |
|
129 |
|
130 if source_username : |
|
131 doc.add_attr('source_username', source_username.encode('utf8')) |
|
132 |
|
133 if source_hostname : |
|
134 doc.add_attr('source_hostname', source_hostname.encode('utf8')) |
|
135 |
|
136 if source_chanflags : |
|
137 doc.add_attr('source_chanflags', source_chanflags.encode('utf8')) |
|
138 |
|
139 # add target attributes? |
|
140 if line.target : |
|
141 target_nickname = line.target |
|
142 |
|
143 if target_nickname : |
|
144 doc.add_attr('target_nickname', target_nickname.encode('utf8')) |
|
145 |
|
146 # add data |
|
147 if line.data : |
|
148 doc.add_text(line.data.encode('utf8')) |
|
149 |
|
150 # put, "clean up dispensable regions of the overwritten document" |
|
151 if not self.db.put_doc(doc, hype.Database.PDCLEAN) : |
|
152 raise Exeception("Index put_doc failed") |
|
153 |
|
154 # count |
99 # count |
155 count += 1 |
100 count += 1 |
156 |
101 |
157 # return |
102 # return |
158 return count |
103 return count |
159 |
104 |
|
105 |
|
106 |
|
107 def insert_line (self, channel, line) : |
|
108 """ |
|
109 Adds a single LogLine for the given LogChannel to the index |
|
110 """ |
|
111 |
|
112 # validate the LogChannel |
|
113 assert channel.id |
|
114 |
|
115 # validate the LogLine |
|
116 assert line.offset |
|
117 assert line.timestamp |
|
118 |
|
119 # create new document |
|
120 doc = hype.Document() |
|
121 |
|
122 # line date |
|
123 date = line.timestamp.date() |
|
124 |
|
125 # ensure that it's not 1900 |
|
126 assert date.year != 1900 |
|
127 |
|
128 # add URI |
|
129 doc.add_attr('@uri', "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset)) |
|
130 |
|
131 # add channel id |
|
132 doc.add_attr('channel', channel.id) |
|
133 |
|
134 # add type |
|
135 doc.add_attr('type', str(line.type)) |
|
136 |
|
137 # add UTC timestamp |
|
138 doc.add_attr('timestamp', str(utils.to_utc_timestamp(line.timestamp))) |
|
139 |
|
140 # add source attribute? |
|
141 if line.source : |
|
142 source_nickname, source_username, source_hostname, source_chanflags = line.source |
|
143 |
|
144 if source_nickname : |
|
145 doc.add_attr('source_nickname', source_nickname.encode('utf8')) |
|
146 |
|
147 if source_username : |
|
148 doc.add_attr('source_username', source_username.encode('utf8')) |
|
149 |
|
150 if source_hostname : |
|
151 doc.add_attr('source_hostname', source_hostname.encode('utf8')) |
|
152 |
|
153 if source_chanflags : |
|
154 doc.add_attr('source_chanflags', source_chanflags.encode('utf8')) |
|
155 |
|
156 # add target attributes? |
|
157 if line.target : |
|
158 target_nickname = line.target |
|
159 |
|
160 if target_nickname : |
|
161 doc.add_attr('target_nickname', target_nickname.encode('utf8')) |
|
162 |
|
163 # add data |
|
164 if line.data : |
|
165 doc.add_text(line.data.encode('utf8')) |
|
166 |
|
167 # put, "clean up dispensable regions of the overwritten document" |
|
168 if not self.db.put_doc(doc, hype.Database.PDCLEAN) : |
|
169 raise Exeception("Index put_doc failed") |
|
170 |
160 def search_cond (self, cond) : |
171 def search_cond (self, cond) : |
161 """ |
172 """ |
162 Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results |
173 Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results |
163 """ |
174 """ |
164 |
175 |