39 |
39 |
40 raise Exception("ID overflow: %s" % id) |
40 raise Exception("ID overflow: %s" % id) |
41 |
41 |
42 def key2int (key) : |
42 def key2int (key) : |
43 # base64 ignores extra padding, but if it doesn't, it's (4 - len%4), if len%4 != 0 |
43 # base64 ignores extra padding, but if it doesn't, it's (4 - len%4), if len%4 != 0 |
44 bytes = base64.b64decode(key + '='*6, '-_') |
44 # and it breaks on unicode strings |
|
45 bytes = base64.b64decode(str(key + '='*6), '-_') |
45 |
46 |
46 type = { |
47 type = { |
47 1: 'B', |
48 1: 'B', |
48 2: 'H', |
49 2: 'H', |
49 4: 'I', |
50 4: 'I', |
50 }[len(bytes)] |
51 }[len(bytes)] |
51 |
52 |
52 return struct.unpack(type, bytes)[0] |
53 return struct.unpack(type, bytes)[0] |
53 |
|
54 def updateDB (root) : |
|
55 """ |
|
56 DeGAL <= 0.2 used a simple key => path mapping, but now we use |
|
57 something more structured, key => (type, dirpath, fname), where |
|
58 |
|
59 type - one of 'img', 'dir' |
|
60 dirpath - the path to the directory, e.g. '.', './foobar', './foobar/quux' |
|
61 fname - the filename, one of '', 'DSC9839.JPG', 'this.png', etc. |
|
62 """ |
|
63 |
|
64 db = shelve.open('shorturls2', 'c', writeback=True) |
|
65 |
|
66 id = db.get('_id', 1) |
|
67 |
|
68 dirqueue = [root] |
|
69 |
|
70 # dict of path -> obj |
|
71 paths = {} |
|
72 |
|
73 index.info("Processing ShortURLs...") |
|
74 |
|
75 while dirqueue : |
|
76 dir = dirqueue.pop(0) |
|
77 |
|
78 dirqueue.extend(dir.subdirs.itervalues()) |
|
79 |
|
80 if dir.alive : |
|
81 paths[dir.path] = dir |
|
82 |
|
83 for img in dir.images.itervalues() : |
|
84 paths[img.path] = img |
|
85 |
|
86 for key in db.keys() : |
|
87 if key.startswith('_') : |
|
88 continue |
|
89 |
|
90 type, dirpath, fname = db[key] |
|
91 |
|
92 path = os.path.join(dirpath, fname).rstrip(os.sep) |
|
93 |
|
94 try : |
|
95 paths.pop(path).shorturl_code = key |
|
96 index.debug("Code for `%s' is %s", path, key) |
|
97 |
|
98 except KeyError : |
|
99 index.debug("Path `%s' in DB does not exist?", path) |
|
100 |
|
101 for obj in paths.itervalues() : |
|
102 key = int2key(id) |
|
103 id += 1 |
|
104 |
|
105 index.info("Alloc code `%s' for `%s'", key, obj.html_path) |
|
106 |
|
107 obj.shorturl_code = key |
|
108 |
|
109 db[key] = obj.getObjInfo() |
|
110 |
|
111 db['_id'] = id |
|
112 db.close() |
|
113 |
54 |
114 class DB (object) : |
55 class DB (object) : |
115 def __init__ (self, read_only=True) : |
56 def __init__ (self, read_only=True) : |
116 self.db = shelve.open('shorturls2', read_only and 'r' or 'c') |
57 self.db = shelve.open('shorturls2', read_only and 'r' or 'c') |
117 |
58 |
156 raise ValueError("Paths not found: %s" % " ".join(paths)) |
97 raise ValueError("Paths not found: %s" % " ".join(paths)) |
157 |
98 |
158 return ret |
99 return ret |
159 |
100 |
160 def html_path (key, index=None) : |
101 def html_path (key, index=None) : |
161 dir, fname = image_info(key) |
102 dir, fname = node_info(key) |
162 |
103 |
163 return utils.url(dir, fname + '.html') |
104 if fname : |
|
105 return utils.url(dir, fname + '.html') |
|
106 else : |
|
107 return utils.url(dir, helpers.url_for_page(index or 0)) |
|
108 |
|
109 def node_info (key) : |
|
110 res = db.select("""SELECT dirpath, filename FROM nodes WHERE id=?""", key2int(key)).fetchone() |
|
111 |
|
112 if res : |
|
113 return res |
|
114 |
|
115 else : |
|
116 raise KeyError(key) |
164 |
117 |
165 def image_info (key) : |
118 def image_info (key) : |
166 res = db.select("""SELECT dirpath, filename FROM images WHERE id=?""", key2int(key)).fetchone() |
119 res = db.select("""SELECT dirpath, filename FROM images WHERE id=?""", key2int(key)).fetchone() |
167 |
120 |
168 if res : |
121 if res : |
179 return res |
132 return res |
180 |
133 |
181 else : |
134 else : |
182 raise KeyError(keys) |
135 raise KeyError(keys) |
183 |
136 |
|
137 def _got_obj_key (obj, id) : |
|
138 key = int2key(id) |
|
139 |
|
140 obj.shorturl_code = key |
|
141 |
|
142 if isinstance(obj, folder.Folder) : |
|
143 dir, fname = utils.strip_path(obj.path), '' |
|
144 elif isinstance(obj, image.Image) : |
|
145 dir, fname = utils.strip_path(obj.dir.path), obj.name |
|
146 else : |
|
147 assert(False, "%r %r" % (obj, id)) |
|
148 |
|
149 index.info("img %50s %15s = %d %s", dir, fname, id, key) |
|
150 |
|
151 def updateDB (root) : |
|
152 """ |
|
153 Update the SQL database |
|
154 |
|
155 type - one of 'img', 'dir' |
|
156 dirpath - the path to the directory, e.g. '.', './foobar', './foobar/quux' |
|
157 fname - the filename, one of '', 'DSC9839.JPG', 'this.png', etc. |
|
158 """ |
|
159 |
|
160 dirqueue = [root] |
|
161 |
|
162 # dict of (dir, fname) -> obj |
|
163 paths = {} |
|
164 |
|
165 index.info("Processing ShortURLs...") |
|
166 |
|
167 while dirqueue : |
|
168 dir = dirqueue.pop(0) |
|
169 |
|
170 dirqueue.extend(dir.subdirs.itervalues()) |
|
171 |
|
172 if dir.alive : |
|
173 pathtuple = (utils.strip_path(dir.path), '') |
|
174 |
|
175 index.debug("dir %50s", pathtuple[0]) |
|
176 |
|
177 paths[pathtuple] = dir |
|
178 |
|
179 for img in dir.images.itervalues() : |
|
180 pathtuple = (utils.strip_path(img.dir.path), img.name) |
|
181 |
|
182 index.debug("img %50s %15s", *pathtuple) |
|
183 |
|
184 paths[pathtuple] = img |
|
185 |
|
186 print "%d nodes:" % (len(paths)) |
|
187 |
|
188 for (id, dir, fname) in db.select("SELECT id, dirpath, filename FROM nodes") : |
|
189 try : |
|
190 obj = paths.pop((dir, fname)) |
|
191 key = int2key(id) |
|
192 |
|
193 obj.shorturl_code = key |
|
194 |
|
195 index.debug("%s %50s %15s -> %d %s", dir and "img" or "dir", dir, fname, id, key) |
|
196 |
|
197 except KeyError : |
|
198 pass |
|
199 # index.warning("non-existant node (%d, %s, %s) in db", id, dir, fname) |
|
200 |
|
201 print "%d NEW nodes:" % (len(paths)) |
|
202 |
|
203 db.insert_many( |
|
204 _got_obj_key, |
|
205 "INSERT INTO nodes (dirpath, filename) VALUES (?, ?)", |
|
206 ((obj, (path, fname)) for ((path, fname), obj) in paths.iteritems()) |
|
207 ) |
|
208 |