1 #!/usr/bin/python |
1 #!/usr/bin/python |
2 |
2 |
3 import optparse, shlex |
3 from pvl.backup.rsync import RSyncCommandFormatError |
|
4 from pvl.backup.invoke import InvokeError |
|
5 from pvl.backup import rsync |
4 |
6 |
5 import subprocess |
7 import optparse |
6 import os, os.path |
8 import os |
7 |
|
8 import contextlib |
|
9 import logging |
9 import logging |
10 |
10 |
11 log = logging.getLogger() |
11 log = logging.getLogger() |
12 |
|
13 class InvokeError (Exception) : |
|
14 def __init__ (self, cmd, exit) : |
|
15 self.cmd = cmd |
|
16 self.exit = exit |
|
17 |
|
18 def __str__ (self) : |
|
19 return "{cmd} failed: {exit}".format(cmd=self.cmd, exit=self.exit) |
|
20 |
|
21 def invoke (cmd, args, data=None) : |
|
22 """ |
|
23 Invoke a command directly. |
|
24 |
|
25 data: data to pass in on stdin, returning stdout. |
|
26 if given as False, passes through our process stdin/out |
|
27 |
|
28 Doesn't give any data on stdin, and keeps process stderr. |
|
29 Returns stdout. |
|
30 """ |
|
31 |
|
32 log.debug("cmd={cmd}, args={args}".format(cmd=cmd, args=args)) |
|
33 |
|
34 if data is False : |
|
35 # keep process stdin/out |
|
36 io = None |
|
37 else : |
|
38 io = subprocess.PIPE |
|
39 |
|
40 p = subprocess.Popen([cmd] + args, stdin=io, stdout=io) |
|
41 |
|
42 # get output |
|
43 stdout, stderr = p.communicate(input=data) |
|
44 |
|
45 if p.returncode : |
|
46 # failed |
|
47 raise InvokeError(cmd, p.returncode) |
|
48 |
|
49 return stdout |
|
50 |
|
51 def optargs (*args, **kwargs) : |
|
52 """ |
|
53 Convert args/options into command-line format |
|
54 """ |
|
55 |
|
56 # process |
|
57 opts = [('--{opt}'.format(opt=opt), value if value != True else None) for opt, value in kwargs.iteritems() if value] |
|
58 |
|
59 # flatten |
|
60 opts = [str(opt_part) for opt_parts in opts for opt_part in opt_parts if opt_part] |
|
61 |
|
62 args = [str(arg) for arg in args if arg] |
|
63 |
|
64 return opts + args |
|
65 |
|
66 def command (cmd, *args, **opts) : |
|
67 """ |
|
68 Invoke a command with options/arguments, given via Python arguments/keyword arguments. |
|
69 |
|
70 Return stdout. |
|
71 """ |
|
72 |
|
73 log.debug("{cmd} {opts} {args}".format(cmd=cmd, args=args, opts=opts)) |
|
74 |
|
75 # invoke |
|
76 return invoke(cmd, optargs(*args, **opts)) |
|
77 |
|
78 class LVM (object) : |
|
79 """ |
|
80 LVM VolumeGroup |
|
81 """ |
|
82 |
|
83 # path to lvm2 binary |
|
84 LVM = '/sbin/lvm' |
|
85 |
|
86 |
|
87 # VG name |
|
88 name = None |
|
89 |
|
90 def __init__ (self, name) : |
|
91 self.name = name |
|
92 |
|
93 def lv_name (self, lv) : |
|
94 """ |
|
95 vg/lv name. |
|
96 """ |
|
97 |
|
98 return '{vg}/{lv}'.format(vg=self.name, lv=lv) |
|
99 |
|
100 def lv_path (self, lv) : |
|
101 """ |
|
102 /dev/vg/lv path. |
|
103 """ |
|
104 |
|
105 return '/dev/{vg}/{lv}'.format(vg=self.name, lv=lv) |
|
106 |
|
107 def command (self, cmd, *args, **opts) : |
|
108 """ |
|
109 Invoke a command with options/arguments, given via Python arguments/keyword arguments |
|
110 """ |
|
111 |
|
112 log.debug("{cmd} {opts} {args}".format(cmd=cmd, args=args, opts=opts)) |
|
113 |
|
114 # invoke |
|
115 invoke(self.LVM, [cmd] + optargs(*args, **opts)) |
|
116 |
|
117 def volume (self, name) : |
|
118 """ |
|
119 Return an LVMVolume for given named LV. |
|
120 """ |
|
121 |
|
122 return LVMVolume(self, name) |
|
123 |
|
124 @contextlib.contextmanager |
|
125 def snapshot (self, base, **kwargs) : |
|
126 """ |
|
127 A Context Manager for handling an LVMSnapshot. |
|
128 |
|
129 See LVMSnapshot.create() |
|
130 |
|
131 with lvm.snapshot(lv) as snapshot : ... |
|
132 """ |
|
133 |
|
134 log.debug("creating snapshot from {base}: {opts}".format(base=base, opts=kwargs)) |
|
135 snapshot = LVMSnapshot.create(self, base, **kwargs) |
|
136 |
|
137 try : |
|
138 log.debug("got: {0}".format(snapshot)) |
|
139 yield snapshot |
|
140 |
|
141 finally: |
|
142 # cleanup |
|
143 # XXX: do we need to wait for it to get closed after mount? |
|
144 log.debug("cleanup: {0}".format(snapshot)) |
|
145 snapshot.close() |
|
146 |
|
147 def __repr__ (self) : |
|
148 return "LVM(name={name})".format(name=repr(self.name)) |
|
149 |
|
150 class LVMVolume (object) : |
|
151 """ |
|
152 LVM Logical Volume. |
|
153 """ |
|
154 |
|
155 # VG |
|
156 lvm = None |
|
157 |
|
158 # name |
|
159 name = None |
|
160 |
|
161 def __init__ (self, lvm, name) : |
|
162 self.lvm = lvm |
|
163 self.name = name |
|
164 |
|
165 @property |
|
166 def lvm_path (self) : |
|
167 return self.lvm.lv_name(self.name) |
|
168 |
|
169 @property |
|
170 def dev_path (self) : |
|
171 return self.lvm.lv_path(self.name) |
|
172 |
|
173 def __repr__ (self) : |
|
174 return "LVMVolume(lvm={lvm}, name={name})".format( |
|
175 lvm = repr(self.lvm), |
|
176 name = repr(self.name), |
|
177 ) |
|
178 |
|
179 class LVMSnapshot (LVMVolume) : |
|
180 """ |
|
181 LVM snapshot |
|
182 """ |
|
183 |
|
184 # default snapshot size |
|
185 LVM_SNAPSHOT_SIZE = '5G' |
|
186 |
|
187 # base lv |
|
188 base = None |
|
189 |
|
190 @classmethod |
|
191 def create (cls, lvm, base, tag, size=LVM_SNAPSHOT_SIZE) : |
|
192 """ |
|
193 Create a new LVM snapshot of the given LV. |
|
194 |
|
195 Returns a (snapshot_name, dev_path) tuple. |
|
196 """ |
|
197 |
|
198 # snapshot name |
|
199 name = '{name}-{tag}'.format(name=base.name, tag=tag) |
|
200 |
|
201 # snapshot |
|
202 snapshot = cls(lvm, base, name) |
|
203 |
|
204 # verify LV exists |
|
205 lvm.command('lvs', base.lvm_path) |
|
206 |
|
207 if not os.path.exists(base.dev_path) : |
|
208 raise Exception("lvm_snapshot: source LV does not exist: {path}".format(path=base.dev_path)) |
|
209 |
|
210 if os.path.exists(snapshot.dev_path) : |
|
211 raise Exception("lvm_snapshot: target LV snapshot already exists: {path}".format(path=snapshot.dev_path)) |
|
212 |
|
213 # create |
|
214 snapshot.open() |
|
215 |
|
216 # verify |
|
217 if not os.path.exists(snapshot.dev_path) : |
|
218 raise Exception("lvm_snapshot: target LV snapshot did not appear: {path}".format(path=snapshot.dev_path)) |
|
219 |
|
220 # yay |
|
221 return snapshot |
|
222 |
|
223 def __init__ (self, lvm, base, name, size=LVM_SNAPSHOT_SIZE) : |
|
224 LVMVolume.__init__(self, lvm, name) |
|
225 |
|
226 self.base = base |
|
227 self.size = size |
|
228 |
|
229 def open (self) : |
|
230 """ |
|
231 Create snapshot volume. |
|
232 """ |
|
233 |
|
234 # create |
|
235 self.lvm.command('lvcreate', self.base.lvm_path, snapshot=True, name=self.name, size=self.size) |
|
236 |
|
237 def close (self) : |
|
238 """ |
|
239 Remove snapshot volume. |
|
240 """ |
|
241 |
|
242 # XXX: can't deactivate snapshot volume |
|
243 #self.lvm.command('lvchange', name, available='n') |
|
244 |
|
245 # XXX: risky! |
|
246 self.lvm.command('lvremove', '-f', self.lvm_path) |
|
247 |
|
248 def __repr__ (self) : |
|
249 return "LVMSnapshot(lvm={lvm}, base={base}, name={name})".format( |
|
250 lvm = str(self.lvm), |
|
251 base = str(self.base), |
|
252 name = repr(self.name), |
|
253 ) |
|
254 |
|
255 |
|
256 class MountError (Exception) : |
|
257 pass |
|
258 |
|
259 class Mount (object) : |
|
260 """ |
|
261 Trivial filesystem mounting |
|
262 """ |
|
263 |
|
264 MOUNT = '/bin/mount' |
|
265 UMOUNT = '/bin/umount' |
|
266 |
|
267 |
|
268 def __init__ (self, dev, mnt, readonly=False) : |
|
269 """ |
|
270 dev - device path |
|
271 mnt - mount path |
|
272 readonly - mount readonly |
|
273 """ |
|
274 |
|
275 self.dev = dev |
|
276 self.mnt = mnt |
|
277 self.readonly = readonly |
|
278 |
|
279 @property |
|
280 def path (self) : |
|
281 return self.mnt |
|
282 |
|
283 def options (self) : |
|
284 """ |
|
285 Mount options as a comma-separated string. |
|
286 """ |
|
287 |
|
288 options = [ |
|
289 ('ro' if self.readonly else None), |
|
290 ] |
|
291 |
|
292 return ','.join(option for option in options if option) |
|
293 |
|
294 def open (self) : |
|
295 """ |
|
296 Mount |
|
297 """ |
|
298 |
|
299 # check |
|
300 if not os.path.isdir(self.mnt) : |
|
301 raise MountError("Mountpoint is not a directory: {mnt}".format(mnt=self.mnt)) |
|
302 |
|
303 if os.path.ismount(self.mnt) : |
|
304 raise MountError("Mountpoint is already mounted: {mnt}".format(mnt=self.mnt)) |
|
305 |
|
306 if not os.path.exists(self.dev) : |
|
307 raise MountError("Device does not exist: {dev}".format(dev=self.dev)) |
|
308 |
|
309 # mount |
|
310 command(self.MOUNT, self.dev, self.mnt, options=self.options()) |
|
311 |
|
312 def close (self) : |
|
313 """ |
|
314 Un-mount |
|
315 """ |
|
316 |
|
317 # check |
|
318 if not os.path.ismount(self.mnt): |
|
319 raise MountError("Mountpoint is not mounted: {mnt}".format(mnt=self.mnt)) |
|
320 |
|
321 # umount |
|
322 command(self.UMOUNT, self.mnt) |
|
323 |
|
324 @contextlib.contextmanager |
|
325 def mount (dev, mnt, **kwargs) : |
|
326 """ |
|
327 Use a temporary mount: |
|
328 |
|
329 with mount('/dev/...', '/mnt', readonly=True) as mount: |
|
330 ... |
|
331 """ |
|
332 |
|
333 mount = Mount(dev, mnt, **kwargs) |
|
334 |
|
335 # open |
|
336 log.debug("open: %s", mount) |
|
337 mount.open() |
|
338 |
|
339 try : |
|
340 log.debug("got: %s", mount) |
|
341 yield mount |
|
342 |
|
343 finally: |
|
344 # cleanup |
|
345 log.debug("cleanup: %s", mount) |
|
346 mount.close() |
|
347 |
|
348 class RSyncCommandFormatError (Exception) : |
|
349 """ |
|
350 Improper rsync command |
|
351 """ |
|
352 |
|
353 pass |
|
354 |
|
355 def parse_rsync (command, restrict_server=True, restrict_readonly=True) : |
|
356 """ |
|
357 Parse given rsync server command into bits. |
|
358 |
|
359 command - the command-string sent by rsync |
|
360 restrict_server - restrict to server-mode |
|
361 restrict_readonly - restrict to read/send-mode |
|
362 |
|
363 Returns: |
|
364 |
|
365 (cmd, options, source, dest) |
|
366 """ |
|
367 |
|
368 # split |
|
369 parts = shlex.split(command) |
|
370 |
|
371 cmd = None |
|
372 options = [] |
|
373 source = None |
|
374 dest = None |
|
375 |
|
376 # parse |
|
377 for part in parts : |
|
378 if cmd is None : |
|
379 cmd = part |
|
380 |
|
381 elif part.startswith('-') : |
|
382 options.append(part) |
|
383 |
|
384 elif source is None : |
|
385 source = part |
|
386 |
|
387 elif dest is None : |
|
388 dest = part |
|
389 |
|
390 # options |
|
391 have_server = ('--server' in options) |
|
392 have_sender = ('--sender' in options) |
|
393 |
|
394 # verify |
|
395 if not have_server : |
|
396 raise RSyncCommandFormatError("Missing --server") |
|
397 |
|
398 if restrict_readonly and not have_sender : |
|
399 raise RSyncCommandFormatError("Missing --sender for readonly") |
|
400 |
|
401 # parse path |
|
402 if have_sender : |
|
403 # read |
|
404 # XXX: which way does the dot go? |
|
405 if source != '.' : |
|
406 raise RSyncCommandFormatError("Invalid dest for sender") |
|
407 |
|
408 path = dest |
|
409 |
|
410 else : |
|
411 # write |
|
412 if source != '.' : |
|
413 raise RSyncCommandFormatError("Invalid source for reciever") |
|
414 |
|
415 path = dest |
|
416 |
|
417 # ok |
|
418 return cmd, options, source, dest |
|
419 |
|
420 class RSyncSource (object) : |
|
421 RSYNC = '/usr/bin/rsync' |
|
422 |
|
423 def _execute (self, options, path) : |
|
424 """ |
|
425 Underlying rsync just reads from filesystem. |
|
426 """ |
|
427 |
|
428 invoke(self.RSYNC, options + [path, '.'], data=False) |
|
429 |
|
430 class RSyncFSSource (RSyncSource) : |
|
431 """ |
|
432 Normal filesystem backup. |
|
433 """ |
|
434 |
|
435 def __init__ (self, path) : |
|
436 RSyncSource.__init__(self) |
|
437 |
|
438 self.path = path |
|
439 |
|
440 def execute (self, options) : |
|
441 return self._execute(options, self.path) |
|
442 |
|
443 class RSyncLVMSource (RSyncSource) : |
|
444 """ |
|
445 Backup LVM LV by snapshotting + mounting it. |
|
446 """ |
|
447 |
|
448 def __init__ (self, volume) : |
|
449 RSyncSource.__init__(self) |
|
450 |
|
451 self.volume = volume |
|
452 |
|
453 def execute (self, options) : |
|
454 """ |
|
455 Snapshot, mount, execute |
|
456 """ |
|
457 |
|
458 # backup target from LVM command |
|
459 lvm = self.volume.lvm |
|
460 volume = self.volume |
|
461 |
|
462 # XXX: generate |
|
463 path = '/mnt' |
|
464 |
|
465 # snapshot |
|
466 log.info("Open snapshot...") |
|
467 |
|
468 # XXX: generate snapshot nametag to be unique? |
|
469 with lvm.snapshot(volume, tag='backup') as snapshot: |
|
470 log.info("Snapshot opened: %s", snapshot.lvm_path) |
|
471 |
|
472 # mount |
|
473 log.info("Mounting snapshot: %s -> %s", snapshot, path) |
|
474 |
|
475 with mount(snapshot.dev_path, path) as mountpoint: |
|
476 log.info("Mounted snapshot: %s", mountpoint) |
|
477 |
|
478 # rsync! |
|
479 log.info("Running rsync: ...") |
|
480 |
|
481 return self._execute(options, mountpoint.path) |
|
482 |
|
483 # cleanup |
|
484 # cleanup |
|
485 |
|
486 def rsync_source (path, restrict_path=False) : |
|
487 """ |
|
488 Figure out source to rsync from, based on pseudo-path given in rsync command. |
|
489 """ |
|
490 |
|
491 # normalize |
|
492 path = os.path.normpath(path) |
|
493 |
|
494 # verify path |
|
495 if restrict_path : |
|
496 if not path.startswith(restrict_path) : |
|
497 raise RSyncCommandFormatError("Restricted path ({restrict})".format(restrict=restrict_path)) |
|
498 |
|
499 if path.startswith('/') : |
|
500 # direct filesystem path |
|
501 # XXX: how to handle= |
|
502 log.info("filesystem: %s", path) |
|
503 |
|
504 return RSyncFSSource(path) |
|
505 |
|
506 elif path.startswith('lvm:') : |
|
507 # LVM LV |
|
508 try : |
|
509 lvm, vg, lv = path.split(':') |
|
510 |
|
511 except ValueError, e: |
|
512 raise RSyncCommandFormatError("Invalid lvm pseudo-path: {error}".format(error=e)) |
|
513 |
|
514 # XXX: validate |
|
515 |
|
516 log.info("LVM: %s/%s", vg, lv) |
|
517 |
|
518 # open |
|
519 lvm = LVM(vg) |
|
520 volume = lvm.volume(lv) |
|
521 |
|
522 return RSyncLVMSource(volume) |
|
523 |
|
524 else : |
|
525 # invalid |
|
526 raise RSyncCommandFormatError("Unrecognized backup path") |
|
527 |
12 |
528 # command-line options |
13 # command-line options |
529 options = None |
14 options = None |
530 |
15 |
531 def parse_options (argv) : |
16 def parse_options (argv) : |