|
1 #!/usr/bin/python |
|
2 |
|
3 """ |
|
4 Manage rsync --link-dest based snapshots. |
|
5 |
|
6 rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current. |
|
7 |
|
8 Updates symlink <dst>/current -> <dst>/snapshots/... |
|
9 |
|
10 Then archives <dst>/current to <dst>/<period>/<date> using --link-dest. |
|
11 """ |
|
12 |
|
13 from pvl.backup import __version__ |
|
14 from pvl.backup import rsync, invoke |
|
15 |
|
16 import optparse, ConfigParser |
|
17 import os, os.path, stat |
|
18 import shutil |
|
19 import datetime |
|
20 import logging |
|
21 |
|
22 log = logging.getLogger() |
|
23 |
|
24 # command-line options, global state |
|
25 options = None |
|
26 |
|
27 def parse_options (argv, defaults) : |
|
28 """ |
|
29 Parse command-line arguments. |
|
30 """ |
|
31 |
|
32 parser = optparse.OptionParser( |
|
33 prog = argv[0], |
|
34 usage = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]', |
|
35 version = __version__, |
|
36 |
|
37 # module docstring |
|
38 # XXX: breaks multi-line descriptions.. |
|
39 description = __doc__, |
|
40 ) |
|
41 |
|
42 # logging |
|
43 general = optparse.OptionGroup(parser, "General Options") |
|
44 |
|
45 general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output") |
|
46 general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output") |
|
47 general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output") |
|
48 |
|
49 parser.add_option_group(general) |
|
50 |
|
51 # rsync |
|
52 rsync = optparse.OptionGroup(parser, "rsync Options") |
|
53 |
|
54 rsync.add_option('--exclude-from', metavar='FILE', |
|
55 help="Read exclude rules from given file") |
|
56 |
|
57 rsync.add_option('--include-from', metavar='FILE', |
|
58 help="Read include rules from given file") |
|
59 |
|
60 parser.add_option_group(rsync) |
|
61 |
|
62 # global |
|
63 parser.add_option('--clean-intervals', action='store_true', |
|
64 help="Clean out old interval links") |
|
65 |
|
66 parser.add_option('--clean-snapshots', action='store_true', |
|
67 help="Clean out unused snapshots (those not linked to)") |
|
68 |
|
69 parser.add_option('--clean', action='store_true', |
|
70 help="Clean out both intervals and snapshots") |
|
71 |
|
72 parser.add_option('-n', '--dry-run', action='store_true', |
|
73 help="Don't actually clean anything") |
|
74 |
|
75 # |
|
76 parser.add_option('-c', '--config', metavar='FILE', |
|
77 help="Load configuration file") |
|
78 |
|
79 parser.add_option('-r', '--run', metavar='NAME', |
|
80 help="Run given set of targets, per config [run/...]") |
|
81 |
|
82 # |
|
83 parser.add_option('-T', '--target', metavar='PATH', |
|
84 help="Target path") |
|
85 |
|
86 parser.add_option('-s', '--source', metavar='RSYNC-PATH', dest='target_source', default=False, |
|
87 help="Run target backup from source in rsync-syntax") |
|
88 |
|
89 parser.add_option('--interval', metavar='NAME', action='append', dest='target_intervals', |
|
90 help="Run target with given given interval(s)") |
|
91 |
|
92 |
|
93 # defaults |
|
94 parser.set_defaults( |
|
95 loglevel = logging.INFO, |
|
96 |
|
97 target_intervals = [], |
|
98 ) |
|
99 parser.set_defaults(**defaults) |
|
100 |
|
101 |
|
102 # parse |
|
103 options, args = parser.parse_args(argv[1:]) |
|
104 |
|
105 # configure |
|
106 logging.basicConfig( |
|
107 format = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s', |
|
108 level = options.loglevel, |
|
109 ) |
|
110 |
|
111 if options.clean : |
|
112 options.clean_intervals = options.clean_snapshots = options.clean |
|
113 |
|
114 if options.include_from : |
|
115 options.rsync_options['include-from'] = options.include_from |
|
116 |
|
117 if options.exclude_from : |
|
118 options.rsync_options['exclude-from'] = options.exclude_from |
|
119 |
|
120 return options, args |
|
121 |
|
122 ## Configuration |
|
123 class ConfigError (Exception) : |
|
124 pass |
|
125 |
|
126 def process_config_name (name) : |
|
127 """ |
|
128 Process config file name into python version |
|
129 """ |
|
130 |
|
131 return name.replace('-', '_') |
|
132 |
|
133 def parse_config (path, defaults) : |
|
134 """ |
|
135 Parse given config file |
|
136 """ |
|
137 |
|
138 log.debug("loading config: %s", path) |
|
139 |
|
140 config = dict(defaults) |
|
141 config_file = ConfigParser.RawConfigParser() |
|
142 config_file.read([path]) |
|
143 |
|
144 # handle each section |
|
145 for section in config_file.sections() : |
|
146 # mangle |
|
147 section_name = process_config_name(section) |
|
148 |
|
149 log.debug("section: %s", section_name) |
|
150 |
|
151 # subsections |
|
152 if ':' in section_name : |
|
153 # legacy! |
|
154 section_path = section_name.split(':') |
|
155 else : |
|
156 # new! shiny! |
|
157 section_path = section_name.split('/') |
|
158 |
|
159 # lookup section dict from config |
|
160 lookup = config |
|
161 |
|
162 # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section |
|
163 for name in section_path : |
|
164 # possibly create |
|
165 if name not in lookup : |
|
166 lookup[name] = {} |
|
167 |
|
168 lookup = lookup[name] |
|
169 |
|
170 # found dict for this section |
|
171 config_section = lookup |
|
172 |
|
173 # values |
|
174 for name, value in config_file.items(section) : |
|
175 # mangle |
|
176 name = process_config_name(name) |
|
177 |
|
178 log.debug("section: %s: %s = %s", '/'.join(section_path), name, value) |
|
179 |
|
180 config_section[name] = value |
|
181 |
|
182 log.debug("config: %s", config) |
|
183 |
|
184 return config |
|
185 |
|
186 def config_bool (name, value, strict=True) : |
|
187 if value.lower() in ('yes', 'true', '1', 'on') : |
|
188 return True |
|
189 |
|
190 elif value.lower() in ('no', 'false', '0', 'off') : |
|
191 return False |
|
192 |
|
193 elif strict : |
|
194 raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value)) |
|
195 |
|
196 else : |
|
197 # allow non-boolean values |
|
198 return value |
|
199 |
|
200 def config_int (name, value, default=False) : |
|
201 if not value and default is not False: |
|
202 # returning default value if one is given |
|
203 return default |
|
204 |
|
205 try : |
|
206 return int(value) |
|
207 |
|
208 except ValueError, e: |
|
209 raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value)) |
|
210 |
|
211 def config_list (name, value) : |
|
212 return value.split() |
|
213 |
|
214 def walk_symlinks (tree, ignore=False) : |
|
215 """ |
|
216 Walk through all symlinks in given dir, yielding: |
|
217 |
|
218 (dirpath, name, target) |
|
219 |
|
220 Passes through errors from os.listdir/os.lstat. |
|
221 """ |
|
222 |
|
223 for name in os.listdir(tree) : |
|
224 if ignore and name in ignore : |
|
225 log.debug("%s: ignore: %s", tree, name) |
|
226 continue |
|
227 |
|
228 path = os.path.join(tree, name) |
|
229 |
|
230 # stat symlink itself |
|
231 st = os.lstat(path) |
|
232 |
|
233 if stat.S_ISDIR(st.st_mode) : |
|
234 # recurse |
|
235 log.debug("%s: tree: %s", tree, name) |
|
236 |
|
237 for item in walk_symlinks(path) : |
|
238 yield item |
|
239 |
|
240 elif stat.S_ISLNK(st.st_mode) : |
|
241 # found |
|
242 target = os.readlink(path) |
|
243 |
|
244 log.debug("%s: link: %s -> %s", tree, name, target) |
|
245 |
|
246 yield tree, name, target |
|
247 |
|
248 else : |
|
249 log.debug("%s: skip: %s", tree, name) |
|
250 |
|
251 |
|
252 class Interval (object) : |
|
253 """ |
|
254 An interval definition. |
|
255 """ |
|
256 |
|
257 @classmethod |
|
258 def from_config (cls, options, name, |
|
259 format, |
|
260 |
|
261 # deprecated |
|
262 keep = None, |
|
263 ) : |
|
264 if not format : |
|
265 # magic to use snapshot name |
|
266 _format = None |
|
267 else : |
|
268 _format = format |
|
269 |
|
270 return cls(name, |
|
271 format = _format, |
|
272 keep = config_int('keep', keep, default=None), |
|
273 ) |
|
274 |
|
275 @classmethod |
|
276 def from_target_config (cls, name, base, arg) : |
|
277 if isinstance(arg, dict) : |
|
278 # full instance |
|
279 return cls(name, |
|
280 format = arg.get('format', base.format if base else None), |
|
281 keep = arg.get('keep', base.keep if base else None), |
|
282 ) |
|
283 else : |
|
284 # partial instance with keep |
|
285 return cls(name, |
|
286 format = base.format, |
|
287 keep = config_int('keep', arg) if arg else base.keep, |
|
288 ) |
|
289 |
|
290 def __init__ (self, name, format, keep) : |
|
291 self.name = name |
|
292 self.format = format |
|
293 self.keep = keep |
|
294 |
|
295 def __str__ (self) : |
|
296 return self.name |
|
297 |
|
298 class Target (object) : |
|
299 """ |
|
300 A target run, i.e. a rsync-snapshot destination dir |
|
301 |
|
302 [target:...] |
|
303 """ |
|
304 |
|
305 @classmethod |
|
306 def config_intervals (cls, name, intervals) : |
|
307 for interval, arg in intervals.iteritems() : |
|
308 # lookup base from options.intervals |
|
309 try : |
|
310 base = options.intervals[process_config_name(interval)] |
|
311 except KeyError: |
|
312 raise ConfigError("Unknown interval for [target/{target}]: {interval}".format(target=name, interval=interval)) |
|
313 |
|
314 # parse |
|
315 yield Interval.from_target_config(interval, base, arg) |
|
316 |
|
317 # type() mapping for lvm_options |
|
318 LVM_OPTIONS = dict( |
|
319 wait = float, |
|
320 size = str, |
|
321 ) |
|
322 |
|
323 @classmethod |
|
324 def from_config (cls, options, name, |
|
325 path = False, |
|
326 source = None, |
|
327 enable = 'no', |
|
328 exclude_from = None, |
|
329 |
|
330 # subsections |
|
331 intervals = None, |
|
332 rsync_options = None, |
|
333 lvm_options = {}, |
|
334 ) : |
|
335 if not source and source is not False : |
|
336 raise ConfigError("Missing required option: source for [target/{name}]".format(name=name)) |
|
337 |
|
338 # process lvm opts by LVM_OPTIONS types |
|
339 lvm_options = dict((opt, cls.LVM_OPTIONS[opt](value)) for opt, value in lvm_options.iteritems()) |
|
340 |
|
341 # parse source -> rsync.RSyncServer |
|
342 source_path = source |
|
343 source = rsync.parse_source(source, lvm_opts=lvm_options) |
|
344 |
|
345 log.info("parse source: %r -> %s", source_path, source) |
|
346 |
|
347 # global defaults |
|
348 _rsync_options = dict(options.rsync_options) |
|
349 |
|
350 if rsync_options : |
|
351 # override |
|
352 _rsync_options.update([ |
|
353 # parse |
|
354 (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems() |
|
355 ]) |
|
356 |
|
357 if not intervals : |
|
358 raise ConfigError("Missing required [target/{name}/intervals]".format(name=name)) |
|
359 |
|
360 # lookup intervals |
|
361 _intervals = list(cls.config_intervals(name, intervals)) |
|
362 |
|
363 return cls(name, |
|
364 path = path if path else name, |
|
365 source = source, |
|
366 enable = config_bool('enable', enable), |
|
367 intervals = _intervals, |
|
368 rsync_options = _rsync_options, |
|
369 exclude_from = exclude_from, |
|
370 ) |
|
371 |
|
372 def __init__ (self, name, |
|
373 path, |
|
374 source, |
|
375 enable = False, |
|
376 intervals = [], |
|
377 rsync_options = {}, |
|
378 exclude_from = None |
|
379 ) : |
|
380 self.name = name |
|
381 |
|
382 self.path = path |
|
383 self.source = source |
|
384 self.enable = enable |
|
385 |
|
386 self.intervals = intervals |
|
387 |
|
388 self.rsync_options = rsync_options |
|
389 self.exclude_from = exclude_from |
|
390 |
|
391 # this snapshot? |
|
392 self.snapshots_dir = os.path.join(self.path, 'snapshots') |
|
393 |
|
394 # 'current' symlink |
|
395 self.current_path = os.path.join(self.path, 'current') |
|
396 |
|
397 def prepare (self, options) : |
|
398 """ |
|
399 Prepare dir for usage |
|
400 """ |
|
401 |
|
402 if not os.path.exists(self.path) : |
|
403 raise Exception("Missing target dir: {path}".format(path=self.path)) |
|
404 |
|
405 if not os.path.exists(self.snapshots_dir) : |
|
406 log.warn("Creating snapshots dir: %s", self.snapshots_dir) |
|
407 os.mkdir(self.snapshots_dir) |
|
408 |
|
409 def snapshot (self, options, now) : |
|
410 """ |
|
411 Perform the rsync from our source to self.snapshot_dir. |
|
412 |
|
413 XXX: allocate snapshot_name here? |
|
414 """ |
|
415 |
|
416 # new snapshot |
|
417 snapshot_name = now.strftime(options.snapshot_format) |
|
418 snapshot_path = os.path.join(self.snapshots_dir, snapshot_name) |
|
419 temp_path = os.path.join(self.snapshots_dir, 'tmp') |
|
420 |
|
421 if os.path.exists(temp_path) : |
|
422 raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path)) |
|
423 |
|
424 log.info("Perform main snapshot: %s -> %s", self.source, snapshot_path) |
|
425 |
|
426 # build rsync options |
|
427 opts = dict(self.rsync_options) |
|
428 |
|
429 if os.path.exists(self.current_path) : |
|
430 # real path to target |
|
431 target = os.readlink(self.current_path) |
|
432 target_path = os.path.join(os.path.dirname(self.current_path), target) |
|
433 target_abs = os.path.abspath(target_path) |
|
434 |
|
435 log.info("Using current -> %s as base", target_path) |
|
436 |
|
437 # use as link-dest base; hardlinks unchanged files; target directory must be empty |
|
438 # rsync links absolute paths.. |
|
439 opts['link-dest'] = target_abs |
|
440 |
|
441 log.debug("rsync %s -> %s", self.source, temp_path) |
|
442 |
|
443 # run the rsync.RSyncServer; None as a placeholder will get replaced with the actual source |
|
444 self.source.execute(invoke.optargs(**opts), srcdst=(None, temp_path)) |
|
445 |
|
446 # move in to final name |
|
447 log.debug("rename %s -> %s", temp_path, snapshot_path) |
|
448 os.rename(temp_path, snapshot_path) |
|
449 |
|
450 return snapshot_name |
|
451 |
|
452 def update_interval (self, options, interval, now, snapshot_name) : |
|
453 """ |
|
454 Update given <interval>/... links for this target, using the given new snapshot |
|
455 """ |
|
456 |
|
457 dir_path = os.path.join(self.path, interval.name) |
|
458 |
|
459 if not os.path.exists(dir_path) : |
|
460 log.warn("Creating interval dir: %s", dir_path) |
|
461 os.mkdir(dir_path) |
|
462 |
|
463 |
|
464 # name |
|
465 if interval.format is None : |
|
466 # per-snapshot |
|
467 name = snapshot_name |
|
468 |
|
469 log.debug("%s: using snapshot_name: %s", interval, name) |
|
470 |
|
471 else : |
|
472 # by date |
|
473 name = now.strftime(interval.format) |
|
474 |
|
475 log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name) |
|
476 |
|
477 # path |
|
478 path_name = os.path.join(interval.name, name) |
|
479 path = os.path.join(self.path, path_name) |
|
480 |
|
481 log.debug("%s: processing %s", interval, path_name) |
|
482 |
|
483 # already there? |
|
484 if os.path.exists(path) : |
|
485 target = os.readlink(path) |
|
486 |
|
487 log.info("%s: Keeping existing: %s -> %s", interval, name, target) |
|
488 |
|
489 else : |
|
490 # update |
|
491 target = os.path.join('..', 'snapshots', snapshot_name) |
|
492 |
|
493 log.info("%s: Updating: %s -> %s", interval, name, target) |
|
494 log.debug("%s -> %s", path, target) |
|
495 |
|
496 os.symlink(target, path) |
|
497 |
|
498 |
|
499 def clean_interval (self, options, interval) : |
|
500 """ |
|
501 Clean out given <interval>/... dir for this target. |
|
502 """ |
|
503 |
|
504 # path |
|
505 dir_path = os.path.join(self.path, interval.name) |
|
506 |
|
507 if not os.path.exists(dir_path) : |
|
508 log.warn("%s: Skipping, no interval dir: %s", interval, dir_path) |
|
509 return |
|
510 |
|
511 # configured |
|
512 keep = interval.keep |
|
513 |
|
514 if not keep : |
|
515 log.info("%s: Zero keep given, not cleaning up anything", interval) |
|
516 return |
|
517 |
|
518 # items to clean? |
|
519 items = os.listdir(dir_path) |
|
520 |
|
521 # sort newest -> oldest |
|
522 items.sort(reverse=True) |
|
523 |
|
524 log.info("%s: Have %d / %d items", interval, len(items), keep) |
|
525 log.debug("%s: items: %s", interval, ' '.join(items)) |
|
526 |
|
527 if len(items) > keep : |
|
528 # select oldest ones |
|
529 clean = items[keep:] |
|
530 |
|
531 log.info("%s: Cleaning out %d items", interval, len(clean)) |
|
532 log.debug("%s: cleaning out: %s", interval, ' '.join(clean)) |
|
533 |
|
534 for item in clean : |
|
535 path = os.path.join(dir_path, item) |
|
536 |
|
537 log.info("%s: Clean: %s", interval, path) |
|
538 |
|
539 if not options.dry_run : |
|
540 log.debug("rmtree: %s", path) |
|
541 os.unlink(path) |
|
542 else : |
|
543 log.debug("dryrun: %s", path) |
|
544 |
|
545 def clean_snapshots (self, options) : |
|
546 """ |
|
547 Clean out all snapshots for this target not linked to from within our root. |
|
548 |
|
549 Fails without doing anything if unable to read the destination dir. |
|
550 """ |
|
551 |
|
552 # real path to snapshots |
|
553 snapshots_path = os.path.realpath(os.path.abspath(self.snapshots_dir)) |
|
554 log.debug("real snapshots_path: %s", snapshots_path) |
|
555 |
|
556 # set of found targets |
|
557 found = set() |
|
558 |
|
559 # walk all symlinks |
|
560 for dirpath, name, target in walk_symlinks(self.path, ignore=set(['snapshots'])) : |
|
561 # target dir |
|
562 target_path = os.path.realpath(os.path.join(dirpath, target)) |
|
563 target_dir = os.path.dirname(target_path) |
|
564 target_name = os.path.basename(target_path) |
|
565 |
|
566 if target_dir == snapshots_path : |
|
567 log.debug("%s: found: %s -> %s", dirpath, name, target_name) |
|
568 found.add(target_name) |
|
569 |
|
570 else : |
|
571 log.debug("%s: ignore: %s -> %s", dirpath, name, target_path) |
|
572 |
|
573 # discover all snapshots |
|
574 snapshots = set(os.listdir(snapshots_path)) |
|
575 |
|
576 # clean out special names |
|
577 snapshots = snapshots - set(['new']) |
|
578 |
|
579 ## compare |
|
580 used = snapshots & found |
|
581 unused = snapshots - found |
|
582 broken = found - snapshots |
|
583 |
|
584 log.info("Found used=%d, unused=%d, broken=%d snapshot symlinks", len(used), len(unused), len(broken)) |
|
585 log.debug("used=%s, unused=%s", used, unused) |
|
586 |
|
587 if broken : |
|
588 log.warn("Found broken symlinks to snapshots: %s", ' '.join(broken)) |
|
589 |
|
590 if unused : |
|
591 log.info("Cleaning out %d unused snapshots:", len(unused)) |
|
592 |
|
593 for name in unused : |
|
594 path = os.path.join(snapshots_path, name) |
|
595 |
|
596 log.info("Clean: %s", name) |
|
597 |
|
598 if not options.dry_run : |
|
599 log.debug("rmtree: %s", path) |
|
600 |
|
601 # nuke |
|
602 shutil.rmtree(path) |
|
603 |
|
604 else : |
|
605 log.debug("dry-run: %s", path) |
|
606 |
|
607 def run_snapshot (self, options, now) : |
|
608 """ |
|
609 Run snapshot + update current. |
|
610 """ |
|
611 |
|
612 # initial rsync |
|
613 snapshot_name = self.snapshot(options, now) |
|
614 |
|
615 # update current |
|
616 log.info("Updating current -> %s", snapshot_name) |
|
617 |
|
618 if os.path.islink(self.current_path) : |
|
619 # replace |
|
620 os.unlink(self.current_path) |
|
621 |
|
622 os.symlink(os.path.join('snapshots', snapshot_name), self.current_path) |
|
623 |
|
624 return snapshot_name |
|
625 |
|
626 def run_intervals (self, options, now, snapshot_name) : |
|
627 """ |
|
628 Run our intervals. |
|
629 """ |
|
630 |
|
631 if not self.intervals : |
|
632 log.info("No intervals given; not running any") |
|
633 |
|
634 else : |
|
635 # maintain intervals |
|
636 log.info("Updating %d intervals...", len(self.intervals)) |
|
637 |
|
638 for interval in self.intervals : |
|
639 log.debug("%s", interval) |
|
640 |
|
641 log.info("Updating interval: %s", interval) |
|
642 |
|
643 # update |
|
644 self.update_interval(options, interval, now, snapshot_name) |
|
645 |
|
646 def run (self, options) : |
|
647 """ |
|
648 Execute |
|
649 """ |
|
650 |
|
651 # prep |
|
652 self.prepare(options) |
|
653 |
|
654 # clean intervals? |
|
655 if options.clean_intervals: |
|
656 for interval in self.intervals : |
|
657 log.info("Cleaning interval: %s...", interval) |
|
658 |
|
659 self.clean_interval(options, interval) |
|
660 |
|
661 # clean snapshots? |
|
662 if options.clean_snapshots : |
|
663 log.info("Cleaning snapshots...") |
|
664 |
|
665 self.clean_snapshots(options) |
|
666 |
|
667 # snapshot from source? |
|
668 if self.source : |
|
669 # timestamp for run |
|
670 now = datetime.datetime.now() |
|
671 |
|
672 log.info("Started snapshot run at: %s", now) |
|
673 |
|
674 # snapshot + current |
|
675 snapshot_name = self.run_snapshot(options, now) |
|
676 |
|
677 # intervals? |
|
678 self.run_intervals(options, now, snapshot_name) |
|
679 |
|
680 # ok |
|
681 return 1 |
|
682 |
|
683 def __str__ (self) : |
|
684 return self.name |
|
685 |
|
686 def _parse_run_targets (options, config, run) : |
|
687 """ |
|
688 Parse given run section from config into a series of target names to run. |
|
689 """ |
|
690 |
|
691 for target, enable in config['run'][process_config_name(options.run)].iteritems() : |
|
692 # enabled? |
|
693 enable = config_bool('enable', enable) |
|
694 |
|
695 if not enable : |
|
696 continue |
|
697 |
|
698 # check |
|
699 if target not in options.targets : |
|
700 raise ConfigError("Unknown [target/{target}] in [run/{run}]".format(target=target, run=run)) |
|
701 |
|
702 yield target |
|
703 |
|
704 def run (options, run_targets) : |
|
705 # default config |
|
706 config = dict( |
|
707 rsync_options = {}, |
|
708 intervals = {}, |
|
709 targets = {}, |
|
710 ) |
|
711 |
|
712 if options.config : |
|
713 # load |
|
714 try : |
|
715 config = parse_config(options.config, config) |
|
716 except ConfigError as e: |
|
717 log.error("Configuration error: %s: %s", options.config, e) |
|
718 return 2 |
|
719 |
|
720 # targets to run |
|
721 options.targets = {} |
|
722 |
|
723 # manual? |
|
724 if options.target : |
|
725 options.targets['console'] = Target.from_config( |
|
726 path = options.target, |
|
727 source = options.target_source, |
|
728 intervals = dict((name, None) for name in options.target_intervals), |
|
729 ) |
|
730 |
|
731 # intervals |
|
732 for name in config['intervals'] : |
|
733 interval_config = config['intervals'][name] |
|
734 |
|
735 # parse |
|
736 interval = Interval.from_config(options, name, **interval_config) |
|
737 |
|
738 log.debug("config interval: %s", name) |
|
739 |
|
740 # store |
|
741 options.intervals[name] = interval |
|
742 |
|
743 # rsync options |
|
744 for option in config['rsync_options'] : |
|
745 value = config['rsync_options'][option] |
|
746 |
|
747 # parse, allowing non-boolean values as well... |
|
748 value = config_bool(option, value, strict=False) |
|
749 |
|
750 log.debug("rsync option: %s=%s", option, value) |
|
751 |
|
752 # store |
|
753 options.rsync_options[option] = value |
|
754 |
|
755 # target definitions |
|
756 for name in config['targets'] : |
|
757 target_config = config['targets'][name] |
|
758 |
|
759 # parse |
|
760 target = Target.from_config(options, name, **target_config) |
|
761 |
|
762 log.debug("config target: %s", name) |
|
763 |
|
764 options.targets[name] = target |
|
765 |
|
766 # what targets? |
|
767 if run_targets : |
|
768 # keep as-is |
|
769 log.debug("Running given targets: %s", run_targets) |
|
770 |
|
771 if options.run : |
|
772 |
|
773 # given [run/...] definition.. |
|
774 run_targets = list(_parse_run_targets(options, config, options.run)) |
|
775 |
|
776 log.info("Running %d given [run/%s] targets", len(run_targets), options.run) |
|
777 log.debug("[run/%s]: %s", options.run, run_targets) |
|
778 |
|
779 # run |
|
780 if run_targets : |
|
781 log.info("Running %d given targets...", len(run_targets)) |
|
782 |
|
783 # run given ones |
|
784 for name in run_targets : |
|
785 try : |
|
786 # get |
|
787 target = options.targets[name] |
|
788 |
|
789 except KeyError: |
|
790 log.error("Unknown target given: %s", name) |
|
791 log.info("Defined targets: %s", ' '.join(options.targets)) |
|
792 return 2 |
|
793 |
|
794 |
|
795 # run |
|
796 log.info("Target: %s", name) |
|
797 |
|
798 target.run(options) |
|
799 |
|
800 else : |
|
801 # all targets |
|
802 log.info("Running all %d targets...", len(options.targets)) |
|
803 |
|
804 # targets |
|
805 for name, target in options.targets.iteritems() : |
|
806 log.info("Target: %s", name) |
|
807 |
|
808 # run |
|
809 target.run(options) |
|
810 |
|
811 # ok |
|
812 return 0 |
|
813 |
|
814 def config_defaults () : |
|
815 return dict( |
|
816 # snapshots/ naming |
|
817 snapshot_format = '%Y%m%d-%H%M%S', |
|
818 |
|
819 # rsync options, in invoke.optargs format |
|
820 rsync_options = { |
|
821 'archive': True, |
|
822 'hard-links': True, |
|
823 'one-file-system': True, |
|
824 'numeric-ids': True, |
|
825 'delete': True, |
|
826 }, |
|
827 |
|
828 # defined intervals |
|
829 intervals = dict((i.name, i) for i in [ |
|
830 Interval('recent', |
|
831 format = None, |
|
832 keep = 4, |
|
833 ), |
|
834 |
|
835 Interval('day', |
|
836 format = '%Y-%m-%d', |
|
837 keep = 7, |
|
838 ), |
|
839 |
|
840 Interval('week', |
|
841 format = '%Y-%W', |
|
842 keep = 4, |
|
843 ), |
|
844 |
|
845 Interval('month', |
|
846 format = '%Y-%m', |
|
847 keep = 4, |
|
848 ), |
|
849 |
|
850 Interval('year', |
|
851 format = '%Y', |
|
852 keep = 1, |
|
853 ) |
|
854 ]), |
|
855 ) |
|
856 |
|
857 def main (argv) : |
|
858 global options |
|
859 |
|
860 # option defaults |
|
861 defaults = config_defaults() |
|
862 |
|
863 # global options + args |
|
864 options, args = parse_options(argv, defaults) |
|
865 |
|
866 # args: filter targets |
|
867 # XXX: fix name mangling |
|
868 targets = [target.replace('-', '_') for target in args] |
|
869 |
|
870 try : |
|
871 # handle it |
|
872 return run(options, targets) |
|
873 |
|
874 except Exception, e: |
|
875 log.error("Internal error:", exc_info=e) |
|
876 return 3 |
|
877 |
|
878 # ok |
|
879 return 0 |
|
880 |
|
881 |
|
882 |
|
883 if __name__ == '__main__' : |
|
884 import sys |
|
885 |
|
886 sys.exit(main(sys.argv)) |
|
887 |