scripts/search-index
changeset 105 e24da9a94ffb
parent 104 34c65a8c8b94
child 106 0690d715385d
equal deleted inserted replaced
104:34c65a8c8b94 105:e24da9a94ffb
     7 # XXX: fix path
     7 # XXX: fix path
     8 import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
     8 import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
     9 
     9 
    10 import os, os.path, fcntl
    10 import os, os.path, fcntl
    11 import datetime, pytz
    11 import datetime, pytz
       
    12 import optparse
    12 
    13 
    13 # configuration and the LogSearchIndex module
    14 # configuration and the LogSearchIndex module
    14 import config, utils, log_search, channels
    15 import config, utils, log_search, channels
    15 
    16 
    16 def _open_index (options, open_mode) :
    17 def _open_index (options, open_mode) :
   114         Parse the given datetime, using the given timezone(defaults to options.tz) and format
   115         Parse the given datetime, using the given timezone(defaults to options.tz) and format
   115     """
   116     """
   116 
   117 
   117     # default tz
   118     # default tz
   118     if not tz :
   119     if not tz :
   119         tz = options.tz
   120         tz = options.timezone
   120 
   121 
   121     try :
   122     try :
   122         # parse
   123         # parse
   123         return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)
   124         return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)
   124 
   125 
   386                     print "\tOnly including dates after %s" % after
   387                     print "\tOnly including dates after %s" % after
   387             
   388             
   388             else :
   389             else :
   389                 if not options.quiet :
   390                 if not options.quiet :
   390                     print "\t[WARN] Ignoring --after because we found a tempfile"
   391                     print "\t[WARN] Ignoring --after because we found a tempfile"
   391         
   392             
   392         # only up to some specific date?
   393         # only up to some specific date?
   393         if options.until :
   394         if options.until :
   394             until = options.until
   395             until = options.until
   395 
   396 
   396             if not options.quiet :
   397             if not options.quiet :
   472             cmd_args = cmd_args[1:]
   473             cmd_args = cmd_args[1:]
   473 
   474 
   474             # display
   475             # display
   475             print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)
   476             print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)
   476 
   477 
       
   478 class MyOption (optparse.Option) :
       
   479     """
       
   480         Our custom types for optparse
       
   481     """
       
   482 
       
   483     def check_date (option, opt, value) :
       
   484         """
       
   485             Parse a date
       
   486         """
       
   487 
       
   488         try :
       
   489             # parse
       
   490             return datetime.datetime.strptime(value, '%Y-%m-%d')
       
   491         
       
   492         # trap -> OptionValueError
       
   493         except Exception, e :
       
   494             raise optparse.OptionValueError("option %s: invalid date value: %r" % (opt, value))
       
   495     
       
   496     def check_timezone (option, opt, value) :
       
   497         """
       
   498             Parse a timezone
       
   499         """
       
   500 
       
   501         try :
       
   502             # parse
       
   503             return pytz.timezone(value)
       
   504         
       
   505         # trap -> OptionValueError
       
   506         except Exception, e :
       
   507             raise optparse.OptionValueError("option %s: invalid timezone: %r" % (opt, value))
       
   508 
       
   509     def take_action (self, action, dest, opt, value, values, parser) :
       
   510         """
       
   511             Override take_action to handle date
       
   512         """
       
   513 
       
   514         if action == "parse_date" :
       
   515             # get timezone
       
   516             tz = values.timezone
       
   517 
       
   518             # set timezone
       
   519             value = value.replace(tzinfo=tz)
       
   520 
       
   521             # store
       
   522             return optparse.Option.take_action(self, 'store', dest, opt, value, values, parser)
       
   523 
       
   524         else :
       
   525             # default
       
   526             return optparse.Option.take_action(self, action, dest, opt, value, values, parser)
       
   527 
       
   528     TYPES = optparse.Option.TYPES + ('date', 'timezone')
       
   529     TYPE_CHECKER = optparse.Option.TYPE_CHECKER.copy()
       
   530     TYPE_CHECKER['date'] = check_date
       
   531     TYPE_CHECKER['timezone'] = check_timezone
       
   532     ACTIONS = optparse.Option.ACTIONS + ('parse_date', )
       
   533     STORE_ACTIONS = optparse.Option.STORE_ACTIONS + ('parse_date', )
       
   534     TYPED_ACTIONS = optparse.Option.TYPED_ACTIONS + ('parse_date', )
       
   535     ACTIONS = optparse.Option.ACTIONS + ('parse_date', )
       
   536 
   477 def main (argv) :
   537 def main (argv) :
   478     """
   538     """
   479         Command-line main, with given argv
   539         Command-line main, with given argv
   480     """
   540     """
   481 
   541 
   482     from optparse import OptionParser, OptionGroup
       
   483     
       
   484     # define parser
   542     # define parser
   485     parser = OptionParser(
   543     parser = optparse.OptionParser(
   486         usage           = "%prog [options] <command> [ ... ]",
   544         usage           = "%prog [options] <command> [ ... ]",
   487         add_help_option = False,
   545         add_help_option = False,
       
   546         option_class    = MyOption,
   488     )
   547     )
   489 
   548 
   490     # define command-line arguments
   549     # define command-line arguments
   491     general = OptionGroup(parser, "General Options")
   550     general = optparse.OptionGroup(parser, "General Options")
   492     general.add_option('-h', "--help",           dest="help",            help="Show this help message and exit",     action="store_true" )
   551     general.add_option('-h', "--help",           dest="help",            help="Show this help message and exit",     action="store_true"    )
   493     general.add_option(      "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 metavar="FMT",  type="choice", default="irssi",
   552     general.add_option(      "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 metavar="FMT",  type="choice", default="irssi",
   494         choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()])
   553         choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()])
   495 
   554 
   496     general.add_option(      "--index",          dest="index_path",      help="Index database path",                 metavar="PATH", default="logs/index"            )
   555     general.add_option(      "--index",          dest="index_path",      help="Index database path",                 metavar="PATH", default="logs/index"               )
   497     general.add_option(      "--timezone",       dest="tz_name",         help="Timezone for output",                 metavar="TZ",   default="UTC"                   )
   556     general.add_option(      "--timezone",       dest="timezone",        help="Timezone for output",                 metavar="TZ",   type="timezone", default=pytz.utc  )
   498     general.add_option(      "--force",          dest="force",           help="Force dangerous operation",           action="store_true" )
   557     general.add_option(      "--force",          dest="force",           help="Force dangerous operation",           action="store_true"    )
   499     general.add_option(      "--quiet",          dest="quiet",           help="Supress status messages",             action="store_true" )
   558     general.add_option(      "--quiet",          dest="quiet",           help="Supress status messages",             action="store_true"    )
   500     parser.add_option_group(general)
   559     parser.add_option_group(general)
   501     
   560     
   502     load = OptionGroup(parser, "Load Options")
   561     load = optparse.OptionGroup(parser, "Load Options")
   503     load.add_option(      "--skip-missing",   dest="skip_missing",    help="Skip missing logfiles",               action="store_true" )
   562     load.add_option(      "--skip-missing",   dest="skip_missing",    help="Skip missing logfiles",               action="store_true"       )
   504     load.add_option(      "--create",         dest="create",          help="Create index database",               action="store_true" )
   563     load.add_option(      "--create",         dest="create",          help="Create index database",               action="store_true"       )
   505     parser.add_option_group(load)
   564     parser.add_option_group(load)
   506     
   565     
   507     autoload = OptionGroup(parser, "Autoload Options")
   566     autoload = optparse.OptionGroup(parser, "Autoload Options")
   508     autoload.add_option(      "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir",      metavar="PATH", default="logs/autoload-state"   )
   567     autoload.add_option(      "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir",      metavar="PATH", default="logs/autoload-state"   )
   509     autoload.add_option(      "--after",          dest="after",           help="Only autoload logfiles after the given date", metavar="DATE", default=None            )
   568     autoload.add_option(      "--after",          dest="after",           help="Only autoload logfiles after the given date", metavar="DATE", type="date", action="parse_date", default=None )
   510     autoload.add_option(      "--until",          dest="until",           help="Only autoload logfiles up to the given date", metavar="DATE", default=None            )
   569     autoload.add_option(      "--until",          dest="until",           help="Only autoload logfiles up to the given date", metavar="DATE", type="date", action="parse_date", default=None )
   511     autoload.add_option(      "--reload",         dest="reload",          help="Force reload lines",                  action="store_true" )
   570     autoload.add_option(      "--reload",         dest="reload",          help="Force reload lines",                  action="store_true"   )
   512     autoload.add_option(      "--reset",          dest="reset",           help="Reset old autload state",             action="store_true" )
   571     autoload.add_option(      "--reset",          dest="reset",           help="Reset old autload state",             action="store_true"   )
   513     autoload.add_option(      "--ignore-resume",  dest="ignore_resume",   help="Do not try and resume interrupted autoload",  action="store_true" )
   572     autoload.add_option(      "--ignore-resume",  dest="ignore_resume",   help="Do not try and resume interrupted autoload",  action="store_true" )
   514     parser.add_option_group(autoload)
   573     parser.add_option_group(autoload)
   515 
   574 
   516     # parse
   575     # parse
   517     options, args = parser.parse_args(argv[1:])
   576     options, args = parser.parse_args(argv[1:])
   518 
   577 
   519     # postprocess stuff
   578     # postprocess stuff
   520     options._parser = parser
   579     options._parser = parser
   521     options.tz = pytz.timezone(options.tz_name)
   580     options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.timezone, "%H:%M:%S", None, None)
   522     options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None)
       
   523 
       
   524     # XXX: convert to custom types
       
   525     if options.after :
       
   526         options.after = _parse_date(options, options.after)
       
   527 
       
   528     if options.until :
       
   529         options.until = _parse_date(options, options.until)
       
   530 
   581 
   531     # special-case --help
   582     # special-case --help
   532     if options.help :
   583     if options.help :
   533         return cmd_help(options, *args)
   584         return cmd_help(options, *args)
   534     
   585