terom@65: """ terom@65: Tool for accessing the search index terom@65: """ terom@65: terom@82: # XXX: fix path terom@65: import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..') terom@65: terom@65: import datetime, pytz terom@65: terom@82: # configuration and the LogSearchIndex module terom@82: import config, log_search, channels terom@82: terom@82: def _open_index_and_channel (options, channel_name, open_mode) : terom@82: """ terom@82: Opens+returns a LogSearchIndex and a LogChannel terom@82: """ terom@82: # open the LogSearchIndex terom@82: index = log_search.LogSearchIndex(options.index_path, open_mode) terom@82: terom@82: # open the channel terom@82: channel = config.LOG_CHANNELS.lookup(channel_name) terom@82: terom@82: # return terom@82: return index, channel terom@82: terom@82: def _load_channel_date (index, options, channel, date) : terom@82: """ terom@82: Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex terom@82: """ terom@82: terom@82: if not options.quiet : terom@82: print "%s %s..." % (channel.id, date.strftime(channel.source.filename_fmt)), terom@82: terom@82: try : terom@82: # load lines for date terom@82: lines = channel.source.get_date(date) terom@82: terom@82: except Exception, e : terom@82: if not options.skip_missing : terom@82: raise terom@82: terom@82: if not options.quiet : terom@82: print "Skipped: %s" % (e, ) terom@82: terom@82: else : terom@82: # insert -> count terom@82: count = index.insert(channel, lines) terom@82: terom@82: if not options.quiet : terom@82: print "OK: %d lines" % count terom@65: terom@65: def cmd_load (options, channel_name, *dates) : terom@65: """ terom@82: Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index terom@65: """ terom@65: terom@82: # open index/channel terom@82: index, channel = _open_index_and_channel(options, channel_name, '*' if options.create_index else 'a') terom@65: terom@82: # handle each date terom@65: for date_name in dates : terom@68: try : terom@68: # parse date terom@68: date = datetime.datetime.strptime(date_name, '%Y-%m-%d').replace(tzinfo=channel.source.tz) terom@65: terom@68: except Exception, e : terom@82: print "[ERROR] Invalid date: %s: %s" % (date_name, e) terom@82: terom@82: if options.skip_missing : terom@82: continue terom@82: terom@82: else : terom@82: raise terom@68: terom@82: # load terom@82: _load_channel_date(index, options, channel, date) terom@65: terom@82: def cmd_load_month (options, channel_name, *months) : terom@82: """ terom@82: Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index terom@82: """ terom@82: terom@82: # open index/channel terom@82: index, channel = _open_index_and_channel(options, channel_name, '*' if options.create_index else 'a') terom@82: terom@82: # handle each date terom@82: for month_name in months : terom@82: try : terom@82: # parse date terom@82: month = datetime.datetime.strptime(month_name, '%Y-%m').replace(tzinfo=channel.source.tz) terom@82: terom@82: except Exception, e : terom@82: print "[ERROR] Invalid date: %s: %s" % (month_name, e) terom@82: terom@82: if options.skip_missing : terom@82: continue terom@82: terom@82: else : terom@82: raise terom@82: terom@82: # get the set of days terom@82: days = channel.source.get_month_days(month) terom@82: terom@82: print "Loading %d days of logs:" % (len(days)) terom@82: terom@82: # load each day terom@82: for date in days : terom@82: # convert to datetime terom@82: dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz) terom@82: terom@82: # load terom@82: _load_channel_date(index, options, channel, dt) terom@65: terom@65: def cmd_search (options, channel_name, query) : terom@65: """ terom@65: Search the index for events on a specific channel with the given query terom@65: """ terom@65: terom@82: # sanity-check terom@82: if options.create_index : terom@82: raise Exception("--create doesn't make sense for 'search'") terom@82: terom@82: # open index/channel terom@82: index, channel = _open_index_and_channel(options, channel_name, 'r') terom@65: terom@65: # search terom@65: lines = index.search_simple(channel, query) terom@65: terom@65: # display as plaintext terom@65: for line in options.formatter.format_txt(lines) : terom@65: print line terom@65: terom@65: if __name__ == '__main__' : terom@65: from optparse import OptionParser terom@65: terom@65: # define parser terom@65: parser = OptionParser( terom@65: usage = "%prog [options] [ ... ]", terom@65: add_help_option = True, terom@65: ) terom@65: terom@65: # define command-line arguments terom@65: parser.add_option("-I", "--index", dest="index_path", help="Index database path", metavar="PATH", default="logs/index") terom@68: parser.add_option("--create", dest="create_index", action="store_true", help="Create index database") terom@65: parser.add_option("-f", "--formatter", dest="formatter_name", help="LogFormatter to use", default="irssi") terom@65: parser.add_option("-z", "--timezone", dest="tz_name", help="Timezone for output", metavar="TZ", default="UTC") terom@68: parser.add_option("--skip-missing", dest="skip_missing", action="store_true", help="Skip missing logfiles") terom@82: parser.add_option("--quiet", dest="quiet", action="store_true", help="Supress status messages") terom@65: terom@65: # parse terom@65: options, args = parser.parse_args() terom@65: terom@65: # postprocess stuff terom@65: options.tz = pytz.timezone(options.tz_name) terom@82: options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None) terom@65: terom@65: # pop command terom@82: if not args : terom@82: raise Exception("Missing command") terom@82: terom@65: command = args.pop(0) terom@65: terom@65: # inspect terom@65: func = globals()['cmd_%s' % command] terom@65: terom@65: # call terom@65: func(options, *args) terom@65: terom@65: