scripts/search-index
author Tero Marttila <terom@fixme.fi>
Tue, 10 Feb 2009 23:59:56 +0200
changeset 88 0b8e2ba5f76f
parent 83 scripts/search-index.py@a34e9f56ddda
child 89 2dc6de43f317
permissions -rwxr-xr-x
improve scripts/search-index with better help info
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     1
#!/usr/bin/env python2.5
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     2
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
    Tool for accessing the search index
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     5
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     6
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
     7
# XXX: fix path
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     9
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    10
import datetime, pytz
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    12
# configuration and the LogSearchIndex module
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    13
import config, log_search, channels
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    14
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    15
def _open_index (options, open_mode) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    16
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    17
        Opens the LogSearchIndex
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    18
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    19
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    20
    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    21
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    22
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    23
def _open_index_and_channel (options, channel_name, open_mode) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    24
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    25
        Opens+returns a LogSearchIndex and a LogChannel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    26
    """
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    27
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    28
    # open the LogSearchIndex
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    29
    index = _open_index(options, open_mode)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    30
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    31
    # open the channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    32
    channel = config.LOG_CHANNELS.lookup(channel_name)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    33
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    34
    # return
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    35
    return index, channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    36
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    37
def _load_channel_date (index, options, channel, date) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    38
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    39
        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    40
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    41
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    42
    if not options.quiet :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    43
        print "%s %s..." % (channel.id, date.strftime(channel.source.filename_fmt)),
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    44
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    45
    try :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    46
        # load lines for date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    47
        lines = channel.source.get_date(date)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    48
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    49
    except Exception, e :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    50
        if not options.skip_missing :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    51
            raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    52
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    53
        if not options.quiet :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    54
            print "Skipped: %s" % (e, )
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    55
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    56
    else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    57
        # insert -> count
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    58
        count = index.insert(channel, lines)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    59
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    60
        if not options.quiet :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    61
            print "OK: %d lines" % count
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    62
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    63
class CommandError (Exception) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    64
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    65
        Error with command-line arguments
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    66
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    67
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    68
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    69
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    70
def cmd_create (options) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    71
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    72
        Creates a new index
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    73
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    74
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    75
    # open index
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    76
    index = _open_index(options, 'c' if not options.force else '*')
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    77
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    78
    # that's all
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    79
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    80
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    81
def cmd_load (options, channel_name, *dates) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    82
    """
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    83
        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    84
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    85
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    86
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    87
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    88
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    89
    # handle each date
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    90
    for date_name in dates :
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
    91
        try :
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
    92
            # parse date
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
    93
            date = datetime.datetime.strptime(date_name, '%Y-%m-%d').replace(tzinfo=channel.source.tz)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    94
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
    95
        except Exception, e :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    96
            print "[ERROR] Invalid date: %s: %s" % (date_name, e)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    97
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    98
            if options.skip_missing :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    99
                continue
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   100
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   101
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   102
                raise
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   103
        
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   104
        # load
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   105
        _load_channel_date(index, options, channel, date)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   106
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   107
def cmd_load_month (options, channel_name, *months) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   108
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   109
        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   110
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   111
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   112
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   113
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   114
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   115
    # handle each date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   116
    for month_name in months :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   117
        try :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   118
            # parse date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   119
            month = datetime.datetime.strptime(month_name, '%Y-%m').replace(tzinfo=channel.source.tz)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   120
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   121
        except Exception, e :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   122
            print "[ERROR] Invalid date: %s: %s" % (month_name, e)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   123
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   124
            if options.skip_missing :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   125
                continue
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   126
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   127
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   128
                raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   129
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   130
        # get the set of days
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   131
        days = list(channel.source.get_month_days(month))
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   132
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   133
        print "Loading %d days of logs:" % (len(days))
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   134
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   135
        # load each day
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   136
        for date in days :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   137
            # convert to datetime
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   138
            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   139
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   140
            # load
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   141
            _load_channel_date(index, options, channel, dt)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   142
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   143
def cmd_search (options, channel_name, query) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   144
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   145
        Search the index for events on a specific channel with the given query
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   146
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   147
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   148
    # sanity-check
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   149
    if options.create_index :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   150
        raise Exception("--create doesn't make sense for 'search'")
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   151
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   152
    # open index/channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   153
    index, channel = _open_index_and_channel(options, channel_name, 'r')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   154
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   155
    # search
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   156
    lines = index.search_simple(channel, query)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   157
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   158
    # display as plaintext
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   159
    for line in options.formatter.format_txt(lines) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   160
        print line
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   161
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   162
def cmd_help (options, *args) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   163
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   164
        Help about commands
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   165
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   166
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   167
    import inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   168
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   169
    # general help stuff
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   170
    options._parser.print_help()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   171
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   172
    # specific command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   173
    if args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   174
        # the command name
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   175
        command, = args
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   176
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   177
        # XXX: display info about specific command
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   178
        xxx
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   179
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   180
    # general
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   181
    else :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   182
        print
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   183
        print "Available commands:"
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   184
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   185
        # build list of all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   186
        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   187
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   188
        # sort alphabetically
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   189
        cmd_objects.sort()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   190
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   191
        # iterate through all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   192
        for cmd_func_name, cmd_func in cmd_objects :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   193
            # remove cmd_ prefix
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   194
            cmd_name = cmd_func_name[4:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   195
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   196
            # inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   197
            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   198
            cmd_doc = inspect.getdoc(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   199
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   200
            # remove the "options" arg
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   201
            cmd_args = cmd_args[1:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   202
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   203
            # display
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   204
            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   205
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   206
def main (argv) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   207
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   208
        Command-line main, with given argv
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   209
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   210
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   211
    from optparse import OptionParser
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   212
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   213
    # define parser
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   214
    parser = OptionParser(
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   215
        usage           = "%prog [options] <command> [ ... ]",
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   216
        add_help_option = False,
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   217
    )
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   218
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   219
    # define command-line arguments
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   220
    parser.add_option('-h', "--help",           dest="help",            help="Show this help message and exit",     action="store_true")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   221
    parser.add_option('-F', "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 metavar="FMT",  type="choice", default="irssi",
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   222
        choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()])
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   223
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   224
    parser.add_option('-I', "--index",          dest="index_path",      help="Index database path",                 metavar="PATH", default="logs/index")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   225
    parser.add_option('-Z', "--timezone",       dest="tz_name",         help="Timezone for output",                 metavar="TZ",   default="UTC")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   226
    parser.add_option('-f', "--force",          dest="force",           help="Force dangerous operation",           action="store_true")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   227
    parser.add_option(      "--create",         dest="create",          help="Create index database",               action="store_true")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   228
    parser.add_option(      "--skip-missing",   dest="skip_missing",    help="Skip missing logfiles",               action="store_true")
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   229
    parser.add_option(      "--quiet",          dest="quiet",           help="Supress status messages",             action="store_true")
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   230
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   231
    # parse
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   232
    options, args = parser.parse_args(argv[1:])
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   233
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   234
    # postprocess stuff
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   235
    options._parser = parser
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   236
    options.tz = pytz.timezone(options.tz_name)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   237
    options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None)
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   238
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   239
    # special-case --help
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   240
    if options.help :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   241
        return cmd_help(options, *args)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   242
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   243
    # must have at least the command argument
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   244
    if not args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   245
        raise CommandError("Missing command")
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   246
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   247
    # pop command
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   248
    command = args.pop(0)
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   249
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   250
    # get func
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   251
    func = globals().get('cmd_%s' % command)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   252
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   253
    # unknown command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   254
    if not func :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   255
        raise CommandError("Unknown command: %s" % command)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   256
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   257
    # call
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   258
    func(options, *args)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   259
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   260
if __name__ == '__main__' :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   261
    try :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   262
        main(sys.argv)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   263
        sys.exit(0)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   264
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   265
    except CommandError, e :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   266
        print e
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   267
        sys.exit(1)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   268