scripts/search-index
author Tero Marttila <terom@fixme.fi>
Wed, 11 Feb 2009 04:14:22 +0200
changeset 103 0e829e6275dc
parent 101 f5f53cc0ce16
child 104 34c65a8c8b94
permissions -rwxr-xr-x
implement --until, and fix handling of ServerMode
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     1
#!/usr/bin/env python2.5
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     2
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
    Tool for accessing the search index
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     5
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     6
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
     7
# XXX: fix path
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     9
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
    10
import os, os.path, fcntl
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
import datetime, pytz
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    12
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    13
# configuration and the LogSearchIndex module
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    14
import config, utils, log_search, channels
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    15
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    16
def _open_index (options, open_mode) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    17
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    18
        Opens the LogSearchIndex
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    19
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    20
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    21
    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    22
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    23
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    24
def _open_index_and_channel (options, channel_name, open_mode) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    25
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    26
        Opens+returns a LogSearchIndex and a LogChannel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    27
    """
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    28
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    29
    # open the LogSearchIndex
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    30
    index = _open_index(options, open_mode)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    31
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    32
    # open the channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    33
    channel = config.LOG_CHANNELS.lookup(channel_name)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    34
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    35
    # return
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    36
    return index, channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    37
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    38
def _iter_insert_stats (index, channel, lines) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    39
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    40
        Insert the given lines into the index.
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    41
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    42
        Assumes the lines will be in time-order, and yields a series of (date, count) tuples for every date that lines
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    43
        are inserted for
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    44
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    45
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    46
    # last date
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    47
    date = None
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    48
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    49
    # count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    50
    count = 0
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    51
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    52
    # iter lines
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    53
    for line in lines :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    54
        # next day?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    55
        if not date or line.timestamp.date() != date :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    56
            if date :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    57
                # yield stats
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    58
                yield date, count
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    59
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    60
            # reset count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    61
            count = 0
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    62
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    63
            # timestamp's date
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    64
            date = line.timestamp.date()
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    65
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    66
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    67
        index.insert_line(channel, line)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    68
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    69
        # count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    70
        count += 1
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    71
    
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
    72
    # final count?
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
    73
    if date and count :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    74
        yield date, count
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    75
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    76
def _insert_lines (index, options, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    77
    """
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    78
        Insert the given lines into the index.
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    79
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    80
        Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    81
    """
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    82
    
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    83
    # iterate insert stats
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    84
    for date, count in _iter_insert_stats(index, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    85
        # output date header?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    86
        if not options.quiet :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    87
            print "%s: %s" % (date.strftime('%Y-%m-%d'), count),
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    88
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    89
def _load_channel_date (index, options, channel, date) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    90
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    91
        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    92
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    93
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    94
    if not options.quiet :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    95
        print "Loading date for channel %s" % channel.id
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    96
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    97
    try :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    98
        # load lines for date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    99
        lines = channel.source.get_date(date)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   100
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   101
    except Exception, e :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   102
        if not options.skip_missing :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   103
            raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   104
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   105
        if not options.quiet :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   106
            print "\tSkipped: %s" % (e, )
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   107
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   108
    else :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   109
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   110
        _insert_lines(index, options, channel, lines)
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   111
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   112
def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   113
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   114
        Parse the given datetime, using the given timezone(defaults to options.tz) and format
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   115
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   116
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   117
    # default tz
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   118
    if not tz :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   119
        tz = options.tz
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   120
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   121
    try :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   122
        # parse
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   123
        return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   124
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   125
    except Exception, e :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   126
        raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e))
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   127
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   128
def _output_lines (options, lines) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   129
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   130
        Display the formatted LogLines
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   131
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   132
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   133
    # display as plaintext
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   134
    for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   135
        print txt_data
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   136
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   137
class CommandError (Exception) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   138
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   139
        Error with command-line arguments
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   140
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   141
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   142
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   143
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   144
def cmd_create (options) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   145
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   146
        Creates a new index
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   147
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   148
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   149
    # open index
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   150
    index = _open_index(options, 'ctrunc' if options.force else 'c')
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   151
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   152
    # that's all
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   153
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   154
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   155
def cmd_load (options, channel_name, *dates) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   156
    """
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   157
        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   158
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   159
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   160
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   161
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   162
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   163
    # handle each date
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   164
    for date_str in dates :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   165
        # prase date
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   166
        try :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   167
            date = _parse_date(options, date_str, channel.source.tz)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   168
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   169
        # handle errors
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   170
        except CommandError, e :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   171
            if options.skip_missing :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   172
                print "[ERROR] %s" % (date_name, e)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   173
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   174
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   175
                raise
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   176
        
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   177
        # otherwise, load
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   178
        else :        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   179
            _load_channel_date(index, options, channel, date)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   180
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   181
def cmd_load_month (options, channel_name, *months) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   182
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   183
        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   184
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   185
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   186
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   187
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   188
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   189
    # handle each date
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   190
    for month_str in months :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   191
        # prase date
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   192
        try :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   193
            month = _parse_date(options, month_str, channel.source.tz, '%Y-%m')
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   194
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   195
        # handle errors
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   196
        except CommandError, e :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   197
            # skip?
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   198
            if options.skip_missing :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   199
                print "[ERROR] %s" % (date_name, e)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   200
                continue
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   201
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   202
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   203
                raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   204
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   205
        # get the set of days
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   206
        days = list(channel.source.get_month_days(month))
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   207
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   208
        print "Loading %d days of logs:" % (len(days))
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   209
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   210
        # load each day
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   211
        for date in days :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   212
            # convert to datetime
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   213
            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   214
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   215
            # load
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   216
            _load_channel_date(index, options, channel, dt)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   217
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   218
def cmd_search (options, channel_name, query) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   219
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   220
        Search the index for events on a specific channel with the given query
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   221
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   222
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   223
    # sanity-check
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   224
    if options.create :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   225
        raise Exception("--create doesn't make sense for 'search'")
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   226
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   227
    # open index/channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   228
    index, channel = _open_index_and_channel(options, channel_name, 'r')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   229
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   230
    # search
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   231
    lines = index.search_simple(channel, query)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   232
    
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   233
    # display
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   234
    _output_lines(options, lines)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   235
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   236
def cmd_list (options, channel_name, *dates) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   237
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   238
        List the indexed events for a specific date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   239
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   240
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   241
    # sanity-check
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   242
    if options.create :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   243
        raise Exception("--create doesn't make sense for 'search'")
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   244
    
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   245
    # open index/channel
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   246
    index, channel = _open_index_and_channel(options, channel_name, 'r')
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   247
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   248
    # ...for each date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   249
    for date_str in dates :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   250
        # parse date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   251
        date = _parse_date(options, date_str)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   252
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   253
        # list
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   254
        lines = index.list(channel, date)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   255
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   256
        # display
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   257
        _output_lines(options, lines)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   258
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   259
def _autoload_reset (options, channels) :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   260
    """
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   261
        Reset old autoload state
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   262
    """
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   263
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   264
    # warn
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   265
    if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   266
        print "[WARN] Resetting autoload state for: %s" % ', '.join(channel.id for channel in channels)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   267
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   268
    # iter
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   269
    for channel in channels :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   270
        # statefile path
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   271
        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   272
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   273
        # is it present?
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   274
        if not os.path.exists(statefile_path) :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   275
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   276
                print "[WARN] No statefile found at %s" % statefile_path
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   277
        
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   278
        else :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   279
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   280
                print "\t%s: " % channel.id,
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   281
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   282
            # remove the statefile
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   283
            os.remove(statefile_path)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   284
            
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   285
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   286
                print "OK"
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   287
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   288
def cmd_autoload (options, *channel_names) :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   289
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   290
        Automatically loads all channel logs that have not been indexed yet (by logfile mtime)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   291
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   292
    
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   293
    # open index, nonblocking
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   294
    index = _open_index(options, 'c?' if options.create else 'a?')
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   295
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   296
    # default to all channels
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   297
    if not channel_names :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   298
        channels = config.LOG_CHANNELS
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   299
    
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   300
    else :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   301
        channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names]
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   302
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   303
    # reset autoload state?
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   304
    if options.reset :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   305
        _autoload_reset(options, channels)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   306
        print
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   307
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   308
    # iterate channels
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   309
    for channel in channels :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   310
        if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   311
            print "Channel %s:" % channel.id
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   312
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   313
        # no 'after' by default
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   314
        after = None
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   315
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   316
        # path to our state file
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   317
        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   318
        statefile_tmppath = statefile_path + '.tmp'
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   319
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   320
        # does it exist?
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   321
        have_tmpfile = os.path.exists(statefile_tmppath)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   322
        
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   323
        # do we have a tempfile from a previous crash?
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   324
        if have_tmpfile and not options.ignore_resume :
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   325
            # first, open it...
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   326
            statefile_tmp = open(statefile_tmppath, 'r+')
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   327
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   328
            # ... then lock it
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   329
            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   330
            
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   331
            # read after timestamp
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   332
            after_str = statefile_tmp.read().rstrip()
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   333
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   334
            if after_str :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   335
                # parse timestamp
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   336
                after = utils.from_utc_timestamp(int(after_str))
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   337
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   338
                # one day backwards
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   339
                after = after.replace(day=after.day - 1)
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   340
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   341
                if not options.quiet :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   342
                    print "\tContinuing earlier progress: after=%s" % after
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   343
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   344
            else :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   345
                # ignore
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   346
                if not options.quiet :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   347
                    print "\t[WARN] Ignoring empty temporary statefile"
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   348
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   349
        else :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   350
            # warn about old tmpfile that was ignored
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   351
            if have_tmpfile and not options.quiet :
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   352
                print "\t[WARN] Ignoring old tmpfile state"
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   353
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   354
            # open new tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   355
            statefile_tmp = open(statefile_tmppath, 'w')
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   356
            
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   357
            # lock
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   358
            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   359
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   360
        # override?
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   361
        if options.reload :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   362
            # load all
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   363
            mtime = None
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   364
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   365
            if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   366
                print "\tForcing reload!"
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   367
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   368
        # stat for mtime
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   369
        else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   370
            # stat for mtime, None if unknown
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   371
            mtime = utils.mtime(statefile_path, ignore_missing=True)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   372
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   373
            if mtime and not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   374
                print "\tLast load time was %s" % mtime
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   375
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   376
            elif not options.quiet :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   377
                print "\t[WARN] No previous load state! Loading full logs"
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   378
 
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   379
        # only after some specific date?
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   380
        if options.after :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   381
            # use unless read from tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   382
            if not after :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   383
                after = options.after
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   384
               
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   385
                if not options.quiet :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   386
                    print "\tOnly including dates after %s" % after
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   387
            
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   388
            else :
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   389
                if not options.quiet :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   390
                    print "\t[WARN] Ignoring --after because we found a tempfile"
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   391
        
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   392
        # only up to some specific date?
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   393
        if options.until :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   394
            until = options.until
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   395
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   396
            if not options.quiet :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   397
                print "\tOnly including dates up to %s" % until
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   398
        else :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   399
            # default to now
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   400
            until = None
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   401
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   402
        # get lines
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   403
        lines = channel.source.get_modified(mtime, after, until)
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   404
        
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   405
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   406
        if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   407
            print "\tLoading and inserting..."
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   408
            print
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   409
     
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   410
        # iterate insert() per day to display info and update progress
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   411
        for date, count in _iter_insert_stats(index, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   412
            # output date header?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   413
            if not options.quiet :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   414
                print "\t%10s: %d" % (date.strftime('%Y-%m-%d'), count)
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   415
            
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   416
            # write temp state
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   417
            statefile_tmp.seek(0)
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   418
            statefile_tmp.write(str(utils.to_utc_timestamp(datetime.datetime.combine(date, datetime.time(0)))))
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   419
            statefile_tmp.flush()
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   420
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   421
        # write autoload state
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   422
        open(statefile_path, 'w').close()
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   423
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   424
        # close+delete tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   425
        statefile_tmp.close()
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   426
        os.remove(statefile_tmppath)
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   427
        
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   428
        print
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   429
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   430
    # done
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   431
    return
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   432
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   433
def cmd_help (options, *args) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   434
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   435
        Help about commands
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   436
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   437
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   438
    import inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   439
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   440
    # general help stuff
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   441
    options._parser.print_help()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   442
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   443
    # specific command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   444
    if args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   445
        # the command name
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   446
        command, = args
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   447
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   448
        # XXX: display info about specific command
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   449
        xxx
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   450
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   451
    # general
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   452
    else :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   453
        print
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   454
        print "Available commands:"
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   455
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   456
        # build list of all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   457
        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   458
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   459
        # sort alphabetically
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   460
        cmd_objects.sort()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   461
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   462
        # iterate through all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   463
        for cmd_func_name, cmd_func in cmd_objects :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   464
            # remove cmd_ prefix
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   465
            cmd_name = cmd_func_name[4:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   466
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   467
            # inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   468
            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   469
            cmd_doc = inspect.getdoc(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   470
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   471
            # remove the "options" arg
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   472
            cmd_args = cmd_args[1:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   473
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   474
            # display
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   475
            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   476
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   477
def main (argv) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   478
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   479
        Command-line main, with given argv
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   480
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   481
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   482
    from optparse import OptionParser
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   483
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   484
    # define parser
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   485
    parser = OptionParser(
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   486
        usage           = "%prog [options] <command> [ ... ]",
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   487
        add_help_option = False,
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   488
    )
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   489
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   490
    # define command-line arguments
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   491
    parser.add_option('-h', "--help",           dest="help",            help="Show this help message and exit",     action="store_true" )
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   492
    parser.add_option('-F', "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 metavar="FMT",  type="choice", default="irssi",
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   493
        choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()])
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   494
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   495
    parser.add_option('-I', "--index",          dest="index_path",      help="Index database path",                 metavar="PATH", default="logs/index"            )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   496
    parser.add_option(      "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir",      metavar="PATH", default="logs/autoload-state"   )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   497
    parser.add_option(      "--after",          dest="after",           help="Only autoload logfiles after the given date", metavar="DATE", default=None            )
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   498
    parser.add_option(      "--until",          dest="until",           help="Only autoload logfiles up to the given date", metavar="DATE", default=None            )
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   499
    parser.add_option('-Z', "--timezone",       dest="tz_name",         help="Timezone for output",                 metavar="TZ",   default="UTC"                   )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   500
    parser.add_option('-f', "--force",          dest="force",           help="Force dangerous operation",           action="store_true" )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   501
    parser.add_option(      "--create",         dest="create",          help="Create index database",               action="store_true" )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   502
    parser.add_option(      "--skip-missing",   dest="skip_missing",    help="Skip missing logfiles",               action="store_true" )
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   503
    parser.add_option(      "--reload",         dest="reload",          help="Force reload lines",                  action="store_true" )
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   504
    parser.add_option(      "--reset",          dest="reset",           help="Reset old autload state",             action="store_true" )
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   505
    parser.add_option(      "--ignore-resume",  dest="ignore_resume",   help="Do not try and resume interrupted autoload",  action="store_true" )
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   506
    parser.add_option(      "--quiet",          dest="quiet",           help="Supress status messages",             action="store_true" )
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   507
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   508
    # parse
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   509
    options, args = parser.parse_args(argv[1:])
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   510
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   511
    # postprocess stuff
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   512
    options._parser = parser
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   513
    options.tz = pytz.timezone(options.tz_name)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   514
    options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None)
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   515
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   516
    # XXX: convert to custom types
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   517
    if options.after :
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   518
        options.after = _parse_date(options, options.after)
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   519
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   520
    if options.until :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   521
        options.until = _parse_date(options, options.until)
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   522
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   523
    # special-case --help
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   524
    if options.help :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   525
        return cmd_help(options, *args)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   526
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   527
    # must have at least the command argument
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   528
    if not args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   529
        raise CommandError("Missing command")
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   530
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   531
    # pop command
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   532
    command = args.pop(0)
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   533
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   534
    # get func
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   535
    func = globals().get('cmd_%s' % command)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   536
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   537
    # unknown command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   538
    if not func :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   539
        raise CommandError("Unknown command: %s" % command)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   540
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   541
    # call
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   542
    func(options, *args)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   543
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   544
if __name__ == '__main__' :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   545
    try :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   546
        main(sys.argv)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   547
        sys.exit(0)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   548
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   549
    except CommandError, e :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   550
        print e
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   551
        sys.exit(1)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   552