scripts/search-index
author Tero Marttila <terom@fixme.fi>
Fri, 13 Feb 2009 00:29:47 +0200
changeset 121 86aebc9cb60b
parent 106 0690d715385d
permissions -rwxr-xr-x
some quickfixes to fix deployment errors
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     1
#!/usr/bin/env python2.5
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
     2
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
    Tool for accessing the search index
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     5
"""
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     6
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
     7
# XXX: fix path
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     8
import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     9
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
    10
import os, os.path, fcntl
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
import datetime, pytz
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
    12
import optparse
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    13
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    14
# configuration and the LogSearchIndex module
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    15
import config, utils, log_search, channels
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    16
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    17
def _open_index (options, open_mode) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    18
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    19
        Opens the LogSearchIndex
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    20
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    21
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    22
    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    23
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    24
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    25
def _open_index_and_channel (options, channel_name, open_mode) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    26
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    27
        Opens+returns a LogSearchIndex and a LogChannel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    28
    """
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    29
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    30
    # open the LogSearchIndex
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
    31
    index = _open_index(options, open_mode)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    32
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    33
    # open the channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    34
    channel = config.LOG_CHANNELS.lookup(channel_name)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    35
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    36
    # return
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    37
    return index, channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    38
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    39
def _iter_insert_stats (index, channel, lines) :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    40
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    41
        Insert the given lines into the index.
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    42
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    43
        Assumes the lines will be in time-order, and yields a series of (date, count) tuples for every date that lines
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    44
        are inserted for
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    45
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    46
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    47
    # last date
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    48
    date = None
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    49
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    50
    # count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    51
    count = 0
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    52
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    53
    # iter lines
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    54
    for line in lines :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    55
        # next day?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    56
        if not date or line.timestamp.date() != date :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    57
            if date :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    58
                # yield stats
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    59
                yield date, count
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    60
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    61
            # reset count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    62
            count = 0
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    63
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    64
            # timestamp's date
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    65
            date = line.timestamp.date()
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    66
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    67
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    68
        index.insert_line(channel, line)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    69
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    70
        # count
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    71
        count += 1
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    72
    
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
    73
    # final count?
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
    74
    if date and count :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    75
        yield date, count
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    76
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    77
def _insert_lines (index, options, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    78
    """
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    79
        Insert the given lines into the index.
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    80
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    81
        Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    82
    """
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    83
    
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    84
    # iterate insert stats
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    85
    for date, count in _iter_insert_stats(index, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    86
        # output date header?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    87
        if not options.quiet :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    88
            print "%s: %s" % (date.strftime('%Y-%m-%d'), count),
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    89
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    90
def _load_channel_date (index, options, channel, date) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    91
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    92
        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    93
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    94
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    95
    if not options.quiet :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    96
        print "Loading date for channel %s" % channel.id
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    97
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    98
    try :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
    99
        # load lines for date
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   100
        lines = channel.source.get_date(date)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   101
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   102
    except Exception, e :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   103
        if not options.skip_missing :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   104
            raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   105
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   106
        if not options.quiet :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   107
            print "\tSkipped: %s" % (e, )
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   108
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   109
    else :
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   110
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   111
        _insert_lines(index, options, channel, lines)
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   112
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   113
def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   114
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   115
        Parse the given datetime, using the given timezone(defaults to options.tz) and format
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   116
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   117
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   118
    # default tz
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   119
    if not tz :
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   120
        tz = options.timezone
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   121
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   122
    try :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   123
        # parse
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   124
        return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   125
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   126
    except Exception, e :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   127
        raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e))
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   128
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   129
def _output_lines (options, lines) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   130
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   131
        Display the formatted LogLines
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   132
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   133
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   134
    # display as plaintext
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   135
    for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   136
        print txt_data
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   137
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   138
class CommandError (Exception) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   139
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   140
        Error with command-line arguments
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   141
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   142
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   143
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   144
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   145
def cmd_create (options) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   146
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   147
        Creates a new index
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   148
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   149
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   150
    # open index
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   151
    index = _open_index(options, 'ctrunc' if options.force else 'c')
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   152
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   153
    # that's all
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   154
    pass
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   155
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   156
def cmd_load (options, channel_name, *dates) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   157
    """
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   158
        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   159
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   160
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   161
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   162
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   163
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   164
    # handle each date
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   165
    for date_str in dates :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   166
        # prase date
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   167
        try :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   168
            date = _parse_date(options, date_str, channel.source.tz)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   169
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   170
        # handle errors
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   171
        except CommandError, e :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   172
            if options.skip_missing :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   173
                print "[ERROR] %s" % (date_name, e)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   174
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   175
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   176
                raise
68
8157c41b3236 improve search form & script
Tero Marttila <terom@fixme.fi>
parents: 65
diff changeset
   177
        
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   178
        # otherwise, load
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   179
        else :        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   180
            _load_channel_date(index, options, channel, date)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   181
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   182
def cmd_load_month (options, channel_name, *months) :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   183
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   184
        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   185
    """
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   186
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   187
    # open index/channel
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   188
    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   189
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   190
    # handle each date
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   191
    for month_str in months :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   192
        # prase date
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   193
        try :
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   194
            month = _parse_date(options, month_str, channel.source.tz, '%Y-%m')
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   195
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   196
        # handle errors
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   197
        except CommandError, e :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   198
            # skip?
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   199
            if options.skip_missing :
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   200
                if not options.quiet :
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   201
                    print "[ERROR] %s" % (date_name, e)
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   202
                continue
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   203
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   204
            else :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   205
                raise
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   206
        
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   207
        # get the set of days
83
a34e9f56ddda improve parser resilience, improve get_month_days, add 'Channel' item to general menu
Tero Marttila <terom@fixme.fi>
parents: 82
diff changeset
   208
        days = list(channel.source.get_month_days(month))
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   209
        
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   210
        if not options.quiet :
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   211
            print "Loading %d days of logs:" % (len(days))
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   212
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   213
        # load each day
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   214
        for date in days :
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   215
            # convert to datetime
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   216
            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   217
            
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   218
            # load
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   219
            _load_channel_date(index, options, channel, dt)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   220
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   221
def cmd_search (options, channel_name, query) :
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   222
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   223
        Search the index for events on a specific channel with the given query
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   224
    """
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   225
    
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   226
    # sanity-check
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   227
    if options.create :
82
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   228
        raise Exception("--create doesn't make sense for 'search'")
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   229
    
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   230
    # open index/channel
afd3120ec71e add a LogSourceDecoder to fallback from utf-8 to latin-1, and improve scripts/search-index.py
Tero Marttila <terom@fixme.fi>
parents: 68
diff changeset
   231
    index, channel = _open_index_and_channel(options, channel_name, 'r')
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   232
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   233
    # search
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   234
    lines = index.search_simple(channel, query)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   235
    
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   236
    # display
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   237
    _output_lines(options, lines)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   238
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   239
def cmd_list (options, channel_name, *dates) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   240
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   241
        List the indexed events for a specific date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   242
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   243
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   244
    # sanity-check
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   245
    if options.create :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   246
        raise Exception("--create doesn't make sense for 'search'")
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   247
    
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   248
    # open index/channel
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   249
    index, channel = _open_index_and_channel(options, channel_name, 'r')
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   250
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   251
    # ...for each date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   252
    for date_str in dates :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   253
        # parse date
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   254
        date = _parse_date(options, date_str)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   255
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   256
        # list
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   257
        lines = index.list(channel, date)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   258
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   259
        # display
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 88
diff changeset
   260
        _output_lines(options, lines)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   261
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   262
def _autoload_reset (options, channels) :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   263
    """
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   264
        Reset old autoload state
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   265
    """
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   266
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   267
    # warn
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   268
    if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   269
        print "[WARN] Resetting autoload state for: %s" % ', '.join(channel.id for channel in channels)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   270
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   271
    # iter
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   272
    for channel in channels :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   273
        # statefile path
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   274
        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   275
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   276
        # is it present?
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   277
        if not os.path.exists(statefile_path) :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   278
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   279
                print "[WARN] No statefile found at %s" % statefile_path
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   280
        
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   281
        else :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   282
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   283
                print "\t%s: " % channel.id,
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   284
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   285
            # remove the statefile
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   286
            os.remove(statefile_path)
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   287
            
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   288
            if not options.quiet :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   289
                print "OK"
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   290
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   291
def cmd_autoload (options, *channel_names) :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   292
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   293
        Automatically loads all channel logs that have not been indexed yet (by logfile mtime)
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   294
    """
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   295
    
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   296
    # open index, nonblocking
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   297
    index = _open_index(options, 'c?' if options.create else 'a?')
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   298
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   299
    # default to all channels
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   300
    if not channel_names :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   301
        channels = config.LOG_CHANNELS
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   302
    
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   303
    else :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   304
        channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names]
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   305
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   306
    # reset autoload state?
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   307
    if options.reset :
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   308
        _autoload_reset(options, channels)
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   309
        if not options.quiet :
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   310
            print
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   311
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   312
    # iterate channels
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   313
    for channel in channels :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   314
        if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   315
            print "Channel %s:" % channel.id
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   316
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   317
        # no 'from' by default
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   318
        after = None
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   319
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   320
        # path to our state file
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   321
        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   322
        statefile_tmppath = statefile_path + '.tmp'
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   323
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   324
        # does it exist?
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   325
        have_tmpfile = os.path.exists(statefile_tmppath)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   326
        
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   327
        # do we have a tempfile from a previous crash?
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   328
        if have_tmpfile and not options.ignore_resume :
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   329
            # first, open it...
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   330
            statefile_tmp = open(statefile_tmppath, 'r+')
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   331
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   332
            # ... then lock it
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   333
            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   334
            
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   335
            # read after timestamp
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   336
            after_str = statefile_tmp.read().rstrip()
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   337
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   338
            if after_str :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   339
                # parse timestamp
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   340
                after = utils.from_utc_timestamp(int(after_str))
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   341
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   342
                if not options.quiet :
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   343
                    print "\tContinuing earlier progress from %s" % after
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   344
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   345
            else :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   346
                # ignore
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   347
                if not options.quiet :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   348
                    print "\t[WARN] Ignoring empty temporary statefile"
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   349
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   350
        else :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   351
            # warn about old tmpfile that was ignored
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   352
            if have_tmpfile and not options.quiet :
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   353
                print "\t[WARN] Ignoring old tmpfile state"
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   354
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   355
            # open new tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   356
            statefile_tmp = open(statefile_tmppath, 'w')
99
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   357
            
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   358
            # lock
8719ac564b22 implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
Tero Marttila <terom@fixme.fi>
parents: 98
diff changeset
   359
            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   360
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   361
        # override?
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   362
        if options.reload :
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   363
            # load all
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   364
            mtime = None
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   365
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   366
            if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   367
                print "\tForcing reload!"
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   368
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   369
        # stat for mtime
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   370
        else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   371
            # stat for mtime, None if unknown
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   372
            mtime = utils.mtime(statefile_path, ignore_missing=True)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   373
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   374
            if mtime and not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   375
                print "\tLast load time was %s" % mtime
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   376
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 94
diff changeset
   377
            elif not options.quiet :
100
147be2910b02 implement ignore-resume option
Tero Marttila <terom@fixme.fi>
parents: 99
diff changeset
   378
                print "\t[WARN] No previous load state! Loading full logs"
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   379
 
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   380
        # only after some specific date?
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   381
        if options.after :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   382
            # use unless read from tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   383
            if not after :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   384
                after = options.after
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   385
               
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   386
                if not options.quiet :
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   387
                    print "\tOnly including dates from %s onwards" % after
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   388
            
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   389
            else :
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   390
                if not options.quiet :
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   391
                    print "\t[WARN] Ignoring --from because we found a tempfile"
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   392
            
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   393
        # only up to some specific date?
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   394
        if options.until :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   395
            until = options.until
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   396
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   397
            if not options.quiet :
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   398
                print "\tOnly including dates up to (and including) %s" % until
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   399
        else :
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   400
            # default to now
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   401
            until = None
94
6673de9bc911 implement --after for autoload
Tero Marttila <terom@fixme.fi>
parents: 93
diff changeset
   402
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   403
        # get lines
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   404
        lines = channel.source.get_modified(mtime, after, until)
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   405
        
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   406
        # insert
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   407
        if not options.quiet :
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   408
            print "\tLoading and inserting..."
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   409
            print
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   410
     
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   411
        # iterate insert() per day to display info and update progress
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   412
        for date, count in _iter_insert_stats(index, channel, lines) :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   413
            # output date header?
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   414
            if not options.quiet :
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   415
                print "\t%10s: %d" % (date.strftime('%Y-%m-%d'), count)
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   416
            
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   417
            # write temp state
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   418
            statefile_tmp.seek(0)
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   419
            statefile_tmp.write(str(utils.to_utc_timestamp(datetime.datetime.combine(date, datetime.time(0)))))
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   420
            statefile_tmp.flush()
93
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   421
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   422
        # write autoload state
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   423
        open(statefile_path, 'w').close()
48fca00689e3 implement scripts/search-index autoload
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   424
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   425
        # close+delete tempfile
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   426
        statefile_tmp.close()
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   427
        os.remove(statefile_tmppath)
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   428
        
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   429
        if not options.quiet :
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   430
            print
101
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   431
    
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   432
    # done
f5f53cc0ce16 implement autoload --reset
Tero Marttila <terom@fixme.fi>
parents: 100
diff changeset
   433
    return
98
8c6e36849f9a implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
   434
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   435
def cmd_help (options, *args) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   436
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   437
        Help about commands
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   438
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   439
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   440
    import inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   441
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   442
    # general help stuff
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   443
    options._parser.print_help()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   444
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   445
    # specific command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   446
    if args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   447
        # the command name
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   448
        command, = args
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   449
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   450
        # XXX: display info about specific command
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   451
        xxx
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   452
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   453
    # general
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   454
    else :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   455
        print
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   456
        print "Available commands:"
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   457
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   458
        # build list of all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   459
        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   460
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   461
        # sort alphabetically
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   462
        cmd_objects.sort()
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   463
        
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   464
        # iterate through all cmd_* objects
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   465
        for cmd_func_name, cmd_func in cmd_objects :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   466
            # remove cmd_ prefix
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   467
            cmd_name = cmd_func_name[4:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   468
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   469
            # inspect
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   470
            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   471
            cmd_doc = inspect.getdoc(cmd_func)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   472
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   473
            # remove the "options" arg
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   474
            cmd_args = cmd_args[1:]
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   475
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   476
            # display
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   477
            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   478
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   479
class MyOption (optparse.Option) :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   480
    """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   481
        Our custom types for optparse
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   482
    """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   483
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   484
    def check_date (option, opt, value) :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   485
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   486
            Parse a date
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   487
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   488
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   489
        try :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   490
            # parse
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   491
            return datetime.datetime.strptime(value, '%Y-%m-%d')
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   492
        
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   493
        # trap -> OptionValueError
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   494
        except Exception, e :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   495
            raise optparse.OptionValueError("option %s: invalid date value: %r" % (opt, value))
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   496
    
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   497
    def check_timezone (option, opt, value) :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   498
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   499
            Parse a timezone
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   500
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   501
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   502
        try :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   503
            # parse
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   504
            return pytz.timezone(value)
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   505
        
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   506
        # trap -> OptionValueError
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   507
        except Exception, e :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   508
            raise optparse.OptionValueError("option %s: invalid timezone: %r" % (opt, value))
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   509
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   510
    def take_action (self, action, dest, opt, value, values, parser) :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   511
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   512
            Override take_action to handle date
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   513
        """
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   514
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   515
        if action == "parse_date" :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   516
            # get timezone
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   517
            tz = values.timezone
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   518
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   519
            # set timezone
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   520
            value = value.replace(tzinfo=tz)
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   521
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   522
            # store
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   523
            return optparse.Option.take_action(self, 'store', dest, opt, value, values, parser)
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   524
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   525
        else :
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   526
            # default
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   527
            return optparse.Option.take_action(self, action, dest, opt, value, values, parser)
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   528
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   529
    TYPES = optparse.Option.TYPES + ('date', 'timezone')
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   530
    TYPE_CHECKER = optparse.Option.TYPE_CHECKER.copy()
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   531
    TYPE_CHECKER['date'] = check_date
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   532
    TYPE_CHECKER['timezone'] = check_timezone
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   533
    ACTIONS = optparse.Option.ACTIONS + ('parse_date', )
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   534
    STORE_ACTIONS = optparse.Option.STORE_ACTIONS + ('parse_date', )
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   535
    TYPED_ACTIONS = optparse.Option.TYPED_ACTIONS + ('parse_date', )
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   536
    ACTIONS = optparse.Option.ACTIONS + ('parse_date', )
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   537
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   538
def main (argv) :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   539
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   540
        Command-line main, with given argv
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   541
    """
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   542
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   543
    # define parser
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   544
    parser = optparse.OptionParser(
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   545
        usage           = "%prog [options] <command> [ ... ]",
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   546
        add_help_option = False,
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   547
        option_class    = MyOption,
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   548
    )
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   549
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   550
    # general options       #                   #                       #                                   #
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   551
    general = optparse.OptionGroup(parser, "General Options")
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   552
    general.add_option('-h', "--help",          dest="help",            help="Show this help message and exit",     
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   553
                                                action="store_true"                                         )
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   554
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   555
    general.add_option(     "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   556
            metavar="FMT",  type="choice",                              default=config.PREF_FORMATTER_DEFAULT.name,
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   557
            choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()]                             )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   558
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   559
    general.add_option(     "--index",          dest="index_path",      help="Index database path",                 
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   560
            metavar="PATH",                                             default=config.SEARCH_INDEX_PATH    )
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   561
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   562
    general.add_option(     "--timezone",       dest="timezone",        help="Timezone for output",                 
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   563
            metavar="TZ",   type="timezone",                            default=pytz.utc                    )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   564
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   565
    general.add_option(     "--force",          dest="force",           help="Force dangerous operation",           
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   566
                                                action="store_true"                                         )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   567
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   568
    general.add_option(     "--quiet",          dest="quiet",           help="Supress status messages",             
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   569
                                                action="store_true"                                         )
104
34c65a8c8b94 split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents: 103
diff changeset
   570
    parser.add_option_group(general)
34c65a8c8b94 split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents: 103
diff changeset
   571
    
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   572
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   573
    # cmd_load options      #                   #                       #                                   #
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   574
    load = optparse.OptionGroup(parser, "Load Options")
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   575
    load.add_option(        "--skip-missing",   dest="skip_missing",    help="Skip missing logfiles",
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   576
                                                action="store_true"                                         )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   577
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   578
    load.add_option(        "--create",         dest="create",          help="Create index database", 
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   579
                                                action="store_true"                                         )
104
34c65a8c8b94 split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents: 103
diff changeset
   580
    parser.add_option_group(load)
34c65a8c8b94 split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents: 103
diff changeset
   581
    
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   582
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   583
    # cmd_autoload options  #                   #                       #                                   #
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   584
    autoload = optparse.OptionGroup(parser, "Autoload Options")
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   585
    autoload.add_option(    "--autoload-state", dest="autoload_state_path", help="Path to autoload state dir",      
121
86aebc9cb60b some quickfixes to fix deployment errors
Tero Marttila <terom@fixme.fi>
parents: 106
diff changeset
   586
            metavar="PATH",                                             default=config.SEARCH_AUTOINDEX_PATH)
106
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   587
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   588
    autoload.add_option(    "--from",           dest="after",           help="Only autoload logfiles from the given date on", 
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   589
            metavar="DATE", type="date",        action="parse_date",    default=None                        )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   590
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   591
    autoload.add_option(    "--until",          dest="until",           help="Only autoload logfiles up to (and including) the given date",  
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   592
            metavar="DATE", type="date",        action="parse_date",    default=None                        )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   593
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   594
    autoload.add_option(    "--reload",         dest="reload",          help="Force reload lines",
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   595
                                                action="store_true"                                         )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   596
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   597
    autoload.add_option(    "--reset",          dest="reset",           help="Reset old autload state",
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   598
                                                action="store_true"                                         )
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   599
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   600
    autoload.add_option(    "--ignore-resume",  dest="ignore_resume",   help="Do not try and resume interrupted autoload",  
0690d715385d reformat add_option's and rename --after -> --from, and make that and --until inclusive
Tero Marttila <terom@fixme.fi>
parents: 105
diff changeset
   601
                                                action="store_true"                                         )
104
34c65a8c8b94 split scripts/search-index options into groups
Tero Marttila <terom@fixme.fi>
parents: 103
diff changeset
   602
    parser.add_option_group(autoload)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   603
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   604
    # parse
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   605
    options, args = parser.parse_args(argv[1:])
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   606
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   607
    # postprocess stuff
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   608
    options._parser = parser
105
e24da9a94ffb implement our own optparse.Option with date/timezone types
Tero Marttila <terom@fixme.fi>
parents: 104
diff changeset
   609
    options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.timezone, "%H:%M:%S", None, None)
103
0e829e6275dc implement --until, and fix handling of ServerMode
Tero Marttila <terom@fixme.fi>
parents: 101
diff changeset
   610
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   611
    # special-case --help
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   612
    if options.help :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   613
        return cmd_help(options, *args)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   614
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   615
    # must have at least the command argument
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   616
    if not args :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   617
        raise CommandError("Missing command")
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   618
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   619
    # pop command
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   620
    command = args.pop(0)
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   621
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   622
    # get func
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   623
    func = globals().get('cmd_%s' % command)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   624
    
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   625
    # unknown command?
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   626
    if not func :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   627
        raise CommandError("Unknown command: %s" % command)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   628
    
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   629
    # call
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   630
    func(options, *args)
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   631
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   632
if __name__ == '__main__' :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   633
    try :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   634
        main(sys.argv)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   635
        sys.exit(0)
65
8b50694f841e improve search further
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
   636
88
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   637
    except CommandError, e :
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   638
        print e
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   639
        sys.exit(1)
0b8e2ba5f76f improve scripts/search-index with better help info
Tero Marttila <terom@fixme.fi>
parents: 83
diff changeset
   640