1

#!/usr/bin/env python2.5

     2

     3

"""

     4

    Tool for accessing the search index

     5

"""

     6

     7

# XXX: fix path

     8

import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')

     9

    10

import os, os.path

    11

import datetime, pytz

    12

    13

# configuration and the LogSearchIndex module

    14

import config, utils, log_search, channels

    15

    16

def _open_index (options, open_mode) :

    17

    """

    18

        Opens the LogSearchIndex

    19

    """

    20

    21

    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)

    22

    23

    24

def _open_index_and_channel (options, channel_name, open_mode) :

    25

    """

    26

        Opens+returns a LogSearchIndex and a LogChannel

    27

    """

    28

    29

    # open the LogSearchIndex

    30

    index = _open_index(options, open_mode)

    31

    32

    # open the channel

    33

    channel = config.LOG_CHANNELS.lookup(channel_name)

    34

    35

    # return

    36

    return index, channel

    37

    38

def _insert_lines (index, options, channel, lines) :

    39

    """

    40

        Insert the given lines into the index.

    41

    42

        Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines

    43

    """

    44

    45

    # last date

    46

    date = None

    47

    48

    # count

    49

    count = 0

    50

    51

    # iter lines

    52

    for line in lines :

    53

        # output new date header?

    54

        if not options.quiet and (not date or line.timestamp.date() != date) :

    55

            # previous date's line count?

    56

            if date :

    57

                print "OK: %d lines" % count

    58

    59

            # reset count

    60

            count = 0

    61

    62

            # timestamp's date

    63

            date = line.timestamp.date()

    64

    65

            # status header

    66

            print "%s:" % (date.strftime('%Y-%m-%d'), ),

    67

    68

        # insert

    69

        index.insert_line(channel, line)

    70

    71

        # count

    72

        count += 1

    73

    74

    # final count line

    75

    if not options.quiet and date :

    76

        print "OK: %d lines" % count

    77

    78

def _load_channel_date (index, options, channel, date) :

    79

    """

    80

        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex

    81

    """

    82

    83

    if not options.quiet :

    84

        print "Loading date for channel %s" % channel.id

    85

    86

    try :

    87

        # load lines for date

    88

        lines = channel.source.get_date(date)

    89

    90

    except Exception, e :

    91

        if not options.skip_missing :

    92

            raise

    93

    94

        if not options.quiet :

    95

            print "\tSkipped: %s" % (e, )

    96

    97

    else :

    98

        # insert

    99

        _insert_lines(index, options, channel, lines)

   100

   101

def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') :

   102

    """

   103

        Parse the given datetime, using the given timezone(defaults to options.tz) and format

   104

    """

   105

   106

    # default tz

   107

    if not tz :

   108

        tz = options.tz

   109

   110

    try :

   111

        # parse

   112

        return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)

   113

   114

    except Exception, e :

   115

        raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e))

   116

   117

def _output_lines (options, lines) :

   118

    """

   119

        Display the formatted LogLines

   120

    """

   121

   122

    # display as plaintext

   123

    for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) :

   124

        print txt_data

   125

   126

class CommandError (Exception) :

   127

    """

   128

        Error with command-line arguments

   129

    """

   130

   131

    pass

   132

   133

def cmd_create (options) :

   134

    """

   135

        Creates a new index

   136

    """

   137

   138

    # open index

   139

    index = _open_index(options, 'c' if not options.force else '*')

   140

   141

    # that's all

   142

    pass

   143

   144

def cmd_load (options, channel_name, *dates) :

   145

    """

   146

        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index

   147

    """

   148

   149

    # open index/channel

   150

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   151

   152

    # handle each date

   153

    for date_str in dates :

   154

        # prase date

   155

        try :

   156

            date = _parse_date(options, date_str, channel.source.tz)

   157

   158

        # handle errors

   159

        except CommandError, e :

   160

            if options.skip_missing :

   161

                print "[ERROR] %s" % (date_name, e)

   162

   163

            else :

   164

                raise

   165

   166

        # otherwise, load

   167

        else :        

   168

            _load_channel_date(index, options, channel, date)

   169

   170

def cmd_load_month (options, channel_name, *months) :

   171

    """

   172

        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index

   173

    """

   174

   175

    # open index/channel

   176

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   177

   178

    # handle each date

   179

    for month_str in months :

   180

        # prase date

   181

        try :

   182

            month = _parse_date(options, month_str, channel.source.tz, '%Y-%m')

   183

   184

        # handle errors

   185

        except CommandError, e :

   186

            # skip?

   187

            if options.skip_missing :

   188

                print "[ERROR] %s" % (date_name, e)

   189

                continue

   190

   191

            else :

   192

                raise

   193

   194

        # get the set of days

   195

        days = list(channel.source.get_month_days(month))

   196

   197

        print "Loading %d days of logs:" % (len(days))

   198

   199

        # load each day

   200

        for date in days :

   201

            # convert to datetime

   202

            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)

   203

   204

            # load

   205

            _load_channel_date(index, options, channel, dt)

   206

   207

def cmd_search (options, channel_name, query) :

   208

    """

   209

        Search the index for events on a specific channel with the given query

   210

    """

   211

   212

    # sanity-check

   213

    if options.create :

   214

        raise Exception("--create doesn't make sense for 'search'")

   215

   216

    # open index/channel

   217

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   218

   219

    # search

   220

    lines = index.search_simple(channel, query)

   221

   222

    # display

   223

    _output_lines(options, lines)

   224

   225

def cmd_list (options, channel_name, *dates) :

   226

    """

   227

        List the indexed events for a specific date

   228

    """

   229

   230

    # sanity-check

   231

    if options.create :

   232

        raise Exception("--create doesn't make sense for 'search'")

   233

   234

    # open index/channel

   235

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   236

   237

    # ...for each date

   238

    for date_str in dates :

   239

        # parse date

   240

        date = _parse_date(options, date_str)

   241

   242

        # list

   243

        lines = index.list(channel, date)

   244

   245

        # display

   246

        _output_lines(options, lines)

   247

   248

def cmd_autoload (options, *channel_names) :

   249

    """

   250

        Automatically loads all channel logs that have not been indexed yet (by logfile mtime)

   251

    """

   252

   253

    # open index

   254

    index = _open_index(options, 'c' if options.create else 'a')

   255

   256

    # default to all channels

   257

    if not channel_names :

   258

        channels = config.LOG_CHANNELS

   259

   260

    else :

   261

        channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names]

   262

   263

    # iterate channels

   264

    for channel in channels :

   265

        if not options.quiet :

   266

            print "Channel %s:" % channel.id,

   267

   268

        # path to our state file

   269

        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)

   271

        # override?

   272

        if options.reload :

   273

            # load all

   274

            mtime = None

   275

   276

            if not options.quiet :

   277

                print "reloading all:",

   278

   279

        # stat for mtime

   280

        # XXX: replace with single utils.mtime()

   281

        elif os.path.exists(statefile_path) :

   282

            # get last update date for channel

   283

            mtime = utils.from_utc_timestamp(os.stat(statefile_path).st_mtime)

   284

   285

            if not options.quiet :

   286

                print "last load=%s:" % mtime,

   287

   288

        else :

   289

            # unknown, load all

   290

            mtime = None

   291

   292

            if not options.quiet :

   293

                print "no previous load state:",

   303

   304

        # get lines

   306

   307

        # insert

   308

        if not options.quiet :

   309

            print "inserting..."

   310

   311

        _insert_lines(index, options, channel, lines)

   312

   313

        # write autoload state

   314

        open(statefile_path, 'w').close()

   315

   316

def cmd_help (options, *args) :

   317

    """

   318

        Help about commands

   319

    """

   320

   321

    import inspect

   322

   323

    # general help stuff

   324

    options._parser.print_help()

   325

   326

    # specific command?

   327

    if args :

   328

        # the command name

   329

        command, = args

   330

   331

        # XXX: display info about specific command

   332

        xxx

   333

   334

    # general

   335

    else :

   336

        print

   337

        print "Available commands:"

   338

   339

        # build list of all cmd_* objects

   340

        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]

   341

   342

        # sort alphabetically

   343

        cmd_objects.sort()

   344

   345

        # iterate through all cmd_* objects

   346

        for cmd_func_name, cmd_func in cmd_objects :

   347

            # remove cmd_ prefix

   348

            cmd_name = cmd_func_name[4:]

   349

   350

            # inspect

   351

            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)

   352

            cmd_doc = inspect.getdoc(cmd_func)

   353

   354

            # remove the "options" arg

   355

            cmd_args = cmd_args[1:]

   356

   357

            # display

   358

            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)

   359

   360

def main (argv) :

   361

    """

   362

        Command-line main, with given argv

   363

    """

   364

   365

    from optparse import OptionParser

   366

   367

    # define parser

   368

    parser = OptionParser(

   369

        usage           = "%prog [options] <command> [ ... ]",

   370

        add_help_option = False,

   371

    )

   372

   373

    # define command-line arguments

   375

    parser.add_option('-F', "--formatter",      dest="formatter_name",  help="LogFormatter to use",                 metavar="FMT",  type="choice", default="irssi",

   376

        choices=[fmt_name for fmt_name in config.LOG_FORMATTERS.iterkeys()])

   377

   387

   388

    # parse

   389

    options, args = parser.parse_args(argv[1:])

   390

   391

    # postprocess stuff

   392

    options._parser = parser

   393

    options.tz = pytz.timezone(options.tz_name)

   394

    options.formatter = config.LOG_FORMATTERS[options.formatter_name](options.tz, "%H:%M:%S", None, None)

   395

   399

    # special-case --help

   400

    if options.help :

   401

        return cmd_help(options, *args)

   402

   403

    # must have at least the command argument

   404

    if not args :

   405

        raise CommandError("Missing command")

   406

   407

    # pop command

   408

    command = args.pop(0)

   409

   410

    # get func

   411

    func = globals().get('cmd_%s' % command)

   412

   413

    # unknown command?

   414

    if not func :

   415

        raise CommandError("Unknown command: %s" % command)

   416

   417

    # call

   418

    func(options, *args)

   419

   420

if __name__ == '__main__' :

   421

    try :

   422

        main(sys.argv)

   423

        sys.exit(0)

   424

   425

    except CommandError, e :

   426

        print e

   427

        sys.exit(1)

   428