1

#!/usr/bin/env python2.5

     2

     3

"""

     4

    Tool for accessing the search index

     5

"""

     6

     7

# XXX: fix path

     8

import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')

     9

    10

import os, os.path, fcntl

    11

import datetime, pytz

    12

    13

# configuration and the LogSearchIndex module

    14

import config, utils, log_search, channels

    15

    16

def _open_index (options, open_mode) :

    17

    """

    18

        Opens the LogSearchIndex

    19

    """

    20

    21

    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)

    22

    23

    24

def _open_index_and_channel (options, channel_name, open_mode) :

    25

    """

    26

        Opens+returns a LogSearchIndex and a LogChannel

    27

    """

    28

    29

    # open the LogSearchIndex

    30

    index = _open_index(options, open_mode)

    31

    32

    # open the channel

    33

    channel = config.LOG_CHANNELS.lookup(channel_name)

    34

    35

    # return

    36

    return index, channel

    37

    38

def _iter_insert_stats (index, channel, lines) :

    39

    """

    40

        Insert the given lines into the index.

    41

    42

        Assumes the lines will be in time-order, and yields a series of (date, count) tuples for every date that lines

    43

        are inserted for

    44

    """

    45

    46

    # last date

    47

    date = None

    48

    49

    # count

    50

    count = 0

    51

    52

    # iter lines

    53

    for line in lines :

    54

        # next day?

    55

        if not date or line.timestamp.date() != date :

    56

            if date :

    57

                # yield stats

    58

                yield date, count

    59

    60

            # reset count

    61

            count = 0

    62

    63

            # timestamp's date

    64

            date = line.timestamp.date()

    65

    66

        # insert

    67

        index.insert_line(channel, line)

    68

    69

        # count

    70

        count += 1

    71

    72

    # final count?

    73

    if date and count :

    74

        yield date, count

    75

    76

def _insert_lines (index, options, channel, lines) :

    77

    """

    78

        Insert the given lines into the index.

    79

    80

        Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines

    81

    """

    82

    83

    # iterate insert stats

    84

    for date, count in _iter_insert_stats(index, channel, lines) :

    85

        # output date header?

    86

        if not options.quiet :

    87

            print "%s: %s" % (date.strftime('%Y-%m-%d'), count),

    88

    89

def _load_channel_date (index, options, channel, date) :

    90

    """

    91

        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex

    92

    """

    93

    94

    if not options.quiet :

    95

        print "Loading date for channel %s" % channel.id

    96

    97

    try :

    98

        # load lines for date

    99

        lines = channel.source.get_date(date)

   100

   101

    except Exception, e :

   102

        if not options.skip_missing :

   103

            raise

   104

   105

        if not options.quiet :

   106

            print "\tSkipped: %s" % (e, )

   107

   108

    else :

   109

        # insert

   110

        _insert_lines(index, options, channel, lines)

   111

   112

def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') :

   113

    """

   114

        Parse the given datetime, using the given timezone(defaults to options.tz) and format

   115

    """

   116

   117

    # default tz

   118

    if not tz :

   119

        tz = options.tz

   120

   121

    try :

   122

        # parse

   123

        return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)

   124

   125

    except Exception, e :

   126

        raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e))

   127

   128

def _output_lines (options, lines) :

   129

    """

   130

        Display the formatted LogLines

   131

    """

   132

   133

    # display as plaintext

   134

    for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) :

   135

        print txt_data

   136

   137

class CommandError (Exception) :

   138

    """

   139

        Error with command-line arguments

   140

    """

   141

   142

    pass

   143

   144

def cmd_create (options) :

   145

    """

   146

        Creates a new index

   147

    """

   148

   149

    # open index

   150

    index = _open_index(options, 'ctrunc' if options.force else 'c')

   151

   152

    # that's all

   153

    pass

   154

   155

def cmd_load (options, channel_name, *dates) :

   156

    """

   157

        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index

   158

    """

   159

   160

    # open index/channel

   161

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   162

   163

    # handle each date

   164

    for date_str in dates :

   165

        # prase date

   166

        try :

   167

            date = _parse_date(options, date_str, channel.source.tz)

   168

   169

        # handle errors

   170

        except CommandError, e :

   171

            if options.skip_missing :

   172

                print "[ERROR] %s" % (date_name, e)

   173

   174

            else :

   175

                raise

   176

   177

        # otherwise, load

   178

        else :        

   179

            _load_channel_date(index, options, channel, date)

   180

   181

def cmd_load_month (options, channel_name, *months) :

   182

    """

   183

        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index

   184

    """

   185

   186

    # open index/channel

   187

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   188

   189

    # handle each date

   190

    for month_str in months :

   191

        # prase date

   192

        try :

   193

            month = _parse_date(options, month_str, channel.source.tz, '%Y-%m')

   194

   195

        # handle errors

   196

        except CommandError, e :

   197

            # skip?

   198

            if options.skip_missing :

   199

                print "[ERROR] %s" % (date_name, e)

   200

                continue

   201

   202

            else :

   203

                raise

   204

   205

        # get the set of days

   206

        days = list(channel.source.get_month_days(month))

   207

   208

        print "Loading %d days of logs:" % (len(days))

   209

   210

        # load each day

   211

        for date in days :

   212

            # convert to datetime

   213

            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)

   214

   215

            # load

   216

            _load_channel_date(index, options, channel, dt)

   217

   218

def cmd_search (options, channel_name, query) :

   219

    """

   220

        Search the index for events on a specific channel with the given query

   221

    """

   222

   223

    # sanity-check

   224

    if options.create :

   225

        raise Exception("--create doesn't make sense for 'search'")

   226

   227

    # open index/channel

   228

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   229

   230

    # search

   231

    lines = index.search_simple(channel, query)

   232

   233

    # display

   234

    _output_lines(options, lines)

   235

   236

def cmd_list (options, channel_name, *dates) :

   237

    """

   238

        List the indexed events for a specific date

   239

    """

   240

   241

    # sanity-check

   242

    if options.create :

   243

        raise Exception("--create doesn't make sense for 'search'")

   244

   245

    # open index/channel

   246

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   247

   248

    # ...for each date

   249

    for date_str in dates :

   250

        # parse date

   251

        date = _parse_date(options, date_str)

   252

   253

        # list

   254

        lines = index.list(channel, date)

   255

   256

        # display

   257

        _output_lines(options, lines)

   258

   259

def _autoload_reset (options, channels) :

   260

    """

   261

        Reset old autoload state

   262

    """

   263

   264

    # warn

   265

    if not options.quiet :

   266

        print "[WARN] Resetting autoload state for: %s" % ', '.join(channel.id for channel in channels)

   267

   268

    # iter

   269

    for channel in channels :

   270

        # statefile path

   271

        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)

   272

   273

        # is it present?

   274

        if not os.path.exists(statefile_path) :

   275

            if not options.quiet :

   276

                print "[WARN] No statefile found at %s" % statefile_path

   277

   278

        else :

   279

            if not options.quiet :

   280

                print "\t%s: " % channel.id,

   281

   282

            # remove the statefile

   283

            os.remove(statefile_path)

   284

   285

            if not options.quiet :

   286

                print "OK"

   287

   288

def cmd_autoload (options, *channel_names) :

   289

    """

   290

        Automatically loads all channel logs that have not been indexed yet (by logfile mtime)

   291

    """

   292

   293

    # open index, nonblocking

   294

    index = _open_index(options, 'c?' if options.create else 'a?')

   295

   296

    # default to all channels

   297

    if not channel_names :

   298

        channels = config.LOG_CHANNELS

   299

   300

    else :

   301

        channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names]

   302

   303

    # reset autoload state?

   304

    if options.reset :

   305

        _autoload_reset(options, channels)

   306

        print

   307

   308

    # iterate channels

   309

    for channel in channels :

   310

        if not options.quiet :

   311

            print "Channel %s:" % channel.id

   312

   313

        # no 'after' by default

   314

        after = None

   315

   316

        # path to our state file

   317

        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)

   318

        statefile_tmppath = statefile_path + '.tmp'

   319

   320

        # does it exist?

   321

        have_tmpfile = os.path.exists(statefile_tmppath)

   322

   323

        # do we have a tempfile from a previous crash?

   324

        if have_tmpfile and not options.ignore_resume :

   325

            # first, open it...

   326

            statefile_tmp = open(statefile_tmppath, 'r+')

   327

   328

            # ... then lock it

   329

            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)

   330

   331

            # read after timestamp

   332

            after_str = statefile_tmp.read().rstrip()

   333

   334

            if after_str :

   335

                # parse timestamp

   336

                after = utils.from_utc_timestamp(int(after_str))

   337

   338

                # one day backwards

   339

                after = after.replace(day=after.day - 1)

   340

   341

                if not options.quiet :

   342

                    print "\tContinuing earlier progress: after=%s" % after

   343

   344

            else :

   345

                # ignore

   346

                if not options.quiet :

   347

                    print "\t[WARN] Ignoring empty temporary statefile"

   348

   349

        else :

   350

            # warn about old tmpfile that was ignored

   351

            if have_tmpfile and not options.quiet :

   352

                print "\t[WARN] Ignoring old tmpfile state"

   353

   354

            # open new tempfile

   355

            statefile_tmp = open(statefile_tmppath, 'w')

   356

   357

            # lock

   358

            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)

   359

   360

        # override?

   361

        if options.reload :

   362

            # load all

   363

            mtime = None

   364

   365

            if not options.quiet :

   366

                print "\tForcing reload!"

   367

   368

        # stat for mtime

   369

        else :

   370

            # stat for mtime, None if unknown

   371

            mtime = utils.mtime(statefile_path, ignore_missing=True)

   372

   373

            if mtime and not options.quiet :

   374

                print "\tLast load time was %s" % mtime

   375

   376

            elif not options.quiet :

   377

                print "\t[WARN] No previous load state! Loading full logs"

   378

   379

        # only after some specific date?

   380

        if options.after :

   381

            # use unless read from tempfile

   382

            if not after :

   383

                after = options.after

   384

   385

                if not options.quiet :

   386

                    print "\tOnly including dates after %s" % after

   387

   388

            else :

   389

                if not options.quiet :

   390

                    print "\t[WARN] Ignoring --after because we found a tempfile"

   391

   392

        # only up to some specific date?

   393

        if options.until :

   394

            until = options.until

   395

   396

            if not options.quiet :

   397

                print "\tOnly including dates up to %s" % until

   398

        else :

   399

            # default to now

   400

            until = None

   401

   402

        # get lines

   403

        lines = channel.source.get_modified(mtime, after, until)

   404

   405

        # insert

   406

        if not options.quiet :

   407

            print "\tLoading and inserting..."

   408

            print

   409

   410

        # iterate insert() per day to display info and update progress

   411

        for date, count in _iter_insert_stats(index, channel, lines) :

   412

            # output date header?

   413

            if not options.quiet :

   414

                print "\t%10s: %d" % (date.strftime('%Y-%m-%d'), count)

   415

   416

            # write temp state

   417

            statefile_tmp.seek(0)

   418

            statefile_tmp.write(str(utils.to_utc_timestamp(datetime.datetime.combine(date, datetime.time(0)))))

   419

            statefile_tmp.flush()

   420

   421

        # write autoload state

   422

        open(statefile_path, 'w').close()

   423

   424

        # close+delete tempfile

   425

        statefile_tmp.close()

   426

        os.remove(statefile_tmppath)

   427

   428

        print

   429

   430

    # done

   431

    return

   432

   433

def cmd_help (options, *args) :

   434

    """

   435

        Help about commands

   436

    """

   437

   438

    import inspect

   439

   440

    # general help stuff

   441

    options._parser.print_help()

   442

   443

    # specific command?

   444

    if args :

   445

        # the command name

   446

        command, = args

   447

   448

        # XXX: display info about specific command

   449

        xxx

   450

   451

    # general

   452

    else :

   453

        print

   454

        print "Available commands:"

   455

   456

        # build list of all cmd_* objects

   457

        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]

   458

   459

        # sort alphabetically

   460

        cmd_objects.sort()

   461

   462

        # iterate through all cmd_* objects

   463

        for cmd_func_name, cmd_func in cmd_objects :

   464

            # remove cmd_ prefix

   465

            cmd_name = cmd_func_name[4:]

   466

   467

            # inspect

   468

            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)

   469

            cmd_doc = inspect.getdoc(cmd_func)

   470

   471

            # remove the "options" arg

   472

            cmd_args = cmd_args[1:]

   473

   474

            # display

   475

            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)

   476

   477

def main (argv) :

   478

    """

   479

        Command-line main, with given argv

   480

    """

   481

   483

   484

    # define parser

   485

    parser = OptionParser(

   486

        usage           = "%prog [options] <command> [ ... ]",

   487

        add_help_option = False,

   488

    )

   489

   490

    # define command-line arguments