1

#!/usr/bin/env python2.5

     2

     3

"""

     4

    Tool for accessing the search index

     5

"""

     6

     7

# XXX: fix path

     8

import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')

     9

    10

import os, os.path, fcntl

    11

import datetime, pytz

    12

import optparse

    13

    14

# configuration and the LogSearchIndex module

    15

import config, utils, log_search, channels

    16

    17

def _open_index (options, open_mode) :

    18

    """

    19

        Opens the LogSearchIndex

    20

    """

    21

    22

    return log_search.LogSearchIndex(config.LOG_CHANNELS, options.index_path, open_mode)

    23

    24

    25

def _open_index_and_channel (options, channel_name, open_mode) :

    26

    """

    27

        Opens+returns a LogSearchIndex and a LogChannel

    28

    """

    29

    30

    # open the LogSearchIndex

    31

    index = _open_index(options, open_mode)

    32

    33

    # open the channel

    34

    channel = config.LOG_CHANNELS.lookup(channel_name)

    35

    36

    # return

    37

    return index, channel

    38

    39

def _iter_insert_stats (index, channel, lines) :

    40

    """

    41

        Insert the given lines into the index.

    42

    43

        Assumes the lines will be in time-order, and yields a series of (date, count) tuples for every date that lines

    44

        are inserted for

    45

    """

    46

    47

    # last date

    48

    date = None

    49

    50

    # count

    51

    count = 0

    52

    53

    # iter lines

    54

    for line in lines :

    55

        # next day?

    56

        if not date or line.timestamp.date() != date :

    57

            if date :

    58

                # yield stats

    59

                yield date, count

    60

    61

            # reset count

    62

            count = 0

    63

    64

            # timestamp's date

    65

            date = line.timestamp.date()

    66

    67

        # insert

    68

        index.insert_line(channel, line)

    69

    70

        # count

    71

        count += 1

    72

    73

    # final count?

    74

    if date and count :

    75

        yield date, count

    76

    77

def _insert_lines (index, options, channel, lines) :

    78

    """

    79

        Insert the given lines into the index.

    80

    81

        Assumes the lines will be in time-order, and prints out as status messages the date and count for the inserted lines

    82

    """

    83

    84

    # iterate insert stats

    85

    for date, count in _iter_insert_stats(index, channel, lines) :

    86

        # output date header?

    87

        if not options.quiet :

    88

            print "%s: %s" % (date.strftime('%Y-%m-%d'), count),

    89

    90

def _load_channel_date (index, options, channel, date) :

    91

    """

    92

        Loads the logs for the given date from the channel's LogSource into the given LogSearchIndex

    93

    """

    94

    95

    if not options.quiet :

    96

        print "Loading date for channel %s" % channel.id

    97

    98

    try :

    99

        # load lines for date

   100

        lines = channel.source.get_date(date)

   101

   102

    except Exception, e :

   103

        if not options.skip_missing :

   104

            raise

   105

   106

        if not options.quiet :

   107

            print "\tSkipped: %s" % (e, )

   108

   109

    else :

   110

        # insert

   111

        _insert_lines(index, options, channel, lines)

   112

   113

def _parse_date (options, date_str, tz=None, fmt='%Y-%m-%d') :

   114

    """

   115

        Parse the given datetime, using the given timezone(defaults to options.tz) and format

   116

    """

   117

   118

    # default tz

   119

    if not tz :

   120

        tz = options.timezone

   121

   122

    try :

   123

        # parse

   124

        return datetime.datetime.strptime(date_str, fmt).replace(tzinfo=tz)

   125

   126

    except Exception, e :

   127

        raise CommandError("[ERROR] Invalid date: %s: %s" % (date_str, e))

   128

   129

def _output_lines (options, lines) :

   130

    """

   131

        Display the formatted LogLines

   132

    """

   133

   134

    # display as plaintext

   135

    for line, txt_data in options.formatter.format_txt(lines, full_timestamps=True) :

   136

        print txt_data

   137

   138

class CommandError (Exception) :

   139

    """

   140

        Error with command-line arguments

   141

    """

   142

   143

    pass

   144

   145

def cmd_create (options) :

   146

    """

   147

        Creates a new index

   148

    """

   149

   150

    # open index

   151

    index = _open_index(options, 'ctrunc' if options.force else 'c')

   152

   153

    # that's all

   154

    pass

   155

   156

def cmd_load (options, channel_name, *dates) :

   157

    """

   158

        Loads the logs for a specific channel for the given dates (in terms of the channe logs' timezone) into the index

   159

    """

   160

   161

    # open index/channel

   162

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   163

   164

    # handle each date

   165

    for date_str in dates :

   166

        # prase date

   167

        try :

   168

            date = _parse_date(options, date_str, channel.source.tz)

   169

   170

        # handle errors

   171

        except CommandError, e :

   172

            if options.skip_missing :

   173

                print "[ERROR] %s" % (date_name, e)

   174

   175

            else :

   176

                raise

   177

   178

        # otherwise, load

   179

        else :        

   180

            _load_channel_date(index, options, channel, date)

   181

   182

def cmd_load_month (options, channel_name, *months) :

   183

    """

   184

        Loads the logs for a specific channel for the given months (in terms of the channel's timezone) into the index

   185

    """

   186

   187

    # open index/channel

   188

    index, channel = _open_index_and_channel(options, channel_name, 'c' if options.create else 'a')

   189

   190

    # handle each date

   191

    for month_str in months :

   192

        # prase date

   193

        try :

   194

            month = _parse_date(options, month_str, channel.source.tz, '%Y-%m')

   195

   196

        # handle errors

   197

        except CommandError, e :

   198

            # skip?

   199

            if options.skip_missing :

   200

                print "[ERROR] %s" % (date_name, e)

   201

                continue

   202

   203

            else :

   204

                raise

   205

   206

        # get the set of days

   207

        days = list(channel.source.get_month_days(month))

   208

   209

        print "Loading %d days of logs:" % (len(days))

   210

   211

        # load each day

   212

        for date in days :

   213

            # convert to datetime

   214

            dt = datetime.datetime.combine(date, datetime.time(0)).replace(tzinfo=channel.source.tz)

   215

   216

            # load

   217

            _load_channel_date(index, options, channel, dt)

   218

   219

def cmd_search (options, channel_name, query) :

   220

    """

   221

        Search the index for events on a specific channel with the given query

   222

    """

   223

   224

    # sanity-check

   225

    if options.create :

   226

        raise Exception("--create doesn't make sense for 'search'")

   227

   228

    # open index/channel

   229

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   230

   231

    # search

   232

    lines = index.search_simple(channel, query)

   233

   234

    # display

   235

    _output_lines(options, lines)

   236

   237

def cmd_list (options, channel_name, *dates) :

   238

    """

   239

        List the indexed events for a specific date

   240

    """

   241

   242

    # sanity-check

   243

    if options.create :

   244

        raise Exception("--create doesn't make sense for 'search'")

   245

   246

    # open index/channel

   247

    index, channel = _open_index_and_channel(options, channel_name, 'r')

   248

   249

    # ...for each date

   250

    for date_str in dates :

   251

        # parse date

   252

        date = _parse_date(options, date_str)

   253

   254

        # list

   255

        lines = index.list(channel, date)

   256

   257

        # display

   258

        _output_lines(options, lines)

   259

   260

def _autoload_reset (options, channels) :

   261

    """

   262

        Reset old autoload state

   263

    """

   264

   265

    # warn

   266

    if not options.quiet :

   267

        print "[WARN] Resetting autoload state for: %s" % ', '.join(channel.id for channel in channels)

   268

   269

    # iter

   270

    for channel in channels :

   271

        # statefile path

   272

        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)

   273

   274

        # is it present?

   275

        if not os.path.exists(statefile_path) :

   276

            if not options.quiet :

   277

                print "[WARN] No statefile found at %s" % statefile_path

   278

   279

        else :

   280

            if not options.quiet :

   281

                print "\t%s: " % channel.id,

   282

   283

            # remove the statefile

   284

            os.remove(statefile_path)

   285

   286

            if not options.quiet :

   287

                print "OK"

   288

   289

def cmd_autoload (options, *channel_names) :

   290

    """

   291

        Automatically loads all channel logs that have not been indexed yet (by logfile mtime)

   292

    """

   293

   294

    # open index, nonblocking

   295

    index = _open_index(options, 'c?' if options.create else 'a?')

   296

   297

    # default to all channels

   298

    if not channel_names :

   299

        channels = config.LOG_CHANNELS

   300

   301

    else :

   302

        channels = [config.LOG_CHANNELS.lookup(channel_name) for channel_name in channel_names]

   303

   304

    # reset autoload state?

   305

    if options.reset :

   306

        _autoload_reset(options, channels)

   307

        print

   308

   309

    # iterate channels

   310

    for channel in channels :

   311

        if not options.quiet :

   312

            print "Channel %s:" % channel.id

   313

   315

        after = None

   316

   317

        # path to our state file

   318

        statefile_path = os.path.join(options.autoload_state_path, 'chan-%s' % channel.id)

   319

        statefile_tmppath = statefile_path + '.tmp'

   320

   321

        # does it exist?

   322

        have_tmpfile = os.path.exists(statefile_tmppath)

   323

   324

        # do we have a tempfile from a previous crash?

   325

        if have_tmpfile and not options.ignore_resume :

   326

            # first, open it...

   327

            statefile_tmp = open(statefile_tmppath, 'r+')

   328

   329

            # ... then lock it

   330

            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)

   331

   332

            # read after timestamp

   333

            after_str = statefile_tmp.read().rstrip()

   334

   335

            if after_str :

   336

                # parse timestamp

   337

                after = utils.from_utc_timestamp(int(after_str))

   338

   339

                if not options.quiet :

   341

   342

            else :

   343

                # ignore

   344

                if not options.quiet :

   345

                    print "\t[WARN] Ignoring empty temporary statefile"

   346

   347

        else :

   348

            # warn about old tmpfile that was ignored

   349

            if have_tmpfile and not options.quiet :

   350

                print "\t[WARN] Ignoring old tmpfile state"

   351

   352

            # open new tempfile

   353

            statefile_tmp = open(statefile_tmppath, 'w')

   354

   355

            # lock

   356

            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)

   357

   358

        # override?

   359

        if options.reload :

   360

            # load all

   361

            mtime = None

   362

   363

            if not options.quiet :

   364

                print "\tForcing reload!"

   365

   366

        # stat for mtime

   367

        else :

   368

            # stat for mtime, None if unknown

   369

            mtime = utils.mtime(statefile_path, ignore_missing=True)

   370

   371

            if mtime and not options.quiet :

   372

                print "\tLast load time was %s" % mtime

   373

   374

            elif not options.quiet :

   375

                print "\t[WARN] No previous load state! Loading full logs"

   376

   377

        # only after some specific date?

   378

        if options.after :

   379

            # use unless read from tempfile

   380

            if not after :

   381

                after = options.after

   382

   383

                if not options.quiet :

   385

   386

            else :

   387

                if not options.quiet :

   389

   390

        # only up to some specific date?

   391

        if options.until :

   392

            until = options.until

   393

   394

            if not options.quiet :

   396

        else :

   397

            # default to now

   398

            until = None

   399

   400

        # get lines

   401

        lines = channel.source.get_modified(mtime, after, until)

   402

   403

        # insert

   404

        if not options.quiet :

   405

            print "\tLoading and inserting..."

   406

            print

   407

   408

        # iterate insert() per day to display info and update progress

   409

        for date, count in _iter_insert_stats(index, channel, lines) :

   410

            # output date header?

   411

            if not options.quiet :

   412

                print "\t%10s: %d" % (date.strftime('%Y-%m-%d'), count)

   413

   414

            # write temp state

   415

            statefile_tmp.seek(0)

   416

            statefile_tmp.write(str(utils.to_utc_timestamp(datetime.datetime.combine(date, datetime.time(0)))))

   417

            statefile_tmp.flush()

   418

   419

        # write autoload state

   420

        open(statefile_path, 'w').close()

   421

   422

        # close+delete tempfile

   423

        statefile_tmp.close()

   424

        os.remove(statefile_tmppath)

   425

   426

        print

   427

   428

    # done

   429

    return

   430

   431

def cmd_help (options, *args) :

   432

    """

   433

        Help about commands

   434

    """

   435

   436

    import inspect

   437

   438

    # general help stuff

   439

    options._parser.print_help()

   440

   441

    # specific command?

   442

    if args :

   443

        # the command name

   444

        command, = args

   445

   446

        # XXX: display info about specific command

   447

        xxx

   448

   449

    # general

   450

    else :

   451

        print

   452

        print "Available commands:"

   453

   454

        # build list of all cmd_* objects

   455

        cmd_objects = [(name, obj) for name, obj in globals().iteritems() if name.startswith('cmd_') and inspect.isfunction(obj)]

   456

   457

        # sort alphabetically

   458

        cmd_objects.sort()

   459

   460

        # iterate through all cmd_* objects

   461

        for cmd_func_name, cmd_func in cmd_objects :

   462

            # remove cmd_ prefix

   463

            cmd_name = cmd_func_name[4:]

   464

   465

            # inspect

   466

            cmd_args, cmd_varargs, cmd_varkw, cmd_default = inspect.getargspec(cmd_func)

   467

            cmd_doc = inspect.getdoc(cmd_func)

   468

   469

            # remove the "options" arg

   470

            cmd_args = cmd_args[1:]

   471

   472

            # display

   473

            print "\t%10s %-30s : %s" % (cmd_name, inspect.formatargspec(cmd_args, cmd_varargs, None, cmd_default), cmd_doc)

   474

   475

class MyOption (optparse.Option) :

   476

    """

   477

        Our custom types for optparse

   478

    """

   479

   480

    def check_date (option, opt, value) :

   481

        """

   482

            Parse a date

   483

        """

   484

   485

        try :

   486

            # parse

   487

            return datetime.datetime.strptime(value, '%Y-%m-%d')

   488

   489

        # trap -> OptionValueError

   490

        except Exception, e :