1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, os.path, errno

     7

import pytz

     8

     9

import config, utils

    10

    14

class LogSourceDecoder (object) :

    15

    """

    16

        Handles decoding of LogSource lines

    17

    """

    18

    19

    def __init__ (self, encoding_list) :

    20

        """

    21

            Will try each of the given (charset, errors) items in turn, until one succeeds

    22

        """

    23

    24

        self.encoding_list = encoding_list

    25

    26

    def decode (self, line) :

    27

        """

    28

            Decode the line of str() text into an unicode object

    29

        """

    30

    31

        # list of errors encountered

    32

        error_list = []

    33

    34

        # try each in turn

    35

        for charset, errors in self.encoding_list :

    36

            # trap UnicodeDecodeError to try with the next one

    37

            try :

    38

                return line.decode(charset, errors)

    39

    40

            except UnicodeDecodeError, e :

    41

                error_list.append("%s:%s - %s" % (charset, errors, e))

    42

                continue

    43

    44

        # failure

    45

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    46

    47

class LogSource (object) :

    48

    """

    49

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    50

    """

    51

    52

    def __init__ (self, decoder, channel=None) :

    53

        """

    54

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    55

            known, then it can be given as None, and set later with bind_channel.

    56

    57

            Uses the given LogSourceDecoder to decode the lines.

    58

        """

    59

    60

        self.channel = channel

    61

        self.decoder = decoder

    62

    63

    def bind_channel (self, channel) :

    64

        """

    65

            Set this source's channel, where None was set before

    66

        """

    67

    68

        assert not self.channel

    69

    70

        self.channel = channel

    71

    72

    def get_latest (self, count) :

    73

        """

    74

            Yield the latest events, up to `count` of them.

    75

        """

    76

    77

        abstract

    78

    79

    def get_date (self, dt) :

    80

        """

    81

            Get logs for the given date (as a datetime).

    82

        """

    83

    84

        abstract

    85

    86

    def get_date_paged (self, dt, count, page=None) :

    87

        """

    88

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    89

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    90

            then the lines for the page containing the given timestamp is returned.

    91

    92

            The return value is a (page, max, lines) tuple.

    93

        """

    94

    95

        # how to act?

    96

        if page :

    97

            # constant skip

    98

            skip = (page - 1) * count

    99

   100

        else :

   101

            skip = None

   102

   103

            # go through the logs a page at a time

   104

            this_page = 1

   105

   106

            # last line's timestamp

   107

            last_ts = None

   108

   109

            # found it yet?

   110

            found = False

   111

   112

        # count the full number of lines

   113

        line_count = 0

   114

   115

        # collect lines

   116

        lines = []

   117

   118

        # iterate using get_date

   119

        for line in self.get_date(dt) :

   120

            # count them

   121

            line_count += 1

   122

   123

            # skip?

   124

            if skip :

   125

                skip -= 1

   126

                continue

   127

   128

            # is this page all that we want/need?

   129

            if page or found :

   130

                # already full?

   131

                if len(lines) >= count :

   132

                    continue

   133

   134

            # specfic timestamp

   135

            else :

   136

                # didn't find it in this page?

   137

                if len(lines) >= count :

   138

                    # reset to next page

   139

                    lines = []

   140

                    this_page += 1

   141

   142

                # is dt between these two timestamps?

   143

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   144

                    # found!

   145

                    found = True

   146

                    page = this_page

   147

   148

                else :

   149

                    # keep looking

   150

                    last_ts = line.timestamp

   151

   152

            # store line

   153

            lines.append(line)

   154

   155

        # calculate max_pages

   156

        max_pages = math.ceil(float(line_count) / count)

   157

   158

        # return

   159

        return (page, max_pages, lines)

   160

   161

    def get_month_days (self, dt) :

   162

        """

   164

        """

   165

   166

        abstract

   167

   168

    def get_modified (self, dt=None, after=None, until=None) :

   169

        """

   170

            Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime.

   171

   172

            If the datetime is not given, *all* lines are returned.

   173

   174

            If after is given, only lines from said date onwards will be returned, regardless of modification.

   175

            If until is given, only lines up to and including said date will be returned, regardless of modification.

   176

   177

            The LogLines should be in time order.

   178

        """

   179

   180

        abstract

   195

   196

class LogFile (object) :

   197

    """

   198

        A file containing LogEvents

   199

   200

        XXX: modify to implement LogSource?

   201

    """

   202

   203

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   204

        """

   205

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   206

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   207

            as the initial date for this log's first line.

   208

   209

            XXX: currently we assume start_date also for the end of the file

   210

        """

   211

   212

        # store

   213

        self.channel = channel

   214

        self.path = path

   215

        self.parser = parser

   216

        self.start_date = start_date

   217

        self.decoder = decoder

   218

        self.sep = sep

   219

   220

        # open

   221

        self.file = open(path, 'rb')

   222

   223

    def __iter__ (self) :

   224

        """

   225

            Yields a series of unicode lines, as read from the top of the file

   226

        """

   227

   228

        # seek to beginning

   229

        self.file.seek(0)

   230

   231

        # iterate over lines, decoding them as well

   232

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   233

   234

    def read_full (self) :

   235

        """

   236

            Reads all LogLines. The LogLines will have a valid offset.

   237

        """

   238

   239

        # just use our __iter__

   240

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   241

   242

    def read_from (self, dt) :

   243

        """

   244

            Reads all LogLines from the given naive timestamp onwards

   245

        """

   246

   247

        # start reading at beginning

   248

        events = self.read_full()

   249

   250

        # skip unwanted events

   251

        for event in events :

   252

            if event.timestamp < dt :

   253

                continue

   254

   255

            else :

   256

                # include this line as well

   257

                yield event

   258

                break

   259

   260

        # yield the rest as-is

   261

        for event in events :

   262

            yield event

   263

   264

    def read_until (self, dt) :

   265

        """

   266

            Reads all LogLines up until the given naive timestamp

   267

        """

   268

   269

        # start reading events at the beginning

   270

        events = self.read_full()

   271

   272

        # yield events until we hit the given timestamp

   273

        for event in events :

   274

            if event.timestamp <= dt :

   275

                yield event

   276

   277

            else :

   278

                break

   279

   280

        # ignore the rest

   281

        return

   282

   283

    def _read_blocks_reverse (self, blocksize=1024) :

   284

        """

   285

            Yields blocks of file data in reverse order, starting at the end of the file

   286

        """

   287

   288

        # seek to end of file

   289

        self.file.seek(0, os.SEEK_END)

   290

   291

        # read offset

   292

        # XXX: hack -1 to get rid of trailing newline

   293

        size = offset = self.file.tell() - 1

   294

   295

        # do not try to read past the beginning of the file

   296

        while offset > 0:

   297

            # calc new offset + size

   298

            if offset > blocksize :

   299

                # full block

   300

                offset -= blocksize

   301

                read_size = blocksize

   302

   303

            else :

   304

                # partial block

   305

                read_size = offset

   306

                offset = 0

   307

   308

            # seek to offset

   309

            self.file.seek(offset)

   310

   311

            # read the data we want

   312

            block = self.file.read(read_size)

   313

   314

            # sanity check

   315

            assert len(block) == read_size

   316

   317

            # yield 

   318

            yield block

   319

   320

    def _read_lines_reverse (self) :

   321

        """

   322

            Yields decoded lines from the end of the file, in reverse order.

   323

        """

   324

   325

        # partial lines

   326

        buf = ''

   327

   328

        # read from end of file, a block at a time

   329

        for block in self._read_blocks_reverse() :

   330

            # add in our previous buf

   331

            buf = block + buf

   332

   333

            # split up lines

   334

            lines = buf.split(self.sep)

   335

   336

            # keep the first one as our buffer, as it's incomplete

   337

            buf = lines[0]

   338

   339

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   340

            # XXX: use something like islice, this has to build a slice object

   341

            for line in lines[:0:-1] :

   342

                yield self.decoder.decode(line)

   343

   344

    def read_latest (self, count) :

   345

        """

   346

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   347

        """

   348

   349

        # the list of lines

   350

        lines = []

   351

   352

        # start reading lines into lines

   353

        for line in self._read_lines_reverse() :

   354

            # append

   355

            lines.append(line)

   356

   357

            # done?

   358

            if len(lines) >= count :

   359

                break

   360

   361

        # decode in reverse order, using our starting date....

   362

        # XXX: use lines[::-1] or reversed?

   363

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   364

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   365

   366

class LogDirectory (LogSource) :

   367

    """

   368

        A directory containing a series of timestamped LogFiles

   369

    """

   370

   371

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   372

        """

   373

            Load the logfiles at the given path, which are for the given LogChannel

   374

   375

            Decode the file lines using the given decoder, the files are named according the the date in the given

   376

            timezone and date format, and will be parsed using the given parser.

   377

        """

   378

   379

        # store

   380

        self.channel = channel

   381

        self.path = path

   382

        self.tz = tz

   383

        self.parser = parser

   384

        self.decoder = decoder

   385

        self.filename_fmt = filename_fmt

   386

   387

    def _get_logfile_datetime (self, dt) :

   388

        """

   389

            Get the logfile corresponding to the given datetime

   390

        """

   391

   392

        # convert to target timezone

   393

        dtz = dt.astimezone(self.tz)

   394

   395

        # convert to date and use that

   396

        return self._get_logfile_date(dtz.date())

   397

   398

    def _get_logfile_date (self, d, load=True, mtime=False, ignore_missing=True) :

   399

        """

   400

            Get the logfile corresponding to the given naive date in our timezone. 

   401

   402

            If load is False, only test for the presence of the logfile, do not actually open it. If mtime is given,

   403

            then this returns the file's mtime

   404

   405

            Returns None if the logfile does not exist, unless ignore_missing is given as False.

   406

        """

   407

   408

        # format filename

   409

        filename = d.strftime(self.filename_fmt)

   410

   411

        # build path

   412

        path = os.path.join(self.path, filename)

   413

   414

        try :

   415

            if load :

   416

                # open+return the LogFile

   417

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   418

   419

            elif mtime :

   420

                # stat

   421

                return utils.mtime(path)

   422

   423

            else :

   424

                # test

   425

                return os.path.exists(path)

   426

   427

        # XXX: move to LogFile

   428

        except IOError, e :

   429

            # return None for missing files

   430

            if e.errno == errno.ENOENT and ignore_missing :

   431

                return None

   432

   433

            else :

   434

                raise

   435

   437

        """

   438

            Yields a series of naive datetime objects representing the logfiles that are available, in time order.

   444

        """

   445

   446

        # listdir

   447

        filenames = os.listdir(self.path)

   448

   449

        # sort

   451

   452

        # iter files

   453

        for filename in filenames :

   454

            try :

   455

                # parse date

   456

                date = self.tz.localize(datetime.datetime.strptime(filename, self.filename_fmt))

   457

   458

            except :

   459

                # ignore

   460

                continue

   461

   462

            else :

   471

                    # yield

   472

                    yield date

   473

   474

    def _iter_date_reverse (self, dt=None) :

   475

        """

   476

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   477

            given *datetime*, or the the current date, if none given

   478

        """

   479

   480

        # default to now

   481

        if not dt :

   482

            dtz = self.tz.localize(datetime.datetime.now())

   483

   484

        else :

   485

            # convert to target timezone

   486

            dtz = dt.astimezone(self.tz)

   487

   488

        # iterate unto infinity

   489

        while True :

   490

            # yield

   491

            yield dtz.date()

   492

   493

            # one day sdrawkcab

   494