1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, os.path, errno

     7

import pytz

     8

     9

import config, utils

    10

    11

class LogSourceDecoder (object) :

    12

    """

    13

        Handles decoding of LogSource lines

    14

    """

    15

    16

    def __init__ (self, encoding_list) :

    17

        """

    18

            Will try each of the given (charset, errors) items in turn, until one succeeds

    19

        """

    20

    21

        self.encoding_list = encoding_list

    22

    23

    def decode (self, line) :

    24

        """

    25

            Decode the line of str() text into an unicode object

    26

        """

    27

    28

        # list of errors encountered

    29

        error_list = []

    30

    31

        # try each in turn

    32

        for charset, errors in self.encoding_list :

    33

            # trap UnicodeDecodeError to try with the next one

    34

            try :

    35

                return line.decode(charset, errors)

    36

    37

            except UnicodeDecodeError, e :

    38

                error_list.append("%s:%s - %s" % (charset, errors, e))

    39

                continue

    40

    41

        # failure

    42

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    43

    44

class LogSource (object) :

    45

    """

    46

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    47

    """

    48

    49

    def __init__ (self, decoder, channel=None) :

    50

        """

    51

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    52

            known, then it can be given as None, and set later with bind_channel.

    53

    54

            Uses the given LogSourceDecoder to decode the lines.

    55

        """

    56

    57

        self.channel = channel

    58

        self.decoder = decoder

    59

    60

    def bind_channel (self, channel) :

    61

        """

    62

            Set this source's channel, where None was set before

    63

        """

    64

    65

        assert not self.channel

    66

    67

        self.channel = channel

    68

    69

    def get_latest (self, count) :

    70

        """

    71

            Yield the latest events, up to `count` of them.

    72

        """

    73

    74

        abstract

    75

    76

    def get_date (self, dt) :

    77

        """

    78

            Get logs for the given date (as a datetime).

    79

        """

    80

    81

        abstract

    82

    83

    def get_date_paged (self, dt, count, page=None) :

    84

        """

    85

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    86

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    87

            then the lines for the page containing the given timestamp is returned.

    88

    89

            The return value is a (page, max, lines) tuple.

    90

        """

    91

    92

        # how to act?

    93

        if page :

    94

            # constant skip

    95

            skip = (page - 1) * count

    96

    97

        else :

    98

            skip = None

    99

   100

            # go through the logs a page at a time

   101

            this_page = 1

   102

   103

            # last line's timestamp

   104

            last_ts = None

   105

   106

            # found it yet?

   107

            found = False

   108

   109

        # count the full number of lines

   110

        line_count = 0

   111

   112

        # collect lines

   113

        lines = []

   114

   115

        # iterate using get_date

   116

        for line in self.get_date(dt) :

   117

            # count them

   118

            line_count += 1

   119

   120

            # skip?

   121

            if skip :

   122

                skip -= 1

   123

                continue

   124

   125

            # is this page all that we want/need?

   126

            if page or found :

   127

                # already full?

   128

                if len(lines) >= count :

   129

                    continue

   130

   131

            # specfic timestamp

   132

            else :

   133

                # didn't find it in this page?

   134

                if len(lines) >= count :

   135

                    # reset to next page

   136

                    lines = []

   137

                    this_page += 1

   138

   139

                # is dt between these two timestamps?

   140

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   141

                    # found!

   142

                    found = True

   143

                    page = this_page

   144

   145

                else :

   146

                    # keep looking

   147

                    last_ts = line.timestamp

   148

   149

            # store line

   150

            lines.append(line)

   151

   152

        # calculate max_pages

   153

        max_pages = math.ceil(float(line_count) / count)

   154

   155

        # return

   156

        return (page, max_pages, lines)

   157

   158

    def get_month_days (self, dt) :

   159

        """

   160

            Return a sequence of dates, telling which days in the given month (as a datetime) have logs available

   161

        """

   162

   163

        abstract

   164

   165

    def get_modified (self, dt=None, after=None, until=None) :

   166

        """

   167

            Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime.

   168

   169

            If the datetime is not given, *all* lines are returned.

   170

   171

            If after is given, only lines after said date will be returned, regardless of modification.

   172

            If until is given, only lines up to said date will be returned, regardless of modification.

   173

   174

            The LogLines should be in time order.

   175

        """

   176

   177

        abstract

   178

   179

class LogFile (object) :

   180

    """

   181

        A file containing LogEvents

   182

   183

        XXX: modify to implement LogSource?

   184

    """

   185

   186

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   187

        """

   188

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   189

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   190

            as the initial date for this log's first line.

   191

   192

            XXX: currently we assume start_date also for the end of the file

   193

        """

   194

   195

        # store

   196

        self.channel = channel

   197

        self.path = path

   198

        self.parser = parser

   199

        self.start_date = start_date

   200

        self.decoder = decoder

   201

        self.sep = sep

   202

   203

        # open

   204

        self.file = open(path, 'rb')

   205

   206

    def __iter__ (self) :

   207

        """

   208

            Yields a series of unicode lines, as read from the top of the file

   209

        """

   210

   211

        # seek to beginning

   212

        self.file.seek(0)

   213

   214

        # iterate over lines, decoding them as well

   215

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   216

   217

    def read_full (self) :

   218

        """

   219

            Reads all LogLines. The LogLines will have a valid offset.

   220

        """

   221

   222

        # just use our __iter__

   223

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   224

   225

    def read_from (self, dt) :

   226

        """

   227

            Reads all LogLines from the given naive timestamp onwards

   228

        """

   229

   230

        # start reading at beginning

   231

        events = self.read_full()

   232

   233

        # skip unwanted events

   234

        for event in events :

   235

            if event.timestamp < dt :

   236

                continue

   237

   238

            else :

   239

                # include this line as well

   240

                yield event

   241

                break

   242

   243

        # yield the rest as-is

   244

        for event in events :

   245

            yield event

   246

   247

    def read_until (self, dt) :

   248

        """

   249

            Reads all LogLines up until the given naive timestamp

   250

        """

   251

   252

        # start reading events at the beginning

   253

        events = self.read_full()

   254

   255

        # yield events until we hit the given timestamp

   256

        for event in events :

   257

            if event.timestamp <= dt :

   258

                yield event

   259

   260

            else :

   261

                break

   262

   263

        # ignore the rest

   264

        return

   265

   266

    def _read_blocks_reverse (self, blocksize=1024) :

   267

        """

   268

            Yields blocks of file data in reverse order, starting at the end of the file

   269

        """

   270

   271

        # seek to end of file

   272

        self.file.seek(0, os.SEEK_END)

   273

   274

        # read offset

   275

        # XXX: hack -1 to get rid of trailing newline

   276

        size = offset = self.file.tell() - 1

   277

   278

        # do not try to read past the beginning of the file

   279

        while offset > 0:

   280

            # calc new offset + size

   281

            if offset > blocksize :

   282

                # full block

   283

                offset -= blocksize

   284

                read_size = blocksize

   285

   286

            else :

   287

                # partial block

   288

                read_size = offset

   289

                offset = 0

   290

   291

            # seek to offset

   292

            self.file.seek(offset)

   293

   294

            # read the data we want

   295

            block = self.file.read(read_size)

   296

   297

            # sanity check

   298

            assert len(block) == read_size

   299

   300

            # yield 

   301

            yield block

   302

   303

    def _read_lines_reverse (self) :

   304

        """

   305

            Yields decoded lines from the end of the file, in reverse order.

   306

        """

   307

   308

        # partial lines

   309

        buf = ''

   310

   311

        # read from end of file, a block at a time

   312

        for block in self._read_blocks_reverse() :

   313

            # add in our previous buf

   314

            buf = block + buf

   315

   316

            # split up lines

   317

            lines = buf.split(self.sep)

   318

   319

            # keep the first one as our buffer, as it's incomplete

   320

            buf = lines[0]

   321

   322

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   323

            # XXX: use something like islice, this has to build a slice object

   324

            for line in lines[:0:-1] :

   325

                yield self.decoder.decode(line)

   326

   327

    def read_latest (self, count) :

   328

        """

   329

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   330

        """

   331

   332

        # the list of lines

   333

        lines = []

   334

   335

        # start reading lines into lines

   336

        for line in self._read_lines_reverse() :

   337

            # append

   338

            lines.append(line)

   339

   340

            # done?

   341

            if len(lines) >= count :

   342

                break

   343

   344

        # decode in reverse order, using our starting date....

   345

        # XXX: use lines[::-1] or reversed?

   346

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   347

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   348

   349

class LogDirectory (LogSource) :

   350

    """

   351

        A directory containing a series of timestamped LogFiles

   352

    """

   353

   354

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   355

        """

   356

            Load the logfiles at the given path, which are for the given LogChannel

   357

   358

            Decode the file lines using the given decoder, the files are named according the the date in the given

   359

            timezone and date format, and will be parsed using the given parser.

   360

        """

   361

   362

        # store

   363

        self.channel = channel

   364

        self.path = path

   365

        self.tz = tz

   366

        self.parser = parser

   367

        self.decoder = decoder

   368

        self.filename_fmt = filename_fmt

   369

   370

    def _get_logfile_datetime (self, dt) :

   371

        """

   372

            Get the logfile corresponding to the given datetime

   373

        """

   374

   375

        # convert to target timezone

   376

        dtz = dt.astimezone(self.tz)

   377

   378

        # convert to date and use that

   379

        return self._get_logfile_date(dtz.date())

   380

   381

    def _get_logfile_date (self, d, load=True, mtime=False, ignore_missing=True) :

   382

        """

   383

            Get the logfile corresponding to the given naive date in our timezone. 

   384

   385

            If load is False, only test for the presence of the logfile, do not actually open it. If mtime is given,

   386

            then this returns the file's mtime

   387

   388

            Returns None if the logfile does not exist, unless ignore_missing is given as False.

   389

        """

   390

   391

        # format filename

   392

        filename = d.strftime(self.filename_fmt)

   393

   394

        # build path

   395

        path = os.path.join(self.path, filename)

   396

   397

        try :

   398

            if load :

   399

                # open+return the LogFile

   400

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   401

   402

            elif mtime :

   403

                # stat

   404

                return utils.mtime(path)

   405

   406

            else :

   407

                # test

   408

                return os.path.exists(path)

   409

   410

        # XXX: move to LogFile

   411

        except IOError, e :

   412

            # return None for missing files

   413

            if e.errno == errno.ENOENT and ignore_missing :

   414

                return None

   415

   416

            else :

   417

                raise

   418

   419

    def _iter_logfile_dates (self, after=None, until=None) :

   420

        """

   421

            Yields a series of naive datetime objects representing the logfiles that are available, in time order.

   422

   423

            If after is given, only dates after said date will be returned

   424

            If until is given, only dates up to said date will be returned

   425

        """

   426

   427

        # listdir

   428

        filenames = os.listdir(self.path)

   429

   430

        # sort

   431

        filenames.sort()

   432

   433

        # iter files

   434

        for filename in filenames :

   435

            try :

   436

                # parse date

   437

                date = datetime.datetime.strptime(filename, self.filename_fmt).replace(tzinfo=self.tz)

   438

   439

            except :

   440

                # ignore

   441

                continue

   442

   443

            else :

   444

                # ignore before?

   445

                if after and date < after :

   446

                    continue

   447

   448

                # ignore after?

   449

                if until and date > until :

   450

                    continue

   451

   452

                else :

   453

                    # yield

   454

                    yield date

   455

   456

    def _iter_date_reverse (self, dt=None) :

   457

        """

   458

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   459

            given *datetime*, or the the current date, if none given

   460

        """

   461

   462

        # default to now

   463

        if not dt :

   464

            dtz = self.tz.localize(datetime.datetime.now())

   465

   466

        else :

   467

            # convert to target timezone

   468

            dtz = dt.astimezone(self.tz)

   469

   470

        # our timedelta

   471

        ONE_DAY = datetime.timedelta(1)

   472

   473

        # iterate unto infinity

   474

        while True :

   475

            # yield

   476

            yield dtz.date()

   477

   478

            # one day sdrawkcab

   479

            dtz -= ONE_DAY

   480

   481

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   482

        """

   483

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   484

            current date, if none given.

   485

   486

            Reads/probes at most max_files files.

   487

        """

   488

   489

        # start counting at zero...

   490

        file_count = 0

   491

   492

        # have we found any files at all so far?

   493

        have_found = False

   494