1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, os.path, errno

     7

import pytz

     8

     9

import config, utils

    10

    11

# a timedelta that represents one day

    12

ONE_DAY = datetime.timedelta(days=1)

    13

    14

class LogSourceDecoder (object) :

    15

    """

    16

        Handles decoding of LogSource lines

    17

    """

    18

    19

    def __init__ (self, encoding_list) :

    20

        """

    21

            Will try each of the given (charset, errors) items in turn, until one succeeds

    22

        """

    23

    24

        self.encoding_list = encoding_list

    25

    26

    def decode (self, line) :

    27

        """

    28

            Decode the line of str() text into an unicode object

    29

        """

    30

    31

        # list of errors encountered

    32

        error_list = []

    33

    34

        # try each in turn

    35

        for charset, errors in self.encoding_list :

    36

            # trap UnicodeDecodeError to try with the next one

    37

            try :

    38

                return line.decode(charset, errors)

    39

    40

            except UnicodeDecodeError, e :

    41

                error_list.append("%s:%s - %s" % (charset, errors, e))

    42

                continue

    43

    44

        # failure

    45

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    46

    47

class LogSource (object) :

    48

    """

    49

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    50

    """

    51

    52

    def __init__ (self, decoder, channel=None) :

    53

        """

    54

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    55

            known, then it can be given as None, and set later with bind_channel.

    56

    57

            Uses the given LogSourceDecoder to decode the lines.

    58

        """

    59

    60

        self.channel = channel

    61

        self.decoder = decoder

    62

    63

    def bind_channel (self, channel) :

    64

        """

    65

            Set this source's channel, where None was set before

    66

        """

    67

    68

        assert not self.channel

    69

    70

        self.channel = channel

    71

    72

    def get_latest (self, count) :

    73

        """

    74

            Yield the latest events, up to `count` of them.

    75

        """

    76

    77

        abstract

    78

    79

    def get_date (self, dt) :

    80

        """

    81

            Get logs for the given date (as a datetime).

    82

        """

    83

    84

        abstract

    85

    86

    def get_date_paged (self, dt, count, page=None) :

    87

        """

    88

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    89

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    90

            then the lines for the page containing the given timestamp is returned.

    91

    96

            The return value is a (page, max, lines) tuple.

   101

        """

   102

   103

        # how to act?

   104

        if page :

   105

            # constant skip

   106

            skip = (page - 1) * count

   107

   108

        else :

   109

            skip = None

   110

   111

            # go through the logs a page at a time

   112

            this_page = 1

   113

   114

            # last line's timestamp

   115

            last_ts = None

   116

   117

            # found it yet?

   118

            found = False

   119

   120

        # count the full number of lines

   121

        line_count = 0

   122

   123

        # collect lines

   124

        lines = []

   125

   126

        # iterate using get_date

   127

        for line in self.get_date(dt) :

   128

            # count them

   129

            line_count += 1

   130

   131

            # skip?

   132

            if skip :

   133

                skip -= 1

   134

                continue

   135

   136

            # is this page all that we want/need?

   137

            if page or found :

   138

                # already full?

   139

                if len(lines) >= count :

   140

                    continue

   141

   142

            # specfic timestamp

   143

            else :

   144

                # didn't find it in this page?

   145

                if len(lines) >= count :

   146

                    # reset to next page

   147

                    lines = []

   148

                    this_page += 1

   149

   150

                # is dt between these two timestamps?

   151

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   152

                    # found!

   153

                    found = True

   154

                    page = this_page

   155

   156

                else :

   157

                    # keep looking

   158

                    last_ts = line.timestamp

   159

   160

            # store line

   161

            lines.append(line)

   162

   163

        # calculate max_pages

   164

        max_pages = math.ceil(float(line_count) / count)

   165

   166

        # return

   167

        return (page, max_pages, lines)

   168

   169

    def get_month_days (self, dt) :

   170

        """

   171

            Return an ordered sequence of dates, telling which days in the given month (as a datetime) have logs available.

   172

        """

   173

   174

        abstract

   175

   176

    def get_modified (self, dt=None, after=None, until=None) :

   177

        """

   178

            Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime.

   179

   180

            If the datetime is not given, *all* lines are returned.

   181

   182

            If after is given, only lines from said date onwards will be returned, regardless of modification.

   183

            If until is given, only lines up to and including said date will be returned, regardless of modification.

   184

   185

            The LogLines should be in time order.

   186

        """

   187

   188

        abstract

   189

   190

    def get_prev_date (self, dt) :

   191

        """

   192

            Get the next distinct date of logs available preceeding the given date, or None

   193

        """

   194

   195

        abstract

   196

   197

    def get_next_date (self, dt) :

   198

        """

   199

            Get the next distinct date of logs following the given date, or None.

   200

        """

   201

   202

        abstract

   203

   204

class LogFile (object) :

   205

    """

   206

        A file containing LogEvents

   207

   208

        XXX: modify to implement LogSource?

   209

    """

   210

   211

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   212

        """

   213

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   214

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   215

            as the initial date for this log's first line.

   216

   217

            XXX: currently we assume start_date also for the end of the file

   218

        """

   219

   220

        # store

   221

        self.channel = channel

   222

        self.path = path

   223

        self.parser = parser

   224

        self.start_date = start_date

   225

        self.decoder = decoder

   226

        self.sep = sep

   227

   228

        # open

   229

        self.file = open(path, 'rb')

   230

   231

    def __iter__ (self) :

   232

        """

   233

            Yields a series of unicode lines, as read from the top of the file

   234

        """

   235

   236

        # seek to beginning

   237

        self.file.seek(0)

   238

   239

        # iterate over lines, decoding them as well

   240

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   241

   242

    def read_full (self) :

   243

        """

   244

            Reads all LogLines. The LogLines will have a valid offset.

   245

        """

   246

   247

        # just use our __iter__

   248

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   249

   250

    def read_from (self, dt) :

   251

        """

   252

            Reads all LogLines from the given naive timestamp onwards

   253

        """

   254

   255

        # start reading at beginning

   256

        events = self.read_full()

   257

   258

        # skip unwanted events

   259

        for event in events :

   260

            if event.timestamp < dt :

   261

                continue

   262

   263

            else :

   264

                # include this line as well

   265

                yield event

   266

                break

   267

   268

        # yield the rest as-is

   269

        for event in events :

   270

            yield event

   271

   272

    def read_until (self, dt) :

   273

        """

   274

            Reads all LogLines up until the given naive timestamp

   275

        """

   276

   277

        # start reading events at the beginning

   278

        events = self.read_full()

   279

   280

        # yield events until we hit the given timestamp

   281

        for event in events :

   282

            if event.timestamp <= dt :

   283

                yield event

   284

   285

            else :

   286

                break

   287

   288

        # ignore the rest

   289

        return

   290

   291

    def _read_blocks_reverse (self, blocksize=1024) :

   292

        """

   293

            Yields blocks of file data in reverse order, starting at the end of the file

   294

        """

   295

   296

        # seek to end of file

   297

        self.file.seek(0, os.SEEK_END)

   298

   299

        # read offset

   300

        # XXX: hack -1 to get rid of trailing newline

   301

        size = offset = self.file.tell() - 1

   302

   303

        # do not try to read past the beginning of the file

   304

        while offset > 0:

   305

            # calc new offset + size

   306

            if offset > blocksize :

   307

                # full block

   308

                offset -= blocksize

   309

                read_size = blocksize

   310

   311

            else :

   312

                # partial block

   313

                read_size = offset

   314

                offset = 0

   315

   316

            # seek to offset

   317

            self.file.seek(offset)

   318

   319

            # read the data we want

   320

            block = self.file.read(read_size)

   321

   322

            # sanity check

   323

            assert len(block) == read_size

   324

   325

            # yield 

   326

            yield block

   327

   328

    def _read_lines_reverse (self) :

   329

        """

   330

            Yields decoded lines from the end of the file, in reverse order.

   331

        """

   332

   333

        # partial lines

   334

        buf = ''

   335

   336

        # read from end of file, a block at a time

   337

        for block in self._read_blocks_reverse() :

   338

            # add in our previous buf

   339

            buf = block + buf

   340

   341

            # split up lines

   342

            lines = buf.split(self.sep)

   343

   344

            # keep the first one as our buffer, as it's incomplete

   345

            buf = lines[0]

   346

   347

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   348

            # XXX: use something like islice, this has to build a slice object

   349

            for line in lines[:0:-1] :

   350

                yield self.decoder.decode(line)

   351

   352

    def read_latest (self, count) :

   353

        """

   354

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   355

        """

   356

   357

        # the list of lines

   358

        lines = []

   359

   360

        # start reading lines into lines

   361

        for line in self._read_lines_reverse() :

   362

            # append

   363

            lines.append(line)

   364

   365

            # done?

   366

            if len(lines) >= count :

   367

                break

   368

   369

        # decode in reverse order, using our starting date....

   370

        # XXX: use lines[::-1] or reversed?

   371

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   372

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   373

   374

class LogDirectory (LogSource) :

   375

    """

   376

        A directory containing a series of timestamped LogFiles

   377

    """

   378

   379

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   380

        """

   381

            Load the logfiles at the given path, which are for the given LogChannel

   382

   383

            Decode the file lines using the given decoder, the files are named according the the date in the given

   384

            timezone and date format, and will be parsed using the given parser.

   385

        """

   386

   387

        # store

   388

        self.channel = channel

   389

        self.path = path

   390

        self.tz = tz

   391

        self.parser = parser

   392

        self.decoder = decoder

   393

        self.filename_fmt = filename_fmt

   394

   395

    def _get_logfile_date (self, d, load=True, mtime=False, ignore_missing=False) :

   396

        """

   397

            Get the logfile corresponding to the given naive date in our timezone. 

   398

   399

            If load is False, only test for the presence of the logfile, do not actually open it. If mtime is given,

   400

            then this returns the file's mtime

   401

   402

            Returns None if the logfile does not exist, unless ignore_missing is given as False.

   403

        """

   404

   405

        # format filename

   406

        filename = d.strftime(self.filename_fmt)

   407

   408

        # build path

   409

        path = os.path.join(self.path, filename)

   410

   411

        try :

   412

            if load :

   413

                # open+return the LogFile

   414

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   415

   416

            elif mtime :

   417

                # stat

   418

                return utils.mtime(path)

   419

   420

            else :

   421

                # test

   422

                return os.path.exists(path)

   423

   424

        # XXX: move to LogFile

   425

        except IOError, e :

   426

            # return None for missing files

   427

            if e.errno == errno.ENOENT and ignore_missing :

   428

                return None

   429

   430

            else :

   431

                raise

   432

   433

    def _iter_logfile_dates (self, after=None, until=None, reverse=False) :

   434

        """

   435

            Yields a series of naive datetime objects representing the logfiles that are available, in time order.

   436

   437

            Parameters :

   438

                after   only dates from said date onwards will be returned

   439

                until   only dates up to and including said date will be returned

   440

                reverse the dates are returned in reverse order instead. Note that the meaning of after/until doesn't change

   441

        """

   442

   443

        # convert timestamps to our timezone's dates

   444

        if after :

   445

            after = after.astimezone(self.tz).date()

   446

   447

        if until :

   448

            until = until.astimezone(self.tz).date()

   449

   450

        # listdir

   451

        filenames = os.listdir(self.path)

   452

   453

        # sort

   454

        filenames.sort(reverse=reverse)

   455

   456

        # iter files

   457

        for filename in filenames :

   458

            try :

   459

                # parse date

   460

                dt = self.tz.localize(datetime.datetime.strptime(filename, self.filename_fmt))

   461

                date = dt.date()

   462

   463

            except :

   464

                # ignore

   465

                continue

   466

   467

            else :

   468

                if (after and date < after) or (until and date > until) :

   469

                    # ignore

   470

                    continue

   471

   472

                else :

   473

                    # yield

   474

                    yield dt

   475

   476

    def _iter_date_reverse (self, dt=None) :

   477

        """

   478

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   479

            given *datetime*, or the the current date, if none given

   480

        """

   481

   482

        # default to now

   483

        if not dt :

   484

            dtz = self.tz.localize(datetime.datetime.now())

   485

   486

        else :

   487

            # convert to target timezone

   488

            dtz = dt.astimezone(self.tz)

   489

   490

        # iterate unto infinity

   491

        while True :

   492

            # yield