1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, errno

     7

import pytz

     8

     9

import config

    10

    11

class LogSourceDecoder (object) :

    12

    """

    13

        Handles decoding of LogSource lines

    14

    """

    15

    16

    def __init__ (self, encoding_list) :

    17

        """

    18

            Will try each of the given (charset, errors) items in turn, until one succeeds

    19

        """

    20

    21

        self.encoding_list = encoding_list

    22

    23

    def decode (self, line) :

    24

        """

    25

            Decode the line of str() text into an unicode object

    26

        """

    27

    28

        # list of errors encountered

    29

        error_list = []

    30

    31

        # try each in turn

    32

        for charset, errors in self.encoding_list :

    33

            # trap UnicodeDecodeError to try with the next one

    34

            try :

    35

                return line.decode(charset, errors)

    36

    37

            except UnicodeDecodeError, e :

    38

                error_list.append("%s:%s - %s" % (charset, errors, e))

    39

                continue

    40

    41

        # failure

    42

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    43

    44

class LogSource (object) :

    45

    """

    46

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    47

    """

    48

    49

    def __init__ (self, decoder, channel=None) :

    50

        """

    51

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    52

            known, then it can be given as None, and set later with bind_channel.

    53

    54

            Uses the given LogSourceDecoder to decode the lines.

    55

        """

    56

    57

        self.channel = channel

    58

        self.decoder = decoder

    59

    60

    def bind_channel (self, channel) :

    61

        """

    62

            Set this source's channel, where None was set before

    63

        """

    64

    65

        assert not self.channel

    66

    67

        self.channel = channel

    68

    69

    def get_latest (self, count) :

    70

        """

    71

            Yield the latest events, up to `count` of them.

    72

        """

    73

    74

        abstract

    75

    76

    def get_date (self, dt) :

    77

        """

    78

            Get logs for the given date (as a datetime).

    79

        """

    80

    81

        abstract

    82

    83

    def get_date_paged (self, dt, count, page=None) :

    84

        """

    85

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    86

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    87

            then the lines for the page containing the given timestamp is returned.

    88

    89

            The return value is a (page, max, lines) tuple.

    90

        """

    91

    92

        # how to act?

    93

        if page :

    94

            # constant skip

    95

            skip = (page - 1) * count

    96

    97

        else :

    98

            skip = None

    99

   100

            # go through the logs a page at a time

   101

            this_page = 1

   102

   103

            # last line's timestamp

   104

            last_ts = None

   105

   106

            # found it yet?

   107

            found = False

   108

   109

        # count the full number of lines

   110

        line_count = 0

   111

   112

        # collect lines

   113

        lines = []

   114

   115

        # iterate using get_date

   116

        for line in self.get_date(dt) :

   117

            # count them

   118

            line_count += 1

   119

   120

            # skip?

   121

            if skip :

   122

                skip -= 1

   123

                continue

   124

   125

            # is this page all that we want/need?

   126

            if page or found :

   127

                # already full?

   128

                if len(lines) >= count :

   129

                    continue

   130

   131

            # specfic timestamp

   132

            else :

   133

                # didn't find it in this page?

   134

                if len(lines) >= count :

   135

                    # reset to next page

   136

                    lines = []

   137

                    this_page += 1

   138

   139

                # is dt between these two timestamps?

   140

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   141

                    # found!

   142

                    found = True

   143

                    page = this_page

   144

   145

                else :

   146

                    # keep looking

   147

                    last_ts = line.timestamp

   148

   149

            # store line

   150

            lines.append(line)

   151

   152

        # calculate max_pages

   153

        max_pages = math.ceil(float(line_count) / count)

   154

   155

        # return

   156

        return (page, max_pages, lines)

   157

   158

    def get_month_days (self, dt) :

   159

        """

   160

            Return a sequence of dates, telling which days in the given month (as a datetime) have logs available

   161

        """

   162

   163

        abstract

   164

   165

class LogFile (object) :

   166

    """

   167

        A file containing LogEvents

   168

   169

        XXX: modify to implement LogSource?

   170

    """

   171

   172

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   173

        """

   174

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   175

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   176

            as the initial date for this log's first line.

   177

   178

            XXX: currently we assume start_date also for the end of the file

   179

        """

   180

   181

        # store

   182

        self.channel = channel

   183

        self.path = path

   184

        self.parser = parser

   185

        self.start_date = start_date

   186

        self.decoder = decoder

   187

        self.sep = sep

   188

   189

        # open

   190

        self.file = open(path, 'rb')

   191

   192

    def __iter__ (self) :

   193

        """

   194

            Yields a series of unicode lines, as read from the top of the file

   195

        """

   196

   197

        # seek to beginning

   198

        self.file.seek(0)

   199

   200

        # iterate over lines, decoding them as well

   201

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   202

   203

    def read_full (self) :

   204

        """

   205

            Reads all LogLines. The LogLines will have a valid offset.

   206

        """

   207

   208

        # just use our __iter__

   209

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   210

   211

    def read_from (self, dt) :

   212

        """

   213

            Reads all LogLines from the given naive timestamp onwards

   214

        """

   215

   216

        # start reading at beginning

   217

        events = self.read_full()

   218

   219

        # skip unwanted events

   220

        for event in events :

   221

            if event.timestamp < dt :

   222

                continue

   223

   224

            else :

   225

                # include this line as well

   226

                yield event

   227

                break

   228

   229

        # yield the rest as-is

   230

        for event in events :

   231

            yield event

   232

   233

    def read_until (self, dt) :

   234

        """

   235

            Reads all LogLines up until the given naive timestamp

   236

        """

   237

   238

        # start reading events at the beginning

   239

        events = self.read_full()

   240

   241

        # yield events until we hit the given timestamp

   242

        for event in events :

   243

            if event.timestamp <= dt :

   244

                yield event

   245

   246

            else :

   247

                break

   248

   249

        # ignore the rest

   250

        return

   251

   252

    def _read_blocks_reverse (self, blocksize=1024) :

   253

        """

   254

            Yields blocks of file data in reverse order, starting at the end of the file

   255

        """

   256

   257

        # seek to end of file

   258

        self.file.seek(0, os.SEEK_END)

   259

   260

        # read offset

   261

        # XXX: hack -1 to get rid of trailing newline

   262

        size = offset = self.file.tell() - 1

   263

   264

        # do not try to read past the beginning of the file

   265

        while offset > 0:

   266

            # calc new offset + size

   267

            if offset > blocksize :

   268

                # full block

   269

                offset -= blocksize

   270

                read_size = blocksize

   271

   272

            else :

   273

                # partial block

   274

                read_size = offset

   275

                offset = 0

   276

   277

            # seek to offset

   278

            self.file.seek(offset)

   279

   280

            # read the data we want

   281

            block = self.file.read(read_size)

   282

   283

            # sanity check

   284

            assert len(block) == read_size

   285

   286

            # yield 

   287

            yield block

   288

   289

    def _read_lines_reverse (self) :

   290

        """

   291

            Yields decoded lines from the end of the file, in reverse order.

   292

        """

   293

   294

        # partial lines

   295

        buf = ''

   296

   297

        # read from end of file, a block at a time

   298

        for block in self._read_blocks_reverse() :

   299

            # add in our previous buf

   300

            buf = block + buf

   301

   302

            # split up lines

   303

            lines = buf.split(self.sep)

   304

   305

            # keep the first one as our buffer, as it's incomplete

   306

            buf = lines[0]

   307

   308

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   309

            # XXX: use something like islice, this has to build a slice object

   310

            for line in lines[:0:-1] :

   311

                yield self.decoder.decode(line)

   312

   313

    def read_latest (self, count) :

   314

        """

   315

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   316

        """

   317

   318

        # the list of lines

   319

        lines = []

   320

   321

        # start reading lines into lines

   322

        for line in self._read_lines_reverse() :

   323

            # append

   324

            lines.append(line)

   325

   326

            # done?

   327

            if len(lines) >= count :

   328

                break

   329

   330

        # decode in reverse order, using our starting date....

   331

        # XXX: use lines[::-1] or reversed?

   332

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   333

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   334

   335

class LogDirectory (LogSource) :

   336

    """

   337

        A directory containing a series of timestamped LogFiles

   338

    """

   339

   340

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   341

        """

   342

            Load the logfiles at the given path, which are for the given LogChannel

   343

   344

            Decode the file lines using the given decoder, the files are named according the the date in the given

   345

            timezone and date format, and will be parsed using the given parser.

   346

        """

   347

   348

        # store

   349

        self.channel = channel

   350

        self.path = path

   351

        self.tz = tz

   352

        self.parser = parser

   353

        self.decoder = decoder

   354

        self.filename_fmt = filename_fmt

   355

   356

    def _get_logfile_datetime (self, dt) :

   357

        """

   358

            Get the logfile corresponding to the given datetime

   359

        """

   360

   361

        # convert to target timezone

   362

        dtz = dt.astimezone(self.tz)

   363

   364

        # convert to date and use that

   365

        return self._get_logfile_date(dtz.date())

   366

   367

    def _get_logfile_date (self, d, load=True) :

   368

        """

   369

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   370

            presence of the logfile, do not actually open it.

   371

   372

            Returns None if the logfile does not exist.

   373

        """

   374

   375

        # format filename

   376

        filename = d.strftime(self.filename_fmt)

   377

   378

        # build path

   379

        path = os.path.join(self.path, filename)

   380

   381

        try :

   382

            if load :

   383

                # open+return the LogFile

   384

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   385

   386

            else :

   387

                # test

   388

                return os.path.exists(path)

   389

   390

        # XXX: move to LogFile

   391

        except IOError, e :

   392

            # return None for missing files

   393

            if e.errno == errno.ENOENT :

   394

                return None

   395

   396

            else :

   397

                raise

   398

   399

    def _iter_date_reverse (self, dt=None) :

   400

        """

   401

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   402

            given *datetime*, or the the current date, if none given

   403

        """

   404

   405

        # default to now

   406

        if not dt :

   407

            dtz = self.tz.localize(datetime.datetime.now())

   408

   409

        else :

   410

            # convert to target timezone

   411

            dtz = dt.astimezone(self.tz)

   412

   413

        # our timedelta

   414

        ONE_DAY = datetime.timedelta(1)

   415

   416

        # iterate unto infinity

   417

        while True :

   418

            # yield

   419

            yield dtz.date()

   420

   421

            # one day sdrawkcab

   422

            dtz -= ONE_DAY

   423

   424

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   425

        """

   426

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   427

            current date, if none given.

   428

   429

            Reads/probes at most max_files files.

   430

        """

   431

   432

        # start counting at zero...

   433

        file_count = 0

   434

   435

        # have we found any files at all so far?

   436

        have_found = False

   437

   438

        # iterate backwards over days

   439

        for day in self._iter_date_reverse(dt) :

   440

            # stop if we've handled enough files by now

   441

            if file_count > max_files :

   442

                break

   443

   444

            # try and open the next logfile

   445

            logfile = None

   446

   447

            file_count += 1

   448

            logfile = self._get_logfile_date(day)

   449

   450

            # no logfile there?

   451

            if not logfile :

   452

                # hit our limit?

   453

                if file_count > max_files :

   454

                    # if we didn't find any logfiles at all, terminate rudely

   455

                    if not have_found :

   456

                        raise Exception("No recent logfiles found")

   457

   458

                    else :

   459

                        # stop looking, deal with what we've got

   460

                        return

   461

   462

                else :

   463

                    # skip to next day

   464

                    continue

   465

   466

            # mark have_found

   467

            have_found = True

   468

   469

            # yield it

   470

            yield logfile

   471

   472

    def get_latest (self, count) :

   473

        """

   474

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   475

        """

   476

   477

        # read the events into here

   478

        lines = []

   479

   480

        # start reading in those logfiles

   481

        for logfile in self._iter_logfile_reverse() :

   482

            # read the events

   483

            # XXX: use a queue

   484

            lines = list(logfile.read_latest(count)) + lines

   485

   486

            # done?

   487

            if len(lines) >= count :

   488

                break

   489

   490

        # return the events

   491

        return lines

   492

   493

    def get_date (self, dt) :

   494

        """

   495