1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     7

import pytz

     8

    10

    11

class LogSourceDecoder (object) :

    12

    """

    13

        Handles decoding of LogSource lines

    14

    """

    15

    16

    def __init__ (self, encoding_list) :

    17

        """

    18

            Will try each of the given (charset, errors) items in turn, until one succeeds

    19

        """

    20

    21

        self.encoding_list = encoding_list

    22

    23

    def decode (self, line) :

    24

        """

    25

            Decode the line of str() text into an unicode object

    26

        """

    27

    28

        # list of errors encountered

    29

        error_list = []

    30

    31

        # try each in turn

    32

        for charset, errors in self.encoding_list :

    33

            # trap UnicodeDecodeError to try with the next one

    34

            try :

    35

                return line.decode(charset, errors)

    36

    37

            except UnicodeDecodeError, e :

    38

                error_list.append("%s:%s - %s" % (charset, errors, e))

    39

                continue

    40

    41

        # failure

    42

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    43

    44

class LogSource (object) :

    45

    """

    46

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    47

    """

    48

    49

    def __init__ (self, decoder, channel=None) :

    50

        """

    51

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    52

            known, then it can be given as None, and set later with bind_channel.

    53

    54

            Uses the given LogSourceDecoder to decode the lines.

    55

        """

    56

    57

        self.channel = channel

    58

        self.decoder = decoder

    59

    60

    def bind_channel (self, channel) :

    61

        """

    62

            Set this source's channel, where None was set before

    63

        """

    64

    65

        assert not self.channel

    66

    67

        self.channel = channel

    68

    69

    def get_latest (self, count) :

    70

        """

    71

            Yield the latest events, up to `count` of them.

    72

        """

    73

    74

        abstract

    75

    76

    def get_date (self, dt) :

    77

        """

    78

            Get logs for the given date (as a datetime).

    79

        """

    80

    81

        abstract

    82

    83

    def get_date_paged (self, dt, count, page=None) :

    84

        """

    85

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    86

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    87

            then the lines for the page containing the given timestamp is returned.

    88

    89

            The return value is a (page, max, lines) tuple.

    90

        """

    91

    92

        # how to act?

    93

        if page :

    94

            # constant skip

    95

            skip = (page - 1) * count

    96

    97

        else :

    98

            skip = None

    99

   100

            # go through the logs a page at a time

   101

            this_page = 1

   102

   103

            # last line's timestamp

   104

            last_ts = None

   105

   106

            # found it yet?

   107

            found = False

   108

   109

        # count the full number of lines

   110

        line_count = 0

   111

   112

        # collect lines

   113

        lines = []

   114

   115

        # iterate using get_date

   116

        for line in self.get_date(dt) :

   117

            # count them

   118

            line_count += 1

   119

   120

            # skip?

   121

            if skip :

   122

                skip -= 1

   123

                continue

   124

   125

            # is this page all that we want/need?

   126

            if page or found :

   127

                # already full?

   128

                if len(lines) >= count :

   129

                    continue

   130

   131

            # specfic timestamp

   132

            else :

   133

                # didn't find it in this page?

   134

                if len(lines) >= count :

   135

                    # reset to next page

   136

                    lines = []

   137

                    this_page += 1

   138

   139

                # is dt between these two timestamps?

   140

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   141

                    # found!

   142

                    found = True

   143

                    page = this_page

   144

   145

                else :

   146

                    # keep looking

   147

                    last_ts = line.timestamp

   148

   149

            # store line

   150

            lines.append(line)

   151

   152

        # calculate max_pages

   153

        max_pages = math.ceil(float(line_count) / count)

   154

   155

        # return

   156

        return (page, max_pages, lines)

   157

   158

    def get_month_days (self, dt) :

   159

        """

   160

            Return a sequence of dates, telling which days in the given month (as a datetime) have logs available

   161

        """

   162

   163

        abstract

   175

   176

class LogFile (object) :

   177

    """

   178

        A file containing LogEvents

   179

   180

        XXX: modify to implement LogSource?

   181

    """

   182

   183

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   184

        """

   185

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   186

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   187

            as the initial date for this log's first line.

   188

   189

            XXX: currently we assume start_date also for the end of the file

   190

        """

   191

   192

        # store

   193

        self.channel = channel

   194

        self.path = path

   195

        self.parser = parser

   196

        self.start_date = start_date

   197

        self.decoder = decoder

   198

        self.sep = sep

   199

   200

        # open

   201

        self.file = open(path, 'rb')

   202

   203

    def __iter__ (self) :

   204

        """

   205

            Yields a series of unicode lines, as read from the top of the file

   206

        """

   207

   208

        # seek to beginning

   209

        self.file.seek(0)

   210

   211

        # iterate over lines, decoding them as well

   212

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   213

   214

    def read_full (self) :

   215

        """

   216

            Reads all LogLines. The LogLines will have a valid offset.

   217

        """

   218

   219

        # just use our __iter__

   220

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   221

   222

    def read_from (self, dt) :

   223

        """

   224

            Reads all LogLines from the given naive timestamp onwards

   225

        """

   226

   227

        # start reading at beginning

   228

        events = self.read_full()

   229

   230

        # skip unwanted events

   231

        for event in events :

   232

            if event.timestamp < dt :

   233

                continue

   234

   235

            else :

   236

                # include this line as well

   237

                yield event

   238

                break

   239

   240

        # yield the rest as-is

   241

        for event in events :

   242

            yield event

   243

   244

    def read_until (self, dt) :

   245

        """

   246

            Reads all LogLines up until the given naive timestamp

   247

        """

   248

   249

        # start reading events at the beginning

   250

        events = self.read_full()

   251

   252

        # yield events until we hit the given timestamp

   253

        for event in events :

   254

            if event.timestamp <= dt :

   255

                yield event

   256

   257

            else :

   258

                break

   259

   260

        # ignore the rest

   261

        return

   262

   263

    def _read_blocks_reverse (self, blocksize=1024) :

   264

        """

   265

            Yields blocks of file data in reverse order, starting at the end of the file

   266

        """

   267

   268

        # seek to end of file

   269

        self.file.seek(0, os.SEEK_END)

   270

   271

        # read offset

   272

        # XXX: hack -1 to get rid of trailing newline

   273

        size = offset = self.file.tell() - 1

   274

   275

        # do not try to read past the beginning of the file

   276

        while offset > 0:

   277

            # calc new offset + size

   278

            if offset > blocksize :

   279

                # full block

   280

                offset -= blocksize

   281

                read_size = blocksize

   282

   283

            else :

   284

                # partial block

   285

                read_size = offset

   286

                offset = 0

   287

   288

            # seek to offset

   289

            self.file.seek(offset)

   290

   291

            # read the data we want

   292

            block = self.file.read(read_size)

   293

   294

            # sanity check

   295

            assert len(block) == read_size

   296

   297

            # yield 

   298

            yield block

   299

   300

    def _read_lines_reverse (self) :

   301

        """

   302

            Yields decoded lines from the end of the file, in reverse order.

   303

        """

   304

   305

        # partial lines

   306

        buf = ''

   307

   308

        # read from end of file, a block at a time

   309

        for block in self._read_blocks_reverse() :

   310

            # add in our previous buf

   311

            buf = block + buf

   312

   313

            # split up lines

   314

            lines = buf.split(self.sep)

   315

   316

            # keep the first one as our buffer, as it's incomplete

   317

            buf = lines[0]

   318

   319

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   320

            # XXX: use something like islice, this has to build a slice object

   321

            for line in lines[:0:-1] :

   322

                yield self.decoder.decode(line)

   323

   324

    def read_latest (self, count) :

   325

        """

   326

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   327

        """

   328

   329

        # the list of lines

   330

        lines = []

   331

   332

        # start reading lines into lines

   333

        for line in self._read_lines_reverse() :

   334

            # append

   335

            lines.append(line)

   336

   337

            # done?

   338

            if len(lines) >= count :

   339

                break

   340

   341

        # decode in reverse order, using our starting date....

   342

        # XXX: use lines[::-1] or reversed?

   343

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   344

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   345

   346

class LogDirectory (LogSource) :

   347

    """

   348

        A directory containing a series of timestamped LogFiles

   349

    """

   350

   351

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   352

        """

   353

            Load the logfiles at the given path, which are for the given LogChannel

   354

   355

            Decode the file lines using the given decoder, the files are named according the the date in the given

   356

            timezone and date format, and will be parsed using the given parser.

   357

        """

   358

   359

        # store

   360

        self.channel = channel

   361

        self.path = path

   362

        self.tz = tz

   363

        self.parser = parser

   364

        self.decoder = decoder

   365

        self.filename_fmt = filename_fmt

   366

   367

    def _get_logfile_datetime (self, dt) :

   368

        """

   369

            Get the logfile corresponding to the given datetime

   370

        """

   371

   372

        # convert to target timezone

   373

        dtz = dt.astimezone(self.tz)

   374

   375

        # convert to date and use that

   376

        return self._get_logfile_date(dtz.date())

   377

   379

        """

   384

   386

        """

   387

   388

        # format filename

   389

        filename = d.strftime(self.filename_fmt)

   390

   391

        # build path

   392

        path = os.path.join(self.path, filename)

   393

   394

        try :

   395

            if load :

   396

                # open+return the LogFile

   397

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   398

   403

            else :

   404

                # test

   405

                return os.path.exists(path)

   406

   407

        # XXX: move to LogFile

   408

        except IOError, e :

   409

            # return None for missing files

   411

                return None

   412

   413

            else :

   414

                raise

   415

   437

    def _iter_date_reverse (self, dt=None) :

   438

        """

   439

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   440

            given *datetime*, or the the current date, if none given

   441

        """

   442

   443

        # default to now

   444

        if not dt :

   445

            dtz = self.tz.localize(datetime.datetime.now())

   446

   447

        else :

   448

            # convert to target timezone

   449

            dtz = dt.astimezone(self.tz)

   450

   451

        # our timedelta

   452

        ONE_DAY = datetime.timedelta(1)

   453

   454

        # iterate unto infinity

   455

        while True :

   456

            # yield

   457

            yield dtz.date()

   458

   459

            # one day sdrawkcab

   460

            dtz -= ONE_DAY

   461

   462

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   463

        """

   464

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   465

            current date, if none given.

   466

   467

            Reads/probes at most max_files files.

   468

        """

   469

   470

        # start counting at zero...

   471

        file_count = 0

   472

   473

        # have we found any files at all so far?

   474

        have_found = False

   475

   476

        # iterate backwards over days

   477

        for day in self._iter_date_reverse(dt) :

   478

            # stop if we've handled enough files by now

   479

            if file_count > max_files :

   480

                break

   481

   482

            # try and open the next logfile

   483

            logfile = None

   484

   485

            file_count += 1

   486

            logfile = self._get_logfile_date(day)

   487

   488

            # no logfile there?

   489

            if not logfile :

   490

                # hit our limit?

   491

                if file_count > max_files :

   492

                    # if we didn't find any logfiles at all, terminate rudely

   493

                    if not have_found :

   494

                        raise Exception("No recent logfiles found")

   495