1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, os.path, errno

     7

import pytz

     8

     9

import config, utils

    10

    11

class LogSourceDecoder (object) :

    12

    """

    13

        Handles decoding of LogSource lines

    14

    """

    15

    16

    def __init__ (self, encoding_list) :

    17

        """

    18

            Will try each of the given (charset, errors) items in turn, until one succeeds

    19

        """

    20

    21

        self.encoding_list = encoding_list

    22

    23

    def decode (self, line) :

    24

        """

    25

            Decode the line of str() text into an unicode object

    26

        """

    27

    28

        # list of errors encountered

    29

        error_list = []

    30

    31

        # try each in turn

    32

        for charset, errors in self.encoding_list :

    33

            # trap UnicodeDecodeError to try with the next one

    34

            try :

    35

                return line.decode(charset, errors)

    36

    37

            except UnicodeDecodeError, e :

    38

                error_list.append("%s:%s - %s" % (charset, errors, e))

    39

                continue

    40

    41

        # failure

    42

        raise UnicodeDecodeError("Failed to decode line: %r: %s" % (line, ', '.join(error_list)))

    43

    44

class LogSource (object) :

    45

    """

    46

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    47

    """

    48

    49

    def __init__ (self, decoder, channel=None) :

    50

        """

    51

            The appropriate LogChannel must be given, as we need to be able to construct the LogLines. If it is not yet

    52

            known, then it can be given as None, and set later with bind_channel.

    53

    54

            Uses the given LogSourceDecoder to decode the lines.

    55

        """

    56

    57

        self.channel = channel

    58

        self.decoder = decoder

    59

    60

    def bind_channel (self, channel) :

    61

        """

    62

            Set this source's channel, where None was set before

    63

        """

    64

    65

        assert not self.channel

    66

    67

        self.channel = channel

    68

    69

    def get_latest (self, count) :

    70

        """

    71

            Yield the latest events, up to `count` of them.

    72

        """

    73

    74

        abstract

    75

    76

    def get_date (self, dt) :

    77

        """

    78

            Get logs for the given date (as a datetime).

    79

        """

    80

    81

        abstract

    82

    83

    def get_date_paged (self, dt, count, page=None) :

    84

        """

    85

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    86

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    87

            then the lines for the page containing the given timestamp is returned.

    88

    89

            The return value is a (page, max, lines) tuple.

    90

        """

    91

    92

        # how to act?

    93

        if page :

    94

            # constant skip

    95

            skip = (page - 1) * count

    96

    97

        else :

    98

            skip = None

    99

   100

            # go through the logs a page at a time

   101

            this_page = 1

   102

   103

            # last line's timestamp

   104

            last_ts = None

   105

   106

            # found it yet?

   107

            found = False

   108

   109

        # count the full number of lines

   110

        line_count = 0

   111

   112

        # collect lines

   113

        lines = []

   114

   115

        # iterate using get_date

   116

        for line in self.get_date(dt) :

   117

            # count them

   118

            line_count += 1

   119

   120

            # skip?

   121

            if skip :

   122

                skip -= 1

   123

                continue

   124

   125

            # is this page all that we want/need?

   126

            if page or found :

   127

                # already full?

   128

                if len(lines) >= count :

   129

                    continue

   130

   131

            # specfic timestamp

   132

            else :

   133

                # didn't find it in this page?

   134

                if len(lines) >= count :

   135

                    # reset to next page

   136

                    lines = []

   137

                    this_page += 1

   138

   139

                # is dt between these two timestamps?

   140

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   141

                    # found!

   142

                    found = True

   143

                    page = this_page

   144

   145

                else :

   146

                    # keep looking

   147

                    last_ts = line.timestamp

   148

   149

            # store line

   150

            lines.append(line)

   151

   152

        # calculate max_pages

   153

        max_pages = math.ceil(float(line_count) / count)

   154

   155

        # return

   156

        return (page, max_pages, lines)

   157

   158

    def get_month_days (self, dt) :

   159

        """

   160

            Return a sequence of dates, telling which days in the given month (as a datetime) have logs available

   161

        """

   162

   163

        abstract

   164

   165

    def get_modified (self, dt=None, after=None) :

   166

        """

   167

            Returns a sequence of LogLines that may have been *modified* from their old values since the given datetime.

   168

   169

            If the datetime is not given, *all* lines are returned.

   170

   171

            If after is given, only lines after said date will be returned, regardless of modification.

   172

   173

            The LogLines should be in time order.

   174

        """

   175

   176

        abstract

   177

   178

class LogFile (object) :

   179

    """

   180

        A file containing LogEvents

   181

   182

        XXX: modify to implement LogSource?

   183

    """

   184

   185

    def __init__ (self, path, parser, decoder, channel=None, start_date=None, sep='\n') :

   186

        """

   187

            Open the file at the given path, which contains lines as separated by the given separator. Lines are

   188

            decoded using the given LogSourceDecoder, and then parsed using the given parser, using the given date

   189

            as the initial date for this log's first line.

   190

   191

            XXX: currently we assume start_date also for the end of the file

   192

        """

   193

   194

        # store

   195

        self.channel = channel

   196

        self.path = path

   197

        self.parser = parser

   198

        self.start_date = start_date

   199

        self.decoder = decoder

   200

        self.sep = sep

   201

   202

        # open

   203

        self.file = open(path, 'rb')

   204

   205

    def __iter__ (self) :

   206

        """

   207

            Yields a series of unicode lines, as read from the top of the file

   208

        """

   209

   210

        # seek to beginning

   211

        self.file.seek(0)

   212

   213

        # iterate over lines, decoding them as well

   214

        return (self.decoder.decode(line.rstrip(self.sep)) for line in self.file)

   215

   216

    def read_full (self) :

   217

        """

   218

            Reads all LogLines. The LogLines will have a valid offset.

   219

        """

   220

   221

        # just use our __iter__

   222

        return self.parser.parse_lines(self.channel, self, self.start_date, starting_offset=1)

   223

   224

    def read_from (self, dt) :

   225

        """

   226

            Reads all LogLines from the given naive timestamp onwards

   227

        """

   228

   229

        # start reading at beginning

   230

        events = self.read_full()

   231

   232

        # skip unwanted events

   233

        for event in events :

   234

            if event.timestamp < dt :

   235

                continue

   236

   237

            else :

   238

                # include this line as well

   239

                yield event

   240

                break

   241

   242

        # yield the rest as-is

   243

        for event in events :

   244

            yield event

   245

   246

    def read_until (self, dt) :

   247

        """

   248

            Reads all LogLines up until the given naive timestamp

   249

        """

   250

   251

        # start reading events at the beginning

   252

        events = self.read_full()

   253

   254

        # yield events until we hit the given timestamp

   255

        for event in events :

   256

            if event.timestamp <= dt :

   257

                yield event

   258

   259

            else :

   260

                break

   261

   262

        # ignore the rest

   263

        return

   264

   265

    def _read_blocks_reverse (self, blocksize=1024) :

   266

        """

   267

            Yields blocks of file data in reverse order, starting at the end of the file

   268

        """

   269

   270

        # seek to end of file

   271

        self.file.seek(0, os.SEEK_END)

   272

   273

        # read offset

   274

        # XXX: hack -1 to get rid of trailing newline

   275

        size = offset = self.file.tell() - 1

   276

   277

        # do not try to read past the beginning of the file

   278

        while offset > 0:

   279

            # calc new offset + size

   280

            if offset > blocksize :

   281

                # full block

   282

                offset -= blocksize

   283

                read_size = blocksize

   284

   285

            else :

   286

                # partial block

   287

                read_size = offset

   288

                offset = 0

   289

   290

            # seek to offset

   291

            self.file.seek(offset)

   292

   293

            # read the data we want

   294

            block = self.file.read(read_size)

   295

   296

            # sanity check

   297

            assert len(block) == read_size

   298

   299

            # yield 

   300

            yield block

   301

   302

    def _read_lines_reverse (self) :

   303

        """

   304

            Yields decoded lines from the end of the file, in reverse order.

   305

        """

   306

   307

        # partial lines

   308

        buf = ''

   309

   310

        # read from end of file, a block at a time

   311

        for block in self._read_blocks_reverse() :

   312

            # add in our previous buf

   313

            buf = block + buf

   314

   315

            # split up lines

   316

            lines = buf.split(self.sep)

   317

   318

            # keep the first one as our buffer, as it's incomplete

   319

            buf = lines[0]

   320

   321

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   322

            # XXX: use something like islice, this has to build a slice object

   323

            for line in lines[:0:-1] :

   324

                yield self.decoder.decode(line)

   325

   326

    def read_latest (self, count) :

   327

        """

   328

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   329

        """

   330

   331

        # the list of lines

   332

        lines = []

   333

   334

        # start reading lines into lines

   335

        for line in self._read_lines_reverse() :

   336

            # append

   337

            lines.append(line)

   338

   339

            # done?

   340

            if len(lines) >= count :

   341

                break

   342

   343

        # decode in reverse order, using our starting date....

   344

        # XXX: use lines[::-1] or reversed?

   345

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   346

        return self.parser.parse_lines(self.channel, reversed(lines), self.start_date)

   347

   348

class LogDirectory (LogSource) :

   349

    """

   350

        A directory containing a series of timestamped LogFiles

   351

    """

   352

   353

    def __init__ (self, path, tz, parser, decoder, filename_fmt, channel=None) :

   354

        """

   355

            Load the logfiles at the given path, which are for the given LogChannel

   356

   357

            Decode the file lines using the given decoder, the files are named according the the date in the given

   358

            timezone and date format, and will be parsed using the given parser.

   359

        """

   360

   361

        # store

   362

        self.channel = channel

   363

        self.path = path

   364

        self.tz = tz

   365

        self.parser = parser

   366

        self.decoder = decoder

   367

        self.filename_fmt = filename_fmt

   368

   369

    def _get_logfile_datetime (self, dt) :

   370

        """

   371

            Get the logfile corresponding to the given datetime

   372

        """

   373

   374

        # convert to target timezone

   375

        dtz = dt.astimezone(self.tz)

   376

   377

        # convert to date and use that

   378

        return self._get_logfile_date(dtz.date())

   379

   381

        """

   382

            Get the logfile corresponding to the given naive date in our timezone. 

   383

   386

   387

            Returns None if the logfile does not exist, unless ignore_missing is given as False.

   388

        """

   389

   390

        # format filename

   391

        filename = d.strftime(self.filename_fmt)

   392

   393

        # build path

   394

        path = os.path.join(self.path, filename)

   395

   396

        try :

   397

            if load :

   398

                # open+return the LogFile

   399

                return LogFile(path, self.parser, self.decoder, start_date=d, channel=self.channel)

   400

   402

                # stat

   404

   405

            else :

   406

                # test

   407

                return os.path.exists(path)

   408

   409

        # XXX: move to LogFile

   410

        except IOError, e :

   411

            # return None for missing files

   412

            if e.errno == errno.ENOENT and ignore_missing :

   413

                return None

   414

   415

            else :

   416

                raise

   417

   418

    def _iter_logfile_dates (self, after=None) :

   419

        """

   420

            Yields a series of naive datetime objects representing the logfiles that are available, in time order.

   421

   422

            If after is given, only dates after said date will be returned

   423

        """

   424

   425

        # listdir

   426

        filenames = os.listdir(self.path)

   427

   428

        # sort

   429

        filenames.sort()

   430

   431

        # iter files

   432

        for filename in filenames :

   433

            try :

   434

                # parse date

   435

                date = datetime.datetime.strptime(filename, self.filename_fmt).replace(tzinfo=self.tz)

   436

   437

            except :

   438

                # ignore

   439

                continue

   440

   441

            else :

   442

                # ignore after?

   443

                if after and date < after :

   444

                    continue

   445

   446

                else :

   447

                    # yield

   448

                    yield date

   449

   450

    def _iter_date_reverse (self, dt=None) :

   451

        """

   452

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   453

            given *datetime*, or the the current date, if none given

   454

        """

   455

   456

        # default to now

   457

        if not dt :

   458

            dtz = self.tz.localize(datetime.datetime.now())

   459

   460

        else :

   461

            # convert to target timezone

   462

            dtz = dt.astimezone(self.tz)

   463

   464

        # our timedelta

   465

        ONE_DAY = datetime.timedelta(1)

   466

   467

        # iterate unto infinity

   468

        while True :

   469

            # yield

   470

            yield dtz.date()

   471

   472

            # one day sdrawkcab

   473

            dtz -= ONE_DAY

   474

   475

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   476

        """

   477

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   478

            current date, if none given.

   479

   480

            Reads/probes at most max_files files.

   481

        """

   482

   483

        # start counting at zero...

   484

        file_count = 0

   485

   486

        # have we found any files at all so far?

   487

        have_found = False

   488

   489

        # iterate backwards over days

   490

        for day in self._iter_date_reverse(dt) :

   491

            # stop if we've handled enough files by now

   492

            if file_count > max_files :

   493

                break

   494