1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, errno

     7

import pytz

     8

    44

class LogSource (object) :

    45

    """

    46

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    47

    """

    48

    56

    def get_latest (self, count) :

    57

        """

    58

            Yield the latest events, up to `count` of them.

    59

        """

    60

    61

        abstract

    62

    63

    def get_date (self, dt) :

    64

        """

    65

            Get logs for the given date (as a datetime).

    66

        """

    67

    68

        abstract

    69

    70

    def get_date_paged (self, dt, count, page=None) :

    71

        """

    72

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    73

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    74

            then the lines for the page containing the given timestamp is returned.

    75

    76

            The return value is a (page, max, lines) tuple.

    77

        """

    78

    79

        # how to act?

    80

        if page :

    81

            # constant skip

    82

            skip = (page - 1) * count

    83

    84

        else :

    85

            skip = None

    86

    87

            # go through the logs a page at a time

    88

            this_page = 1

    89

    90

            # last line's timestamp

    91

            last_ts = None

    92

    93

            # found it yet?

    94

            found = False

    95

    96

        # count the full number of lines

    97

        line_count = 0

    98

    99

        # collect lines

   100

        lines = []

   101

   102

        # iterate using get_date

   103

        for line in self.get_date(dt) :

   104

            # count them

   105

            line_count += 1

   106

   107

            # skip?

   108

            if skip :

   109

                skip -= 1

   110

                continue

   111

   112

            # is this page all that we want/need?

   113

            if page or found :

   114

                # already full?

   115

                if len(lines) >= count :

   116

                    continue

   117

   118

            # specfic timestamp

   119

            else :

   120

                # didn't find it in this page?

   121

                if len(lines) >= count :

   122

                    # reset to next page

   123

                    lines = []

   124

                    this_page += 1

   125

   126

                # is dt between these two timestamps?

   127

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

   128

                    # found!

   129

                    found = True

   130

                    page = this_page

   131

   132

                else :

   133

                    # keep looking

   134

                    last_ts = line.timestamp

   135

   136

            # store line

   137

            lines.append(line)

   138

   139

        # calculate max_pages

   140

        max_pages = math.ceil(float(line_count) / count)

   141

   142

        # return

   143

        return (page, max_pages, lines)

   144

   145

    def get_month_days (self, dt) :

   146

        """

   147

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

   148

        """

   149

   150

        abstract

   152

class LogFile (object) :

   153

    """

   154

        A file containing LogEvents

   155

   156

        XXX: modify to implement LogSource?

   157

    """

   158

   160

        """

   166

        """

   167

   168

        # store

   169

        self.path = path

   170

        self.parser = parser

   171

        self.start_date = start_date

   173

        self.sep = sep

   174

   175

        # open

   176

        self.file = open(path, 'rb')

   177

   178

    def __iter__ (self) :

   179

        """

   180

            Yields a series of unicode lines, as read from the top of the file

   181

        """

   182

   183

        # seek to beginning

   184

        self.file.seek(0)

   185

   186

        # iterate over lines, decoding them as well

   188

   189

    def read_full (self) :

   190

        """

   191

            Reads all LogLines. The LogLines will have a valid offset.

   192

        """

   193

   194

        # just use our __iter__

   195

        return self.parser.parse_lines(self, self.start_date, starting_offset=1)

   196

   197

    def read_from (self, dt) :

   198

        """

   199

            Reads all LogLines from the given naive timestamp onwards

   200

        """

   201

   202

        # start reading at beginning

   203

        events = self.read_full()

   204

   205

        # skip unwanted events

   206

        for event in events :

   207

            if event.timestamp < dt :

   208

                continue

   209

   210

            else :

   211

                # include this line as well

   212

                yield event

   213

                break

   214

   215

        # yield the rest as-is

   216

        for event in events :

   217

            yield event

   218

   219

    def read_until (self, dt) :

   220

        """

   221

            Reads all LogLines up until the given naive timestamp

   222

        """

   223

   224

        # start reading events at the beginning

   225

        events = self.read_full()

   226

   227

        # yield events until we hit the given timestamp

   228

        for event in events :

   229

            if event.timestamp <= dt :

   230

                yield event

   231

   232

            else :

   233

                break

   234

   235

        # ignore the rest

   236

        return

   237

   238

    def _read_blocks_reverse (self, blocksize=1024) :

   239

        """

   240

            Yields blocks of file data in reverse order, starting at the end of the file

   241

        """

   242

   243

        # seek to end of file

   244

        self.file.seek(0, os.SEEK_END)

   245

   246

        # read offset

   247

        # XXX: hack -1 to get rid of trailing newline

   248

        size = offset = self.file.tell() - 1

   249

   250

        # do not try to read past the beginning of the file

   251

        while offset > 0:

   252

            # calc new offset + size

   253

            if offset > blocksize :

   254

                # full block

   255

                offset -= blocksize

   256

                read_size = blocksize

   257

   258

            else :

   259

                # partial block

   260

                read_size = offset

   261

                offset = 0

   262

   263

            # seek to offset

   264

            self.file.seek(offset)

   265

   266

            # read the data we want

   267

            block = self.file.read(read_size)

   268

   269

            # sanity check

   270

            assert len(block) == read_size

   271

   272

            # yield 

   273

            yield block

   274

   275

    def _read_lines_reverse (self) :

   276

        """

   277

            Yields decoded lines from the end of the file, in reverse order.

   278

        """

   279

   280

        # partial lines

   281

        buf = ''

   282

   283

        # read from end of file, a block at a time

   284

        for block in self._read_blocks_reverse() :

   285

            # add in our previous buf

   286

            buf = block + buf

   287

   288

            # split up lines

   289

            lines = buf.split(self.sep)

   290

   291

            # keep the first one as our buffer, as it's incomplete

   292

            buf = lines[0]

   293

   294

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   295

            # XXX: use something like islice, this has to build a slice object

   296

            for line in lines[:0:-1] :

   297

                yield line.decode(self.charset)

   298

   299

    def read_latest (self, count) :

   300

        """

   301

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   302

        """

   303

   304

        # the list of lines

   305

        lines = []

   306

   307

        # start reading lines into lines

   308

        for line in self._read_lines_reverse() :

   309

            # append

   310

            lines.append(line)

   311

   312

            # done?

   313

            if len(lines) >= count :

   314

                break

   315

   316

        # decode in reverse order, using our starting date....

   317

        # XXX: use lines[::-1] or reversed?

   318

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   319

        return self.parser.parse_lines(reversed(lines), self.start_date)

   320

   321

class LogDirectory (LogSource) :

   322

    """

   323

        A directory containing a series of timestamped LogFiles

   324

    """

   325

   327

        """

   328

            Load the logfiles at the given path.

   329

   332

        """

   333

   334

        # store

   335

        self.path = path

   336

        self.tz = tz

   337

        self.parser = parser

   339

        self.filename_fmt = filename_fmt

   340

   341

    def _get_logfile_datetime (self, dt) :

   342

        """

   343

            Get the logfile corresponding to the given datetime

   344

        """

   345

   346

        # convert to target timezone

   347

        dtz = dt.astimezone(self.tz)

   348

   349

        # convert to date and use that

   350

        return self._get_logfile_date(dtz.date())

   351

   352

    def _get_logfile_date (self, d, load=True) :

   353

        """

   354

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   355

            presence of the logfile, do not actually open it.

   356

   357

            Returns None if the logfile does not exist.

   358

        """

   359

   360

        # format filename

   361

        filename = d.strftime(self.filename_fmt)

   362

   363

        # build path

   364

        path = os.path.join(self.path, filename)

   365

   366

        try :

   367

            if load :

   368

                # open+return the LogFile

   370

   371

            else :

   372

                # test

   373

                return os.path.exists(path)

   374

   375

        # XXX: move to LogFile

   376

        except IOError, e :

   377

            # return None for missing files

   378

            if e.errno == errno.ENOENT :

   379

                return None

   380

   381

            else :

   382

                raise

   383

   384

    def _iter_date_reverse (self, dt=None) :

   385

        """

   386

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   387

            given *datetime*, or the the current date, if none given

   388

        """

   389

   390

        # default to now

   391

        if not dt :

   392

            dtz = self.tz.localize(datetime.datetime.now())

   393

   394

        else :

   395

            # convert to target timezone

   396

            dtz = dt.astimezone(self.tz)

   397

   398

        # our timedelta

   399

        ONE_DAY = datetime.timedelta(1)

   400

   401

        # iterate unto infinity

   402

        while True :

   403

            # yield

   404

            yield dtz.date()

   405

   406

            # one day sdrawkcab

   407

            dtz -= ONE_DAY

   408

   409

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   410

        """

   411

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   412

            current date, if none given.

   413

   414

            Reads/probes at most max_files files.

   415

        """

   416

   417

        # start counting at zero...

   418

        file_count = 0

   419

   420

        # have we found any files at all so far?

   421

        have_found = False

   422

   423

        # iterate backwards over days

   424

        for day in self._iter_date_reverse(dt) :

   425

            # stop if we've handled enough files by now

   426

            if file_count > max_files :

   427

                break

   428

   429

            # try and open the next logfile

   430

            logfile = None

   431

   432

            file_count += 1

   433

            logfile = self._get_logfile_date(day)

   434

   435

            # no logfile there?

   436

            if not logfile :

   437

                # hit our limit?

   438

                if file_count > max_files :

   439

                    # if we didn't find any logfiles at all, terminate rudely

   440

                    if not have_found :

   441

                        raise Exception("No recent logfiles found")

   442

   443

                    else :

   444

                        # stop looking, deal with what we've got

   445

                        return

   446

   447

                else :

   448

                    # skip to next day

   449

                    continue

   450

   451

            # mark have_found

   452

            have_found = True

   453

   454

            # yield it

   455

            yield logfile

   456

   457

    def get_latest (self, count) :

   458

        """

   459

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   460

        """

   461

   462

        # read the events into here

   463

        lines = []

   464

   465

        # start reading in those logfiles

   466

        for logfile in self._iter_logfile_reverse() :

   467

            # read the events

   468

            # XXX: use a queue

   469

            lines = list(logfile.read_latest(count)) + lines

   470

   471

            # done?

   472

            if len(lines) >= count :

   473

                break

   474

   475

        # return the events

   476

        return lines

   477

   478

    def get_date (self, dt) :

   479

        """

   480

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   481

            differs from our native datetime, this may involve lines from more than one logfile.

   482

        """

   483

   484

        # begin/end of 24h period, in target timezone

   485

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   486

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   487

   488

        # as dates

   489

        d_begin = dtz_begin.date() 

   490

        d_end = dtz_end.date()

   491

   492

#        print

   493

#        print "LogDirectory.get_date - %s" % dt

   494

#        print "\t   %s %s" % (d_begin, dtz_begin)

   495

#        print "\t-> %s %s" % (d_end, dtz_end)