1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools, functools, math

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    23

            Get logs for the given date (as a datetime).

    24

        """

    25

    26

        abstract

    27

    28

    def get_date_paged (self, dt, count, page=None) :

    29

        """

    30

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    31

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    32

            then the lines for the page containing the given timestamp is returned.

    33

    34

            The return value is a (page, max, lines) tuple.

    35

        """

    36

    37

        # how to act?

    38

        if page :

    39

            # constant skip

    40

            skip = (page - 1) * count

    41

    42

        else :

    43

            skip = None

    44

    45

            # go through the logs a page at a time

    46

            this_page = 1

    47

    48

            # last line's timestamp

    49

            last_ts = None

    50

    51

            # found it yet?

    52

            found = False

    53

    54

        # count the full number of lines

    55

        line_count = 0

    56

    57

        # collect lines

    58

        lines = []

    59

    60

        # iterate using get_date

    61

        for line in self.get_date(dt) :

    62

            # count them

    63

            line_count += 1

    64

    65

            # skip?

    66

            if skip :

    67

                skip -= 1

    68

                continue

    69

    70

            # is this page all that we want/need?

    71

            if page or found :

    72

                # already full?

    73

                if len(lines) >= count :

    74

                    continue

    75

    76

            # specfic timestamp

    77

            else :

    78

                # didn't find it in this page?

    79

                if len(lines) >= count :

    80

                    # reset to next page

    81

                    lines = []

    82

                    this_page += 1

    83

    84

                # is dt between these two timestamps?

    85

                if (not last_ts or last_ts <= dt) and (dt <= line.timestamp) :

    86

                    # found!

    87

                    found = True

    88

                    page = this_page

    89

    90

                else :

    91

                    # keep looking

    92

                    last_ts = line.timestamp

    93

    94

            # store line

    95

            lines.append(line)

    96

    97

        # calculate max_pages

    98

        max_pages = math.ceil(float(line_count) / count)

    99

   100

        # return

   101

        return (page, max_pages, lines)

   102

   103

    def get_month_days (self, dt) :

   104

        """

   105

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

   106

        """

   107

   108

        abstract

   109

   110

class LogFile (object) :

   111

    """

   112

        A file containing LogEvents

   113

   114

        XXX: modify to implement LogSource?

   115

    """

   116

   117

    def __init__ (self, path, parser, charset, start_date=None, sep='\n') :

   118

        """

   119

            Open the file at the given path, which contains data with the given charset, as lines separated by the

   120

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

   121

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

   122

        """

   123

   124

        # store

   125

        self.path = path

   126

        self.parser = parser

   127

        self.start_date = start_date

   128

        self.charset = charset

   129

        self.sep = sep

   130

   131

        # open

   132

        self.file = open(path, 'rb')

   133

   134

    def __iter__ (self) :

   135

        """

   136

            Yields a series of unicode lines, as read from the top of the file

   137

        """

   138

   139

        # seek to beginning

   140

        self.file.seek(0)

   141

   142

        # iterate over lines, decoding them as well

   143

        return (line.decode(self.charset).rstrip(self.sep) for line in self.file)

   144

   145

    def read_full (self) :

   146

        """

   147

            Reads all LogLines. The LogLines will have a valid offset.

   148

        """

   149

   150

        # just use our __iter__

   151

        return self.parser.parse_lines(self, self.start_date, starting_offset=1)

   152

   153

    def read_from (self, dt) :

   154

        """

   155

            Reads all LogLines from the given naive timestamp onwards

   156

        """

   157

   158

        # start reading at beginning

   159

        events = self.read_full()

   160

   161

        # skip unwanted events

   162

        for event in events :

   163

            if event.timestamp < dt :

   164

                continue

   165

   166

            else :

   167

                # include this line as well

   168

                yield event

   169

                break

   170

   171

        # yield the rest as-is

   172

        for event in events :

   173

            yield event

   174

   175

    def read_until (self, dt) :

   176

        """

   177

            Reads all LogLines up until the given naive timestamp

   178

        """

   179

   180

        # start reading events at the beginning

   181

        events = self.read_full()

   182

   183

        # yield events until we hit the given timestamp

   184

        for event in events :

   185

            if event.timestamp <= dt :

   186

                yield event

   187

   188

            else :

   189

                break

   190

   191

        # ignore the rest

   192

        return

   193

   194

    def _read_blocks_reverse (self, blocksize=1024) :

   195

        """

   196

            Yields blocks of file data in reverse order, starting at the end of the file

   197

        """

   198

   199

        # seek to end of file

   200

        self.file.seek(0, os.SEEK_END)

   201

   202

        # read offset

   203

        # XXX: hack -1 to get rid of trailing newline

   204

        size = offset = self.file.tell() - 1

   205

   206

        # do not try to read past the beginning of the file

   207

        while offset > 0:

   208

            # calc new offset + size

   209

            if offset > blocksize :

   210

                # full block

   211

                offset -= blocksize

   212

                read_size = blocksize

   213

   214

            else :

   215

                # partial block

   216

                read_size = offset

   217

                offset = 0

   218

   219

            # seek to offset

   220

            self.file.seek(offset)

   221

   222

            # read the data we want

   223

            block = self.file.read(read_size)

   224

   225

            # sanity check

   226

            assert len(block) == read_size

   227

   228

            # yield 

   229

            yield block

   230

   231

    def _read_lines_reverse (self) :

   232

        """

   233

            Yields decoded lines from the end of the file, in reverse order.

   234

        """

   235

   236

        # partial lines

   237

        buf = ''

   238

   239

        # read from end of file, a block at a time

   240

        for block in self._read_blocks_reverse() :

   241

            # add in our previous buf

   242

            buf = block + buf

   243

   244

            # split up lines

   245

            lines = buf.split(self.sep)

   246

   247

            # keep the first one as our buffer, as it's incomplete

   248

            buf = lines[0]

   249

   250

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   251

            # XXX: use something like islice, this has to build a slice object

   252

            for line in lines[:0:-1] :

   253

                yield line.decode(self.charset)

   254

   255

    def read_latest (self, count) :

   256

        """

   257

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   258

        """

   259

   260

        # the list of lines

   261

        lines = []

   262

   263

        # start reading lines into lines

   264

        for line in self._read_lines_reverse() :

   265

            # append

   266

            lines.append(line)

   267

   268

            # done?

   269

            if len(lines) >= count :

   270

                break

   271

   272

        # decode in reverse order, using our starting date....

   273

        # XXX: use lines[::-1] or reversed?

   274

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   275

        return self.parser.parse_lines(reversed(lines), self.start_date)

   276

   277

class LogDirectory (LogSource) :

   278

    """

   279

        A directory containing a series of timestamped LogFiles

   280

    """

   281

   282

    def __init__ (self, path, tz, parser, charset, filename_fmt) :

   283

        """

   284

            Load the logfiles at the given path.

   285

   286

            The files contain data in the given charset, and are named according the the date in the given timezone and

   287

            date format, and will be parsed using the given parser.

   288

        """

   289

   290

        # store

   291

        self.path = path

   292

        self.tz = tz

   293

        self.parser = parser

   294

        self.charset = charset

   295

        self.filename_fmt = filename_fmt

   296

   297

    def _get_logfile_datetime (self, dt) :

   298

        """

   299

            Get the logfile corresponding to the given datetime

   300

        """

   301

   302

        # convert to target timezone

   303

        dtz = dt.astimezone(self.tz)

   304

   305

        # convert to date and use that

   306

        return self._get_logfile_date(dtz.date())

   307

   308

    def _get_logfile_date (self, d, load=True) :

   309

        """

   310

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   311

            presence of the logfile, do not actually open it.

   312

   313

            Returns None if the logfile does not exist.

   314

        """

   315

   316

        # format filename

   317

        filename = d.strftime(self.filename_fmt)

   318

   319

        # build path

   320

        path = os.path.join(self.path, filename)

   321

   322

        try :

   323

            if load :

   324

                # open+return the LogFile

   325

                return LogFile(path, self.parser, self.charset, d)

   326

   327

            else :

   328

                # test

   329

                return os.path.exists(path)

   330

   331

        # XXX: move to LogFile

   332

        except IOError, e :

   333

            # return None for missing files

   334

            if e.errno == errno.ENOENT :

   335

                return None

   336

   337

            else :

   338

                raise

   339

   340

    def _iter_date_reverse (self, dt=None) :

   341

        """

   342

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   343

            given *datetime*, or the the current date, if none given

   344

        """

   345

   346

        # default to now

   347

        if not dt :

   348

            dt = datetime.datetime.now(pytz.utc)

   349

   350

        # convert to target timezone

   351

        dtz = dt.astimezone(self.tz)

   352

   353

        # our timedelta

   354

        ONE_DAY = datetime.timedelta(1)

   355

   356

        # iterate unto infinity

   357

        while True :

   358

            # yield

   359

            yield dtz.date()

   360

   361

            # one day sdrawkcab

   362

            dtz -= ONE_DAY

   363

   364

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   365

        """

   366

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   367

            current date, if none given.

   368

   369

            Reads/probes at most max_files files.

   370

        """

   371

   372

        # start counting at zero...

   373

        file_count = 0

   374

   375

        # iterate backwards over days

   376

        for day in self._iter_date_reverse(dt) :

   377

            # stop if we've handled enough files by now

   378

            if file_count > max_files :

   379

                break

   380

   381

            # try and open the next logfile

   382

            logfile = None

   383

   384

            file_count += 1

   385

            logfile = self._get_logfile_date(day)

   386

   387

            # no logfile there?

   388

            if not logfile :

   389

                # if we didn't find any logfiles at all, terminate rudely

   390

                if file_count > max_files :

   391

                    raise Exception("No recent logfiles found")

   392

   393

                else :

   394

                    # skip to next day

   395

                    continue

   396

   397

            # yield it

   398

            yield logfile

   399

   400

    def get_latest (self, count) :

   401

        """

   402

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   403

        """

   404

   405

        # iterate over logfiles

   406

        iter = self._iter_logfile_reverse()

   407

   408

        # read the events into here

   409

        lines = []

   410

   411

        # loop until done

   412

        while len(lines) < count :

   413

            # next logfile

   414

            logfile = iter.next()

   415

   416

            # read the events

   417

            # XXX: use a queue

   418

            lines = list(logfile.read_latest(count)) + lines

   419

   420

        # return the events

   421

        return lines

   422

   423

    def get_date (self, dt) :

   424

        """

   425

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   426

            differs from our native datetime, this may involve lines from more than one logfile.

   427

        """

   428

   429

        # begin/end of 24h period, in target timezone

   430

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   431

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   432

   433

        # as dates

   434

        d_begin = dtz_begin.date() 

   435

        d_end = dtz_end.date()

   436

   437

#        print

   438

#        print "LogDirectory.get_date - %s" % dt

   439

#        print "\t   %s %s" % (d_begin, dtz_begin)

   440

#        print "\t-> %s %s" % (d_end, dtz_end)

   441

   442

        # if they're the same, just pull the full log for that date

   443

        if d_begin == d_end :

   444

            # open that log

   445

            logfile = self._get_logfile_date(d_begin)

   446

   447

            if not logfile :

   448

                raise Exception("No logfile for date=%r" % (dt, ))

   449

   450

            # return the full data

   451

            return logfile.read_full()

   452

   453

        # otherwise, we need to pull two partial logs

   454

        else :

   455

            # open both of them

   456

            f_begin = self._get_logfile_date(d_begin)

   457

            f_end = self._get_logfile_date(d_end)

   458

   459

            # chain together the two sources

   460

            return itertools.chain(

   461

                f_begin.read_from(dtz_begin), 

   462

                f_end.read_until(dtz_end) if f_end else []

   463

            )

   464

   465

    def get_month_days (self, month) :

   466

        """

   467

            Returns a set of dates for which logfiles are available in the given datetime's month

   468

        """

   469

   470

        # the set of days

   471

        days = set()

   472

   473

        # iterate over month's days using Calendar

   474

        for date in calendar.Calendar().itermonthdates(month.year, month.month) :

   475

            # convert date to target datetime

   476

            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)

   477

   478

            # date in our target timezone

   479

            log_date = dtz.date()

   480

   481

            # test for it

   482

            if self._get_logfile_date(log_date, load=False) :

   483

                # add to set

   484

                days.add(date)

   485

   486

        # return set

   487

        return days

   488