1

"""

     2

    A source of IRC log files

     3

"""

     4

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    23

            Get logs for the given date (as a datetime).

    24

        """

    25

    26

        abstract

    27

    28

    def get_date_paged (self, dt, count, page=None) :

    29

        """

    30

            Get the logs for a given date (as a datetime), divided into pages of count each. If page is given, the time

    31

            portion of the dt is ignored, and the lines for the given page are returned. Otherwise, if page is None,

    32

            then the lines for the page containing the given timestamp is returned.

    33

    34

            The return value is a (page, max, lines) tuple.

    35

        """

    36

    37

        # how to act?

    38

        if page :

    39

            # constant skip

    40

            skip = (page - 1) * count

    41

    42

        else :

    43

            skip = None

    44

    48

        # collect lines

    49

        lines = []

    50

    51

        # iterate using get_date

    52

        for line in self.get_date(dt) :

    56

            # skip?

    57

            if skip :

    58

                skip -= 1

    59

                continue

    65

            # store line

    66

            lines.append(line)

    73

    74

    def get_month_days (self, dt) :

    75

        """

    76

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

    77

        """

    78

    79

        abstract

    80

    81

class LogFile (object) :

    82

    """

    83

        A file containing LogEvents

    84

    85

        XXX: modify to implement LogSource?

    86

    """

    87

    88

    def __init__ (self, path, parser, charset, start_date=None, sep='\n') :

    89

        """

    90

            Open the file at the given path, which contains data with the given charset, as lines separated by the

    91

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

    92

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

    93

        """

    94

    95

        # store

    96

        self.path = path

    97

        self.parser = parser

    98

        self.start_date = start_date

    99

        self.charset = charset

   100

        self.sep = sep

   101

   102

        # open

   103

        self.file = open(path, 'rb')

   104

   105

    def __iter__ (self) :

   106

        """

   107

            Yields a series of unicode lines, as read from the top of the file

   108

        """

   109

   110

        # seek to beginning

   111

        self.file.seek(0)

   112

   113

        # iterate over lines, decoding them as well

   114

        return (line.decode(self.charset).rstrip(self.sep) for line in self.file)

   115

   116

    def read_full (self) :

   117

        """

   118

            Reads all LogLines. The LogLines will have a valid offset.

   119

        """

   120

   121

        # just use our __iter__

   122

        return self.parser.parse_lines(self, self.start_date, starting_offset=1)

   123

   124

    def read_from (self, dt) :

   125

        """

   126

            Reads all LogLines from the given naive timestamp onwards

   127

        """

   128

   129

        # start reading at beginning

   130

        events = self.read_full()

   131

   132

        # skip unwanted events

   133

        for event in events :

   134

            if event.timestamp < dt :

   135

                continue

   136

   137

            else :

   138

                # include this line as well

   139

                yield event

   140

                break

   141

   142

        # yield the rest as-is

   143

        for event in events :

   144

            yield event

   145

   146

    def read_until (self, dt) :

   147

        """

   148

            Reads all LogLines up until the given naive timestamp

   149

        """

   150

   151

        # start reading events at the beginning

   152

        events = self.read_full()

   153

   154

        # yield events until we hit the given timestamp

   155

        for event in events :

   156

            if event.timestamp <= dt :

   157

                yield event

   158

   159

            else :

   160

                break

   161

   162

        # ignore the rest

   163

        return

   164

   165

    def _read_blocks_reverse (self, blocksize=1024) :

   166

        """

   167

            Yields blocks of file data in reverse order, starting at the end of the file

   168

        """

   169

   170

        # seek to end of file

   171

        self.file.seek(0, os.SEEK_END)

   172

   173

        # read offset

   174

        # XXX: hack -1 to get rid of trailing newline

   175

        size = offset = self.file.tell() - 1

   176

   177

        # do not try to read past the beginning of the file

   178

        while offset > 0:

   179

            # calc new offset + size

   180

            if offset > blocksize :

   181

                # full block

   182

                offset -= blocksize

   183

                read_size = blocksize

   184

   185

            else :

   186

                # partial block

   187

                read_size = offset

   188

                offset = 0

   189

   190

            # seek to offset

   191

            self.file.seek(offset)

   192

   193

            # read the data we want

   194

            block = self.file.read(read_size)

   195

   196

            # sanity check

   197

            assert len(block) == read_size

   198

   199

            # yield 

   200

            yield block

   201

   202

    def _read_lines_reverse (self) :

   203

        """

   204

            Yields decoded lines from the end of the file, in reverse order.

   205

        """

   206

   207

        # partial lines

   208

        buf = ''

   209

   210

        # read from end of file, a block at a time

   211

        for block in self._read_blocks_reverse() :

   212

            # add in our previous buf

   213

            buf = block + buf

   214

   215

            # split up lines

   216

            lines = buf.split(self.sep)

   217

   218

            # keep the first one as our buffer, as it's incomplete

   219

            buf = lines[0]

   220

   221

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   222

            # XXX: use something like islice, this has to build a slice object

   223

            for line in lines[:0:-1] :

   224

                yield line.decode(self.charset)

   225

   226

    def read_latest (self, count) :

   227

        """

   228

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   229

        """

   230

   231

        # the list of lines

   232

        lines = []

   233

   234

        # start reading lines into lines

   235

        for line in self._read_lines_reverse() :

   236

            # append

   237

            lines.append(line)

   238

   239

            # done?

   240

            if len(lines) >= count :

   241

                break

   242

   243

        # decode in reverse order, using our starting date....

   244

        # XXX: use lines[::-1] or reversed?

   245

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   246

        return self.parser.parse_lines(reversed(lines), self.start_date)

   247

   248

class LogDirectory (LogSource) :

   249

    """

   250

        A directory containing a series of timestamped LogFiles

   251

    """

   252

   253

    def __init__ (self, path, tz, parser, charset, filename_fmt) :

   254

        """

   255

            Load the logfiles at the given path.

   256

   257

            The files contain data in the given charset, and are named according the the date in the given timezone and

   258

            date format, and will be parsed using the given parser.

   259

        """

   260

   261

        # store

   262

        self.path = path

   263

        self.tz = tz

   264

        self.parser = parser

   265

        self.charset = charset

   266

        self.filename_fmt = filename_fmt

   267

   268

    def _get_logfile_datetime (self, dt) :

   269

        """

   270

            Get the logfile corresponding to the given datetime

   271

        """

   272

   273

        # convert to target timezone

   274

        dtz = dt.astimezone(self.tz)

   275

   276

        # convert to date and use that

   277

        return self._get_logfile_date(dtz.date())

   278

   279

    def _get_logfile_date (self, d, load=True) :

   280

        """

   281

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   282

            presence of the logfile, do not actually open it.

   283

   284

            Returns None if the logfile does not exist.

   285

        """

   286

   287

        # format filename

   288

        filename = d.strftime(self.filename_fmt)

   289

   290

        # build path

   291

        path = os.path.join(self.path, filename)

   292

   293

        try :

   294

            if load :

   295

                # open+return the LogFile

   296

                return LogFile(path, self.parser, self.charset, d)

   297

   298

            else :

   299

                # test

   300

                return os.path.exists(path)

   301

   302

        # XXX: move to LogFile

   303

        except IOError, e :

   304

            # return None for missing files

   305

            if e.errno == errno.ENOENT :

   306

                return None

   307

   308

            else :

   309

                raise

   310

   311

    def _iter_date_reverse (self, dt=None) :

   312

        """

   313

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   314

            given *datetime*, or the the current date, if none given

   315

        """

   316

   317

        # default to now

   318

        if not dt :

   319

            dt = datetime.datetime.now(pytz.utc)

   320

   321

        # convert to target timezone

   322

        dtz = dt.astimezone(self.tz)

   323

   324

        # our timedelta

   325

        ONE_DAY = datetime.timedelta(1)

   326

   327

        # iterate unto infinity

   328

        while True :

   329

            # yield

   330

            yield dtz.date()

   331

   332

            # one day sdrawkcab

   333

            dtz -= ONE_DAY

   334

   335

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   336

        """

   337

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   338

            current date, if none given.

   339

   340

            Reads/probes at most max_files files.

   341

        """

   342

   343

        # start counting at zero...

   344

        file_count = 0

   345

   346

        # iterate backwards over days

   347

        for day in self._iter_date_reverse(dt) :

   348

            # stop if we've handled enough files by now

   349

            if file_count > max_files :

   350

                break

   351

   352

            # try and open the next logfile

   353

            logfile = None

   354

   355

            file_count += 1

   356

            logfile = self._get_logfile_date(day)

   357

   358

            # no logfile there?

   359

            if not logfile :

   360

                # if we didn't find any logfiles at all, terminate rudely

   361

                if file_count > max_files :

   362

                    raise Exception("No recent logfiles found")

   363

   364

                else :

   365

                    # skip to next day

   366

                    continue

   367

   368

            # yield it

   369

            yield logfile

   370

   371

    def get_latest (self, count) :

   372

        """

   373

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   374

        """

   375

   376

        # iterate over logfiles

   377

        iter = self._iter_logfile_reverse()

   378

   379

        # read the events into here

   380

        lines = []

   381

   382

        # loop until done

   383

        while len(lines) < count :

   384

            # next logfile

   385

            logfile = iter.next()

   386

   387

            # read the events

   388

            # XXX: use a queue

   389

            lines = list(logfile.read_latest(count)) + lines

   390

   391

        # return the events

   392

        return lines

   393

   394

    def get_date (self, dt) :

   395

        """

   396

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   397

            differs from our native datetime, this may involve lines from more than one logfile.

   398

        """

   399

   400

        # begin/end of 24h period, in target timezone

   401

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   402

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   403

   404

        # as dates

   405

        d_begin = dtz_begin.date() 

   406

        d_end = dtz_end.date()

   407

   408

#        print

   409

#        print "LogDirectory.get_date - %s" % dt

   410

#        print "\t   %s %s" % (d_begin, dtz_begin)

   411

#        print "\t-> %s %s" % (d_end, dtz_end)

   412

   413

        # if they're the same, just pull the full log for that date

   414

        if d_begin == d_end :

   415

            # open that log

   416

            logfile = self._get_logfile_date(d_begin)

   417

   418

            if not logfile :

   419

                raise Exception("No logfile for date=%r" % (dt, ))

   420

   421

            # return the full data

   422

            return logfile.read_full()

   423

   424

        # otherwise, we need to pull two partial logs

   425

        else :

   426

            # open both of them

   427

            f_begin = self._get_logfile_date(d_begin)

   428

            f_end = self._get_logfile_date(d_end)

   429

   430

            # chain together the two sources

   431

            return itertools.chain(

   432

                f_begin.read_from(dtz_begin), 

   433

                f_end.read_until(dtz_end) if f_end else []

   434

            )

   435

   436

    def get_month_days (self, month) :

   437

        """

   438

            Returns a set of dates for which logfiles are available in the given datetime's month

   439

        """

   440

   441

        # the set of days

   442

        days = set()

   443

   444

        # iterate over month's days using Calendar

   445

        for date in calendar.Calendar().itermonthdates(month.year, month.month) :

   446

            # convert date to target datetime

   447

            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)

   448

   449

            # date in our target timezone

   450

            log_date = dtz.date()

   451

   452

            # test for it

   453

            if self._get_logfile_date(log_date, load=False) :

   454

                # add to set

   455

                days.add(date)

   456

   457

        # return set

   458

        return days

   459