1

"""

     2

    A source of IRC log files

     3

"""

     4

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    24

        """

    25

    26

        abstract

    27

    64

    def get_month_days (self, dt) :

    65

        """

    66

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

    67

        """

    68

    69

        abstract

    71

class LogFile (object) :

    72

    """

    73

        A file containing LogEvents

    74

    75

        XXX: modify to implement LogSource?

    76

    """

    77

    78

    def __init__ (self, path, parser, charset, start_date=None, sep='\n') :

    79

        """

    80

            Open the file at the given path, which contains data with the given charset, as lines separated by the

    81

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

    82

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

    83

        """

    84

    85

        # store

    86

        self.path = path

    87

        self.parser = parser

    88

        self.start_date = start_date

    89

        self.charset = charset

    90

        self.sep = sep

    91

    92

        # open

    93

        self.file = open(path, 'rb')

    94

    95

    def __iter__ (self) :

    96

        """

    97

            Yields a series of unicode lines, as read from the top of the file

    98

        """

    99

   100

        # seek to beginning

   101

        self.file.seek(0)

   102

   103

        # iterate over lines, decoding them as well

   104

        return (line.decode(self.charset).rstrip(self.sep) for line in self.file)

   105

   106

    def read_full (self) :

   107

        """

   109

        """

   110

   111

        # just use our __iter__

   112

        return self.parser.parse_lines(self, self.start_date, starting_offset=1)

   113

   114

    def read_from (self, dt) :

   115

        """

   116

            Reads all LogLines from the given naive timestamp onwards

   117

        """

   118

   119

        # start reading at beginning

   120

        events = self.read_full()

   121

   122

        # skip unwanted events

   123

        for event in events :

   124

            if event.timestamp < dt :

   125

                continue

   126

   127

            else :

   128

                # include this line as well

   129

                yield event

   130

                break

   131

   132

        # yield the rest as-is

   133

        for event in events :

   134

            yield event

   135

   136

    def read_until (self, dt) :

   137

        """

   138

            Reads all LogLines up until the given naive timestamp

   139

        """

   140

   141

        # start reading events at the beginning

   142

        events = self.read_full()

   143

   144

        # yield events until we hit the given timestamp

   145

        for event in events :

   146

            if event.timestamp <= dt :

   147

                yield event

   148

   149

            else :

   150

                break

   151

   152

        # ignore the rest

   153

        return

   154

   155

    def _read_blocks_reverse (self, blocksize=1024) :

   156

        """

   157

            Yields blocks of file data in reverse order, starting at the end of the file

   158

        """

   159

   160

        # seek to end of file

   161

        self.file.seek(0, os.SEEK_END)

   162

   163

        # read offset

   164

        # XXX: hack -1 to get rid of trailing newline

   165

        size = offset = self.file.tell() - 1

   166

   167

        # do not try to read past the beginning of the file

   168

        while offset > 0:

   169

            # calc new offset + size

   170

            if offset > blocksize :

   171

                # full block

   172

                offset -= blocksize

   173

                read_size = blocksize

   174

   175

            else :

   176

                # partial block

   177

                read_size = offset

   178

                offset = 0

   179

   180

            # seek to offset

   181

            self.file.seek(offset)

   182

   183

            # read the data we want

   184

            block = self.file.read(read_size)

   185

   186

            # sanity check

   187

            assert len(block) == read_size

   188

   189

            # yield 

   190

            yield block

   191

   192

    def _read_lines_reverse (self) :

   193

        """

   194

            Yields decoded lines from the end of the file, in reverse order.

   195

        """

   196

   197

        # partial lines

   198

        buf = ''

   199

   200

        # read from end of file, a block at a time

   201

        for block in self._read_blocks_reverse() :

   202

            # add in our previous buf

   203

            buf = block + buf

   204

   205

            # split up lines

   206

            lines = buf.split(self.sep)

   207

   208

            # keep the first one as our buffer, as it's incomplete

   209

            buf = lines[0]

   210

   211

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   212

            # XXX: use something like islice, this has to build a slice object

   213

            for line in lines[:0:-1] :

   214

                yield line.decode(self.charset)

   215

   216

    def read_latest (self, count) :

   217

        """

   218

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   219

        """

   220

   221

        # the list of lines

   222

        lines = []

   223

   224

        # start reading lines into lines

   225

        for line in self._read_lines_reverse() :

   226

            # append

   227

            lines.append(line)

   228

   229

            # done?

   230

            if len(lines) >= count :

   231

                break

   232

   233

        # decode in reverse order, using our starting date....

   234

        # XXX: use lines[::-1] or reversed?

   235

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   236

        return self.parser.parse_lines(reversed(lines), self.start_date)

   237

   238

class LogDirectory (LogSource) :

   239

    """

   240

        A directory containing a series of timestamped LogFiles

   241

    """

   242

   243

    def __init__ (self, path, tz, parser, charset, filename_fmt) :

   244

        """

   245

            Load the logfiles at the given path.

   246

   247

            The files contain data in the given charset, and are named according the the date in the given timezone and

   248

            date format, and will be parsed using the given parser.

   249

        """

   250

   251

        # store

   252

        self.path = path

   253

        self.tz = tz

   254

        self.parser = parser

   255

        self.charset = charset

   256

        self.filename_fmt = filename_fmt

   257

   258

    def _get_logfile_datetime (self, dt) :

   259

        """

   260

            Get the logfile corresponding to the given datetime

   261

        """

   262

   263

        # convert to target timezone

   264

        dtz = dt.astimezone(self.tz)

   265

   266

        # convert to date and use that

   267

        return self._get_logfile_date(dtz.date())

   268

   269

    def _get_logfile_date (self, d, load=True) :

   270

        """

   271

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   272

            presence of the logfile, do not actually open it.

   273

   274

            Returns None if the logfile does not exist.

   275

        """

   276

   277

        # format filename

   278

        filename = d.strftime(self.filename_fmt)

   279

   280

        # build path

   281

        path = os.path.join(self.path, filename)

   282

   283

        try :

   284

            if load :

   285

                # open+return the LogFile

   286

                return LogFile(path, self.parser, self.charset, d)

   287

   288

            else :

   289

                # test

   290

                return os.path.exists(path)

   291

   292

        # XXX: move to LogFile

   293

        except IOError, e :

   294

            # return None for missing files

   295

            if e.errno == errno.ENOENT :

   296

                return None

   297

   298

            else :

   299

                raise

   300

   301

    def _iter_date_reverse (self, dt=None) :

   302

        """

   303

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   304

            given *datetime*, or the the current date, if none given

   305

        """

   306

   307

        # default to now

   308

        if not dt :

   309

            dt = datetime.datetime.now(pytz.utc)

   310

   311

        # convert to target timezone

   312

        dtz = dt.astimezone(self.tz)

   313

   314

        # our timedelta

   315

        ONE_DAY = datetime.timedelta(1)

   316

   317

        # iterate unto infinity

   318

        while True :

   319

            # yield

   320

            yield dtz.date()

   321

   322

            # one day sdrawkcab

   323

            dtz -= ONE_DAY

   324

   325

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   326

        """

   327

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   328

            current date, if none given.

   329

   330

            Reads/probes at most max_files files.

   331

        """

   332

   333

        # start counting at zero...

   334

        file_count = 0

   335

   336

        # iterate backwards over days

   337

        for day in self._iter_date_reverse(dt) :

   338

            # stop if we've handled enough files by now

   339

            if file_count > max_files :

   340

                break

   341

   342

            # try and open the next logfile

   343

            logfile = None

   344

   345

            file_count += 1

   346

            logfile = self._get_logfile_date(day)

   347

   348

            # no logfile there?

   349

            if not logfile :

   350

                # if we didn't find any logfiles at all, terminate rudely

   351

                if file_count > max_files :

   352

                    raise Exception("No recent logfiles found")

   353

   354

                else :

   355

                    # skip to next day

   356

                    continue

   357

   358

            # yield it

   359

            yield logfile

   360

   361

    def get_latest (self, count) :

   362

        """

   363

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   364

        """

   365

   366

        # iterate over logfiles

   367

        iter = self._iter_logfile_reverse()

   368

   369

        # read the events into here

   370

        lines = []

   371

   372

        # loop until done

   373

        while len(lines) < count :

   374

            # next logfile

   375

            logfile = iter.next()

   376

   377

            # read the events

   378

            # XXX: use a queue

   379

            lines = list(logfile.read_latest(count)) + lines

   380

   381

        # return the events

   382

        return lines

   383

   384

    def get_date (self, dt) :

   385

        """

   386

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   387

            differs from our native datetime, this may involve lines from more than one logfile.

   388

        """

   389

   390

        # begin/end of 24h period, in target timezone

   391

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   392

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   393

   394

        # as dates

   395

        d_begin = dtz_begin.date() 

   396

        d_end = dtz_end.date()

   397

   398

#        print

   399

#        print "LogDirectory.get_date - %s" % dt

   400

#        print "\t   %s %s" % (d_begin, dtz_begin)

   401

#        print "\t-> %s %s" % (d_end, dtz_end)

   402

   403

        # if they're the same, just pull the full log for that date

   404

        if d_begin == d_end :

   405

            # open that log

   406

            logfile = self._get_logfile_date(d_begin)

   407

   408

            if not logfile :

   409

                raise Exception("No logfile for date=%r" % (dt, ))

   410

   411

            # return the full data

   412

            return logfile.read_full()

   413

   414

        # otherwise, we need to pull two partial logs

   415

        else :

   416

            # open both of them

   417

            f_begin = self._get_logfile_date(d_begin)

   418

            f_end = self._get_logfile_date(d_end)

   420

            # chain together the two sources

   421

            return itertools.chain(

   422

                f_begin.read_from(dtz_begin), 

   423

                f_end.read_until(dtz_end) if f_end else []

   424

            )

   425

   426

    def get_month_days (self, month) :

   427

        """

   428

            Returns a set of dates for which logfiles are available in the given datetime's month

   429

        """

   430

   431

        # the set of days

   432

        days = set()

   433

   434

        # iterate over month's days using Calendar

   435

        for date in calendar.Calendar().itermonthdates(month.year, month.month) :

   436

            # convert date to target datetime

   437

            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)

   438

   439

            # date in our target timezone

   440

            log_date = dtz.date()

   441

   442

            # test for it

   443

            if self._get_logfile_date(log_date, load=False) :

   444

                # add to set

   445

                days.add(date)

   446

   447

        # return set

   448

        return days

   449