1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    23

            Get logs for the given date (as a datetime)

    24

        """

    25

    26

        abstract

    27

    28

    def get_month_days (self, dt) :

    29

        """

    30

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

    31

        """

    32

    33

        abstract

    34

    35

class LogFile (object) :

    36

    """

    37

        A file containing LogEvents

    38

    39

        XXX: modify to implement LogSource?

    40

    """

    41

    43

        """

    44

            Open the file at the given path, which contains data with the given charset, as lines separated by the

    45

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

    46

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

    47

        """

    48

    49

        # store

    50

        self.path = path

    51

        self.parser = parser

    52

        self.start_date = start_date

    53

        self.charset = charset

    54

        self.sep = sep

    55

    56

        # open

    57

        self.file = open(path, 'rb')

    58

    59

    def __iter__ (self) :

    60

        """

    61

            Yields a series of unicode lines, as read from the top of the file

    62

        """

    63

    64

        # seek to beginning

    65

        self.file.seek(0)

    66

    67

        # iterate over lines, decoding them as well

    68

        return (line.decode(self.charset).rstrip(self.sep) for line in self.file)

    69

    70

    def read_full (self) :

    71

        """

    72

            Reads all LogLines. The LogLines will have a valid offset

    73

        """

    74

    75

        # just use our __iter__

    76

        return self.parser.parse_lines(self, self.start_date, starting_offset=1)

    77

    78

    def read_from (self, dt) :

    79

        """

    80

            Reads all LogLines from the given naive timestamp onwards

    81

        """

    82

    83

        # start reading at beginning

    84

        events = self.read_full()

    85

    86

        # skip unwanted events

    87

        for event in events :

    88

            if event.timestamp < dt :

    89

                continue

    90

    91

            else :

    92

                # include this line as well

    93

                yield event

    94

                break

    95

    96

        # yield the rest as-is

    97

        for event in events :

    98

            yield event

    99

   100

    def read_until (self, dt) :

   101

        """

   102

            Reads all LogLines up until the given naive timestamp

   103

        """

   104

   105

        # start reading events at the beginning

   106

        events = self.read_full()

   107

   108

        # yield events until we hit the given timestamp

   109

        for event in events :

   110

            if event.timestamp <= dt :

   111

                yield event

   112

   113

            else :

   114

                break

   115

   116

        # ignore the rest

   117

        return

   118

   119

    def _read_blocks_reverse (self, blocksize=1024) :

   120

        """

   121

            Yields blocks of file data in reverse order, starting at the end of the file

   122

        """

   123

   124

        # seek to end of file

   125

        self.file.seek(0, os.SEEK_END)

   126

   127

        # read offset

   128

        # XXX: hack -1 to get rid of trailing newline

   129

        size = offset = self.file.tell() - 1

   130

   131

        # do not try to read past the beginning of the file

   132

        while offset > 0:

   133

            # calc new offset + size

   134

            if offset > blocksize :

   135

                # full block

   136

                offset -= blocksize

   137

                read_size = blocksize

   138

   139

            else :

   140

                # partial block

   141

                read_size = offset

   142

                offset = 0

   143

   144

            # seek to offset

   145

            self.file.seek(offset)

   146

   147

            # read the data we want

   148

            block = self.file.read(read_size)

   149

   150

            # sanity check

   151

            assert len(block) == read_size

   152

   153

            # yield 

   154

            yield block

   155

   156

    def _read_lines_reverse (self) :

   157

        """

   158

            Yields decoded lines from the end of the file, in reverse order.

   159

        """

   160

   161

        # partial lines

   162

        buf = ''

   163

   164

        # read from end of file, a block at a time

   165

        for block in self._read_blocks_reverse() :

   166

            # add in our previous buf

   167

            buf = block + buf

   168

   169

            # split up lines

   170

            lines = buf.split(self.sep)

   171

   172

            # keep the first one as our buffer, as it's incomplete

   173

            buf = lines[0]

   174

   175

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   176

            # XXX: use something like islice, this has to build a slice object

   177

            for line in lines[:0:-1] :

   178

                yield line.decode(self.charset)

   179

   180

    def read_latest (self, count) :

   181

        """

   182

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   183

        """

   184

   185

        # the list of lines

   186

        lines = []

   187

   188

        # start reading lines into lines

   189

        for line in self._read_lines_reverse() :

   190

            # append

   191

            lines.append(line)

   192

   193

            # done?

   194

            if len(lines) >= count :

   195

                break

   196

   197

        # decode in reverse order, using our starting date....

   198

        # XXX: use lines[::-1] or reversed?

   199

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   200

        return self.parser.parse_lines(reversed(lines), self.start_date)

   201

   202

class LogDirectory (LogSource) :

   203

    """

   204

        A directory containing a series of timestamped LogFiles

   205

    """

   206

   208

        """

   209

            Load the logfiles at the given path.

   210

   211

            The files contain data in the given charset, and are named according the the date in the given timezone and

   212

            date format, and will be parsed using the given parser.

   213

        """

   214

   215

        # store

   216

        self.path = path

   217

        self.tz = tz

   218

        self.parser = parser

   219

        self.charset = charset

   220

        self.filename_fmt = filename_fmt

   221

   222

    def _get_logfile_datetime (self, dt) :

   223

        """

   224

            Get the logfile corresponding to the given datetime

   225

        """

   226

   227

        # convert to target timezone

   228

        dtz = dt.astimezone(self.tz)

   229

   230

        # convert to date and use that

   231

        return self._get_logfile_date(dtz.date())

   232

   233

    def _get_logfile_date (self, d, load=True) :

   234

        """

   235

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   236

            presence of the logfile, do not actually open it.

   237

   238

            Returns None if the logfile does not exist.

   239

        """

   240

   241

        # format filename

   242

        filename = d.strftime(self.filename_fmt)

   243

   244

        # build path

   245

        path = os.path.join(self.path, filename)

   246

   247

        try :

   248

            if load :

   249

                # open+return the LogFile

   251

   252

            else :

   253

                # test

   254

                return os.path.exists(path)

   255

   256

        # XXX: move to LogFile

   257

        except IOError, e :

   258

            # return None for missing files

   259

            if e.errno == errno.ENOENT :

   260

                return None

   261

   262

            else :

   263

                raise

   264

   265

    def _iter_date_reverse (self, dt=None) :

   266

        """

   267

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   268

            given *datetime*, or the the current date, if none given

   269

        """

   270

   271

        # default to now

   272

        if not dt :

   273

            dt = datetime.datetime.now(pytz.utc)

   274

   275

        # convert to target timezone

   276

        dtz = dt.astimezone(self.tz)

   277

   278

        # our timedelta

   279

        ONE_DAY = datetime.timedelta(1)

   280

   281

        # iterate unto infinity

   282

        while True :

   283

            # yield

   284

            yield dtz.date()

   285

   286

            # one day sdrawkcab

   287

            dtz -= ONE_DAY

   288

   289

    def _iter_logfile_reverse (self, dt=None, max_files=100) :

   290

        """

   291

            Yields a series of LogFile objects, iterating backwards in time starting at the given datetime, or the

   292

            current date, if none given.

   293

   294

            Reads/probes at most max_files files.

   295

        """

   296

   297

        # start counting at zero...

   298

        file_count = 0

   299

   300

        # iterate backwards over days

   301

        for day in self._iter_date_reverse(dt) :

   302

            # stop if we've handled enough files by now

   303

            if file_count > max_files :

   304

                break

   305

   306

            # try and open the next logfile

   307

            logfile = None

   308

   309

            file_count += 1

   310

            logfile = self._get_logfile_date(day)

   311

   312

            # no logfile there?

   313

            if not logfile :

   314

                # if we didn't find any logfiles at all, terminate rudely

   315

                if file_count > max_files :

   316

                    raise Exception("No recent logfiles found")

   317

   318

                else :

   319

                    # skip to next day

   320

                    continue

   321

   322

            # yield it

   323

            yield logfile

   324

   325

    def get_latest (self, count) :

   326

        """

   327

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   328

        """

   329

   330

        # iterate over logfiles

   331

        iter = self._iter_logfile_reverse()

   332

   333

        # read the events into here

   334

        lines = []

   335

   336

        # loop until done

   337

        while len(lines) < count :

   338

            # next logfile

   339

            logfile = iter.next()

   340

   341

            # read the events

   342

            # XXX: use a queue

   343

            lines = list(logfile.read_latest(count)) + lines

   344

   345

        # return the events

   346

        return lines

   347

   348

    def get_date (self, dt) :

   349

        """

   350

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   351

            differs from our native datetime, this may involve lines from more than one logfile.

   352

        """

   353

   354

        # begin/end of 24h period, in target timezone

   355

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   356

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   357

   358

        # as dates

   359

        d_begin = dtz_begin.date() 

   360

        d_end = dtz_end.date()

   361

   362

#        print

   363

#        print "LogDirectory.get_date - %s" % dt

   364

#        print "\t   %s %s" % (d_begin, dtz_begin)

   365

#        print "\t-> %s %s" % (d_end, dtz_end)

   366

   367

        # if they're the same, just pull the full log for that date

   368

        if d_begin == d_end :

   369

            # open that log

   370

            logfile = self._get_logfile_date(d_begin)

   371

   372

            if not logfile :

   373

                raise Exception("No logfile for date=%r" % (dt, ))

   374

   375

            # return the full data

   376

            return logfile.read_full()

   377

   378

        # otherwise, we need to pull two partial logs

   379

        else :

   380

            # open both of them

   381

            f_begin = self._get_logfile_date(d_begin)

   382

            f_end = self._get_logfile_date(d_end)

   383

   384

            # chain together the two sources

   385

            return itertools.chain(

   386

                f_begin.read_from(dtz_begin), 

   387

                f_end.read_until(dtz_end) if f_end else []

   388

            )

   389

   390

    def get_month_days (self, month) :

   391

        """

   392

            Returns a set of dates for which logfiles are available in the given datetime's month

   393

        """

   394

   395

        # the set of days

   396

        days = set()

   397

   398

        # iterate over month's days using Calendar

   399

        for date in calendar.Calendar().itermonthdates(month.year, month.month) :

   400

            # convert date to target datetime

   401

            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)

   402

   403

            # date in our target timezone

   404

            log_date = dtz.date()

   405

   406

            # test for it

   407

            if self._get_logfile_date(log_date, load=False) :

   408

                # add to set

   409

                days.add(date)

   410

   411

        # return set

   412

        return days

   413