1

"""

     2

    Full-text searching of logs

     3

"""

     4

     5

import datetime, calendar, pytz

     6

import os.path

     7

     8

import HyperEstraier as hype

     9

    10

import log_line

    11

    12

class LogSearchError (Exception) :

    13

    """

    14

        General search error

    15

    """

    16

    17

    pass

    18

    19

class NoResultsFound (LogSearchError) :

    20

    """

    21

        No results found

    22

    """

    23

    24

    pass

    25

    26

class LogSearchIndex (object) :

    27

    """

    28

        An index on the logs for a group of channels.

    29

    30

        This uses Hyper Estraier to handle searching, whereby each log line is a document (yes, I have a powerful server).

    31

    32

        These log documents have the following attributes:

    33

            @uri            - channel/date/line

    34

            channel         - channel code

    35

            type            - the LogType id

    36

            timestamp       - UTC timestamp

    37

            source_nickname - source nickname

    38

    39

        Each document then has a single line of data, which is the log message itself

    40

    """

    41

    42

    def __init__ (self, channels, path, mode='r') :

    43

        """

    44

            Open the database at the given path, with the given mode:

    45

                r       - read-only

    46

                w       - write, create if not exists

    47

                a       - write, error if not exists

    48

                c       - write, create, error if exists

    49

                *       - write, create, truncate if exists

    50

    51

            Channels is the ChannelList.

    52

        """

    53

    54

        # store

    55

        self.channels = channels

    56

        self.path = path

    57

        self.mode = mode

    58

    59

        # check it does not already exist?

    60

        if mode in 'c' and os.path.exists(path) :

    61

            raise LogSearchError("Index already exists: %s" % (path, ))

    62

    63

        # mapping of { mode -> flags }

    64

        mode_to_flag = {

    65

            'r':    hype.Database.DBREADER,

    66

            'w':    hype.Database.DBWRITER | hype.Database.DBCREAT,

    67

            'a':    hype.Database.DBWRITER,

    68

            'c':    hype.Database.DBWRITER | hype.Database.DBCREAT,

    69

            '*':    hype.Database.DBWRITER | hype.Database.DBCREAT | hype.Database.DBTRUNC,

    70

        }

    71

    72

        # look up flags

    73

        flags = mode_to_flag[mode]

    74

    75

        # make instance

    76

        self.db = hype.Database()

    77

    78

        # open

    79

        if not self.db.open(path, flags) :

    80

            raise Exception("Index open failed: %s, mode=%s, flags=%#06x: %s" % (path, mode, flags, self.db.err_msg(self.db.error())))

    81

    82

    def insert (self, channel, lines) :

    83

        """

    84

            Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items

    85

        """

    86

    87

        # validate the LogChannel

    88

        assert channel.name

    89

    90

        count = 0

    91

    92

        # iterate

    93

        for line in lines :

    94

            # validate the LogLine

    95

            assert line.offset

    96

            assert line.timestamp

    97

    98

            # create new document

    99

            doc = hype.Document()

   100

   101

            # line date

   102

            date = line.timestamp.date()

   103

   104

            # convert to UTC timestamp

   105

            utc_timestamp = calendar.timegm(line.timestamp.utctimetuple())

   106

   107

            # ensure that it's not 1900

   108

            assert date.year != 1900

   109

   110

            # add URI

   111

            doc.add_attr('@uri',        "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset))

   112

   113

            # add channel id

   114

            doc.add_attr('channel',     channel.id)

   115

   116

            # add type

   117

            doc.add_attr('type',        str(line.type))

   118

   119

            # add UTC timestamp

   120

            doc.add_attr('timestamp',   str(utc_timestamp))

   121

   122

            # add source attribute?

   123

            if line.source :

   124

                source_nickname, source_username, source_hostname, source_chanflags = line.source

   125

   126

                # XXX: handle source_nickname is None

   127

                if not source_nickname is None :

   128

                    source_nickname = str(source_nickname)

   129

   130

                doc.add_attr('source_nickname', source_nickname)

   131

   132

            # add data

   133

            if line.data :

   134

                doc.add_text(line.data.encode('utf8'))

   135

   136

            # put, "clean up dispensable regions of the overwritten document"

   137

            if not self.db.put_doc(doc, hype.Database.PDCLEAN) :

   138

                raise Exeception("Index put_doc failed")

   139

   140

            # count

   141

            count += 1

   142

   143

        # return

   144

        return count

   145

   146

    def search_cond (self, cond) :

   147

        """

   148

            Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results

   149

        """

   150

   151

        # execute search, unused 'flags' arg stays zero

   152

        results = self.db.search(cond, 0)

   153

   154

        # no results?

   155

        if not results :

   156

            raise NoResultsFound()

   157

   158

        # iterate over the document IDs

   159

        for doc_id in results :

   160

            # load document, this throws an exception...

   161

            # option constants are hype.Database.GDNOATTR/GDNOTEXT

   162

            doc = self.db.get_doc(doc_id, 0)

   163

   164

            # load the attributes/text

   165

            channel         = self.channels.lookup(doc.attr('channel'))

   166

            type            = int(doc.attr('type'))

   167

            timestamp       = datetime.datetime.fromtimestamp(int(doc.attr('timestamp')), pytz.utc)

   168

            source_nickname = doc.attr('source_nickname')

   169

            message         = doc.cat_texts().decode('utf8')

   170

   171

            # build+yield to as LogLine

   172

            yield log_line.LogLine(channel, None, type, timestamp, (source_nickname, None, None, None), None, message)

   173

   174

    def search (self, options=None, channel=None, phrase=None, order=None, max=None, skip=None) :

   175

        """

   176

            Search with flexible parameters

   177

   178

                options     - bitmask of hype.Condition.*

   179

                channel     - LogChannel object

   180

                phrase      - the search query phrase

   181

                order       - order attribute expression

   182

                max         - number of results to return

   183

                skip        - number of results to skip

   184

        """

   185

   186

        # build condition

   187

        cond = hype.Condition()

   188

   189

        if options :

   190

            # set options

   191

            cond.set_options(options)

   192

   193

        if channel :

   194

            # add channel attribute

   195

            cond.add_attr("@channel STREQ %s" % (channel.id, ))

   196

   197

        if phrase :

   198

            # add phrase

   199

            cond.set_phrase(phrase)

   200

   201

        if order :

   202

            # set order

   203

            cond.set_order(order)

   204

   205

        if max :

   206

            # set max

   207

            cond.set_max(max)

   208

   209

        if skip :

   210

            # set skip

   211

            cond.set_skip(skip)

   212

   213

        # execute

   214

        return self.search_cond(cond)

   215

   216

    def search_simple (self, channel, query, count=None, offset=None) :

   217

        """

   218

            Search for lines from the given channel for the given simple query

   219

        """

   220

   221

        # use search(), backwards

   222

        results = list(self.search(

   223

            # simplified phrase

   224

            options     = hype.Condition.SIMPLE,

   225

   226

            # specific channel

   227

            channel     = channel,

   228

   229

            # given phrase

   230

            phrase      = query,

   231

   232

            # order by timestamp

   233

            order       = "@timestamp NUMD",

   234

   235

            # count/offset

   236

            max         = count,

   237

            skip        = offset,

   238

        ))

   239

   240

        # reverse

   241

        return reversed(results)

   242