1

"""

     2

    Full-text searching of logs

     3

"""

     4

     5

import datetime, calendar, pytz

     6

     7

import HyperEstraier as hype

     8

     9

import log_line

    10

    11

class LogSearchError (Exception) :

    12

    """

    13

        General search error

    14

    """

    15

    16

    pass

    17

    18

class NoResultsFound (LogSearchError) :

    19

    """

    20

        No results found

    21

    """

    22

    23

    pass

    24

    25

class LogSearchIndex (object) :

    26

    """

    27

        An index on the logs for a group of channels.

    28

    29

        This uses Hyper Estraier to handle searching, whereby each log line is a document (yes, I have a powerful server).

    30

    31

        These log documents have the following attributes:

    32

            @uri        - channel/date/line

    33

            @channel    - channel id

    34

            @type       - the LogType id

    35

            @timestamp  - UTC timestamp

    36

            @source     - nickname

    37

    38

        Each document then has a single line of data, which is the log message itself

    39

    """

    40

    41

    def __init__ (self, path, mode='r') :

    42

        """

    43

            Open the database, with the given mode:

    44

                r       - read-only

    45

                w       - read-write, create if not exists

    46

                a       - read-write, do not create

    47

                *       - read-write, truncate and create new

    48

        """

    49

    50

        # mapping of { mode -> flags }

    51

        mode_to_flag = {

    52

            'r':    hype.Database.DBREADER,

    53

            'w':    hype.Database.DBWRITER | hype.Database.DBCREAT,

    54

            'a':    hype.Database.DBWRITER,

    55

            '*':    hype.Database.DBWRITER | hype.Database.DBCREAT | hype.Database.DBTRUNC,

    56

        }

    57

    58

        # look up flags

    59

        flags = mode_to_flag[mode]

    60

    61

        # make instance

    62

        self.db = hype.Database()

    63

    64

        # open

    65

        if not self.db.open(path, flags) :

    66

            raise Exception("Index open failed: %s, mode=%s, flags=%#06x: %s" % (path, mode, flags, self.db.err_msg(self.db.error())))

    67

    68

    def insert (self, channel, lines) :

    69

        """

    70

            Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items

    71

        """

    72

    73

        # validate the LogChannel

    74

        assert channel.name

    75

    76

        count = 0

    77

    78

        # iterate

    79

        for line in lines :

    80

            # validate the LogLine

    81

            assert line.offset

    82

            assert line.timestamp

    83

    84

            # create new document

    85

            doc = hype.Document()

    86

    87

            # line date

    88

            date = line.timestamp.date()

    89

    90

            # convert to UTC timestamp

    91

            utc_timestamp = calendar.timegm(line.timestamp.utctimetuple())

    92

    93

            # ensure that it's not 1900

    94

            assert date.year != 1900

    95

    96

            # add URI

    97

            doc.add_attr('@uri',        "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset))

    98

    99

            # add channel id

   100

            doc.add_attr('@channel',    channel.id)

   101

   102

            # add type

   103

            doc.add_attr('@type',       str(line.type))

   104

   105

            # add UTC timestamp

   106

            doc.add_attr('@timestamp',  str(utc_timestamp))

   107

   108

            # add source attribute?

   109

            if line.source :

   110

                doc.add_attr('@source', str(line.source))

   111

   112

            # add data text

   113

            doc.add_text(line.data.encode('utf8'))

   114

   115

            # put

   116

            # XXX: what does this flag mean?

   117

            if not self.db.put_doc(doc, hype.Database.PDCLEAN) :

   118

                raise Exeception("Index put_doc failed")

   119

   120

            # count

   121

            count += 1

   122

   123

        # return

   124

        return count

   125

   126

    def search_cond (self, cond) :

   127

        """

   128

            Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results

   129

        """

   130

   131

        # execute search, unused 'flags' arg stays zero

   132

        results = self.db.search(cond, 0)

   133

   134

        # no results?

   135

        if not results :

   136

            raise NoResultsFound()

   137

   138

        # iterate over the document IDs

   139

        for doc_id in results :

   140

            # load document, this throws an exception...

   141

            # option constants are hype.Database.GDNOATTR/GDNOTEXT

   142

            doc = self.db.get_doc(doc_id, 0)

   143

   144

            # load the attributes/text

   145

            channel_id  = doc.attr('@channel')

   146

            type        = int(doc.attr('@type'))

   147

            timestamp   = datetime.datetime.fromtimestamp(int(doc.attr('@timestamp')), pytz.utc)

   148

            source      = doc.attr('@source')

   149

            data        = doc.cat_texts().decode('utf8')

   150

   151

            # build+yield to as LogLine

   152

            # XXX: ignore channel_id for now

   153

            yield log_line.LogLine(None, type, timestamp, source, data)

   154

   155

    def search (self, options=None, channel=None, phrase=None, order=None, max=None, skip=None) :

   156

        """

   157

            Search with flexible parameters

   158

   159

                options     - bitmask of hype.Condition.*

   160

                channel     - LogChannel object

   161

                phrase      - the search query phrase

   162

                order       - order attribute expression

   163

                max         - number of results to return

   164

                skip        - number of results to skip

   165

        """

   166

   167

        # build condition

   168

        cond = hype.Condition()

   169

   170

        if options :

   171

            # set options

   172

            cond.set_options(options)

   173

   174

        if channel :

   175

            # add channel attribute

   176

            cond.add_attr("@channel STREQ %s" % (channel.id, ))

   177

   178

        if phrase :

   179

            # add phrase

   180

            cond.set_phrase(phrase)

   181

   182

        if order :

   183

            # set order

   184

            cond.set_order(order)

   185

   186

        if max :

   187

            # set max

   188

            cond.set_max(max)

   189

   190

        if skip :

   191

            # set skip

   192

            cond.set_skip(skip)

   193

   194

        # execute

   195

        return self.search_cond(cond)

   196

   197

    def search_simple (self, channel, query, count=None, offset=None) :

   198

        """

   199

            Search for lines from the given channel for the given simple query

   200

        """

   201

   202

        # use search(), backwards

   203

        results = list(self.search(

   204

            # simplified phrase

   205

            options     = hype.Condition.SIMPLE,

   206

   207

            # specific channel

   208

            channel     = channel,

   209

   210

            # given phrase

   211

            phrase      = query,

   212

   213

            # order by timestamp

   214

            order       = "@timestamp NUMD",

   215

   216

            # count/offset

   217

            max         = count,

   218

            skip        = offset,

   219

        ))

   220

   221

        # reverse

   222

        return reversed(results)

   223