implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
authorTero Marttila <terom@fixme.fi>
Wed, 11 Feb 2009 03:46:59 +0200
changeset 99 8719ac564b22
parent 98 8c6e36849f9a
child 100 147be2910b02
implement non-blocking locking for the estdb, and our own locking for the autoload statetmpfile... it should work well now
log_search.py
scripts/search-index
--- a/log_search.py	Wed Feb 11 03:32:21 2009 +0200
+++ b/log_search.py	Wed Feb 11 03:46:59 2009 +0200
@@ -46,11 +46,16 @@
     def __init__ (self, channels, path, mode='r') :
         """
             Open the database at the given path, with the given mode:
-                r       - read, error if not exists
-                w       - write, create if not exists
-                a       - write, error if not exists
-                c       - write, create, error if exists
-                *       - write, create, truncate if exists
+                First char:
+                    r       - read, error if not exists
+                    w       - write, create if not exists
+                    a       - write, error if not exists
+                    c       - create, error if exists
+                
+                Additional chars:
+                    trunc   - truncate if exists
+                    +       - read as well as write
+                    ?       - non-blocking lock open, i.e. it fails if already open
             
             Channels is the ChannelList.
         """
@@ -69,13 +74,25 @@
             'r':    hype.Database.DBREADER,
             'w':    hype.Database.DBWRITER | hype.Database.DBCREAT,
             'a':    hype.Database.DBWRITER,
-            'c':    hype.Database.DBWRITER | hype.Database.DBCREAT,
-            '*':    hype.Database.DBWRITER | hype.Database.DBCREAT | hype.Database.DBTRUNC,
+            'c':    hype.Database.DBCREAT,
         }
 
-        # look up flags
-        flags = mode_to_flag[mode]
+        # flags to use, standard modes
+        flags = mode_to_flag[mode[0]]
+ 
+        # mode-flags
+        if '?' in mode :
+            # non-blocking locking
+            flags |= hype.Database.DBLCKNB
         
+        elif '+' in mode :
+            # read
+            flags |= hype.Database.DBREADER
+
+        elif 'trunc' in mode :
+            # truncate. Dangerous!
+            flags |= hype.Database.DBTRUNC
+       
         # make instance
         self.db = hype.Database()
         
--- a/scripts/search-index	Wed Feb 11 03:32:21 2009 +0200
+++ b/scripts/search-index	Wed Feb 11 03:46:59 2009 +0200
@@ -7,7 +7,7 @@
 # XXX: fix path
 import sys; sys.path.insert(0, '.'); sys.path.insert(0, '..')
 
-import os, os.path
+import os, os.path, fcntl
 import datetime, pytz
 
 # configuration and the LogSearchIndex module
@@ -147,7 +147,7 @@
     """
 
     # open index
-    index = _open_index(options, 'c' if not options.force else '*')
+    index = _open_index(options, 'ctrunc' if options.force else 'c')
 
     # that's all
     pass
@@ -261,8 +261,8 @@
         Automatically loads all channel logs that have not been indexed yet (by logfile mtime)
     """
     
-    # open index
-    index = _open_index(options, 'c' if options.create else 'a')
+    # open index, nonblocking
+    index = _open_index(options, 'c?' if options.create else 'a?')
 
     # default to all channels
     if not channel_names :
@@ -284,10 +284,12 @@
         statefile_tmppath = statefile_path + '.tmp'
         
         # do we have a tempfile from a previous crash?
-        # XXX: locking
         if os.path.exists(statefile_tmppath) :
-            # load after from it
+            # first, open it...
             statefile_tmp = open(statefile_tmppath, 'r+')
+
+            # ... then lock it
+            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
             
             # read after timestamp
             after_str = statefile_tmp.read().rstrip()
@@ -307,6 +309,9 @@
         else :
             # open new tempfile
             statefile_tmp = open(statefile_tmppath, 'w')
+            
+            # lock
+            fcntl.lockf(statefile_tmp, fcntl.LOCK_EX | fcntl.LOCK_NB)
 
         # override?
         if options.reload :