pngtile.store: split PNGTileApplication and PNGTileStore with lookup/list/open Image handling logic and a threadsafe cache
authorTero Marttila <terom@qmsk.net>
Sat, 04 Oct 2014 01:01:43 +0300
changeset 166 986052d7d0ce
parent 165 1dc09e81a4e2
child 167 16b600e927fe
pngtile.store: split PNGTileApplication and PNGTileStore with lookup/list/open Image handling logic and a threadsafe cache
pngtile/application.py
pngtile/store.py
--- a/pngtile/application.py	Fri Oct 03 23:56:25 2014 +0300
+++ b/pngtile/application.py	Sat Oct 04 01:01:43 2014 +0300
@@ -1,91 +1,67 @@
 from werkzeug import Request, Response, exceptions
-from werkzeug.utils import html
-import werkzeug.urls
 
 import pypngtile
-
-import os.path
-
-def url (server, *path, **args):
-    """
-        >>> url('http://foo/', 'bar, 'quux.png', x=5, y=2)
-        'http://foo/bar/quux.png?x=5&y=2'
-    """
-
-    return werkzeug.urls.Href(server)(*path, **args)
+import pngtile.store
 
-class BaseApplication (object):
-    IMAGE_TYPES = (
-        'png',
-    )
-
-    def __init__ (self, image_root):
-        if not os.path.isdir(image_root) :
-            raise Exception("Given image_root does not exist: {image_root}".format(image_root=image_root))
+class PNGTileApplication (pngtile.store.PNGTileStore):
+    """
+        Web application with a PNGTileStore.
+    """
+    
+    def list (self, url):
+        """
+            Yield names by request.url.
 
-        self.image_root = os.path.abspath(image_root)
-
-        self.image_cache = { }
-
-    def lookup_path (self, url):
-        """
-            Lookup image by request path.
-
-            Returns name, path, type. For dirs, type will be None.
+            Raises HTTPException
         """
 
-        if not os.path.isdir(self.image_root):
-            raise exceptions.InternalServerError("Server image_root has gone missing")
-    
-        # path to image
-        name = url.lstrip('/')
-        
-        # build absolute path
-        path = os.path.abspath(os.path.join(self.image_root, name))
+        try:
+            return super(PNGTileApplication, self).list(url)
+        except pngtile.store.NotFound as error:
+            raise exceptions.NotFound(str(error))
+        except pngtile.store.InvalidImage as error:
+            raise exceptions.BadRequest(str(error))
+   
+    def lookup (self, url):
+        """
+            Lookup neme, path, type by request.url.
 
-        # ensure the path points inside the data root
-        if not path.startswith(self.image_root):
-            raise exceptions.NotFound(name)
-        
-        if not os.path.exists(path):
-            raise exceptions.NotFound(name)
-        
-        # determine time
-        if os.path.isdir(path):
-            return name, path, None
-        else:
-            basename, type = path.rsplit('.', 1)
- 
-            return name, path, type
-
-    def get_image (self, url):
-        """
-            Return Image object.
+            Raises HTTPException
         """
 
-        name, path, type = self.lookup_path(url)
-
-        if type not in self.IMAGE_TYPES:
-            raise exceptions.BadRequest("Not a supported image: {name}: {type}".format(name=name, type=type))
-
-        # get Image object
-        image = self.image_cache.get(path)
+        try:
+            return super(PNGTileApplication, self).lookup(url)
+        except pngtile.store.Error as error:
+            raise exceptions.InternalServerError(str(error))
+        except pngtile.store.NotFound as error:
+            raise exceptions.NotFound(str(error))
+        except pngtile.store.InvalidImage  as error:
+            raise exceptions.BadRequest(str(error))
 
-        if not image:
-            # open
-            image = pypngtile.Image(path)
+    def open (self, url):
+        """
+            Return Image, name by request.url
 
-            # check
-            if image.status() not in (pypngtile.CACHE_FRESH, pypngtile.CACHE_STALE):
-                raise exceptions.InternalServerError("Image cache not available: {name}".format(name=name))
+            Raises HTTPException.
+        """
 
-            # load
-            image.open()
+        try:
+            return super(PNGTileApplication, self).open(url)
 
-            # cache
-            self.image_cache[path] = image
-        
-        return image, name
+        except pypngtile.Error as error:
+            raise exceptions.InternalServerError(str(error))
+
+        except pngtile.store.Error as error:
+            raise exceptions.InternalServerError(str(error))
+
+        except pngtile.store.NotFound as error:
+            raise exceptions.NotFound(str(error))
+
+        except pngtile.store.InvalidImage  as error:
+            raise exceptions.BadRequest(str(error))
+
+        except pngtile.store.UncachedImage as error:
+            raise exceptions.InternalServerError("Requested image has not yet been cached: {image}".format(image=error))
 
     def handle (self, request):
         """
@@ -106,28 +82,3 @@
         except exceptions.HTTPException as error:
             return error
 
-    STYLESHEETS = ( )
-    SCRIPTS = ( )
-
-    def render_html (self, title, body, stylesheets=None, scripts=None, end=()):
-        if stylesheets is None:
-            stylesheets = self.STYLESHEETS
-
-        if scripts is None:
-            scripts = self.SCRIPTS
-
-        return html.html(lang='en', *[
-            html.head(
-                html.title(title),
-                *[
-                    html.link(rel='stylesheet', href=href) for href in stylesheets
-                ]
-            ),
-            html.body(
-                *(body + tuple(
-                    html.script(src=src) for src in scripts
-                ) + end)
-            ),
-        ])
-
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pngtile/store.py	Sat Oct 04 01:01:43 2014 +0300
@@ -0,0 +1,153 @@
+
+import pypngtile
+
+import os.path
+import threading
+
+class Error (Exception):
+    pass
+
+class NotFound (KeyError):
+    pass
+
+class InvalidImage (Exception):
+    pass
+
+class UncachedImage (Exception):
+    pass
+
+class PNGTileStore (object):
+    """
+        Access pypngtile.Image's on a filesystem.
+        
+        Intended to be threadsafe for open()
+    """
+
+    IMAGE_TYPES = (
+        'png',
+    )
+
+    def __init__ (self, image_root):
+        if not os.path.isdir(image_root) :
+            raise Error("Given image_root does not exist: {image_root}".format(image_root=image_root))
+
+        self.image_root = os.path.abspath(image_root)
+        
+        # cache opened Images
+        self.images = { }
+        self.images_lock = threading.Lock()
+
+    def lookup (self, url):
+        """
+            Lookup image by request path.
+
+            Returns name, path, type. For dirs, type will be None.
+
+            Raises Error, NotFound, InvalidImage
+
+            Threadless.
+        """
+
+        if not os.path.isdir(self.image_root):
+            raise Error("Server image_root has gone missing")
+    
+        # path to image
+        name = url.lstrip('/')
+        
+        # build absolute path
+        path = os.path.abspath(os.path.join(self.image_root, name))
+
+        # ensure the path points inside the data root
+        if not path.startswith(self.image_root):
+            raise NotFound(name)
+        
+        if not os.path.exists(path):
+            raise NotFound(name)
+        
+        # determine time
+        if os.path.isdir(path):
+            return name, path, None
+        else:
+            basename, type = path.rsplit('.', 1)
+
+            if type not in self.IMAGE_TYPES:
+                raise InvalidImage(name)
+ 
+            return name, path, type
+
+    def list (self, url):
+        """
+            Yield a series of valid sub-names for the given directory:
+                foo.type
+                foo/
+            
+            The yielded items should be sorted before use.
+        """
+
+        name, root, type = self.lookup(url)
+
+        if type:
+            raise InvalidImage(name)
+
+        for name in os.listdir(root):
+            path = os.path.join(root, name)
+
+            # skip dotfiles
+            if name.startswith('.'):
+                continue
+            
+            # show dirs
+            if os.path.isdir(path):
+                if not os.access(path, os.R_OK):
+                    # skip inaccessible dirs
+                    continue
+
+                yield name + '/'
+
+            # examine ext
+            if '.' in name:
+                name_base, name_type = name.rsplit('.', 1)
+            else:
+                name_base = name
+                name_type = None
+
+            # show .png files with a .cache file
+            if name_type in self.IMAGE_TYPES and os.path.exists(os.path.join(root, name_base + '.cache')):
+                yield name
+
+    def open (self, url):
+        """
+            Return Image object for given URL.
+
+            Raises UncachedImage, pypngtile.Error
+
+            Threadsafe.
+        """
+
+        name, path, type = self.lookup(url)
+
+        if not type:
+            raise InvalidImage(name)
+
+        # get Image object
+        with self.images_lock:
+            image = self.images.get(path)
+
+        if not image:
+            # open
+            image = pypngtile.Image(path)
+
+            # check
+            if image.status() not in (pypngtile.CACHE_FRESH, pypngtile.CACHE_STALE):
+                raise UncachedImage(name)
+
+            # load
+            image.open()
+
+            # cache
+            with self.images_lock:
+                # we don't really care if some other thread raced us on the same Image and opened it up concurrently...
+                # this is just a cache
+                self.images[path] = image
+        
+        return image, name