# HG changeset patch # User Tero Marttila # Date 1410697498 -10800 # Node ID 4e6e067b3472c9f81b356121333f471037dc3425 # Parent 305f6d5904405c92dfed4f6dcc276ad5a6807688# Parent aaae02944832630608158a4ba2ca96172cec257c merge diff -r aaae02944832 -r 4e6e067b3472 .hgignore --- a/.hgignore Sun Sep 14 15:19:59 2014 +0300 +++ b/.hgignore Sun Sep 14 15:24:58 2014 +0300 @@ -4,8 +4,9 @@ \.pyc$ ^build/ -^bin/ -^lib/ +^bin/pngtile(-static)?$ +^data +^lib/libpngtile.(so|a)$ ^doc/html ^dist -^Learning Diary\.pdf$ +^python/pypngtile.c$ diff -r aaae02944832 -r 4e6e067b3472 Makefile --- a/Makefile Sun Sep 14 15:19:59 2014 +0300 +++ b/Makefile Sun Sep 14 15:24:58 2014 +0300 @@ -1,12 +1,17 @@ # :set noexpandtab CFLAGS_ALL = -Wall -std=gnu99 +LDFLAGS_ALL = CFLAGS_DBG = -g CFLAGS_REL = -O2 +CFLAGS_PRF = -g -O2 -pg +LDFLAGS_PRF = -pg CFLAGS_SEL = ${CFLAGS_REL} +LDFLAGS_SEL = ${LDFLAGS_REL} # warnings, and use C99 with GNU extensions CFLAGS = ${CFLAGS_ALL} ${CFLAGS_SEL} +LDFLAGS = ${LDFLAGS_ALL} ${LDFLAGS_SEL} # preprocessor flags CPPFLAGS = -Iinclude -Isrc/ @@ -15,21 +20,30 @@ LOADLIBES = -lpng -lpthread # output name -DIST_NAME = 78949E-as2 -DIST_RESOURCES = README "Learning Diary.pdf" $(shell "echo python/*.{py,pyx}") +DIST_NAME = pngtile-${shell hg id -i} +DIST_DEPS = python/pypngtile.c +DIST_RESOURCES = README python/ pngtile/ static/ bin/ -all: depend lib/libpngtile.so bin/util +all: depend lib/libpngtile.so bin/pngtile lib/pypngtile.so lib/libpngtile.so : \ - build/obj/lib/ctx.o build/obj/lib/image.o build/obj/lib/cache.o build/obj/lib/tile.o build/obj/lib/error.o \ + build/obj/lib/ctx.o build/obj/lib/image.o build/obj/lib/cache.o build/obj/lib/tile.o build/obj/lib/png.o build/obj/lib/error.o \ + build/obj/shared/util.o build/obj/shared/log.o + +lib/libpngtile.a : \ + build/obj/lib/ctx.o build/obj/lib/image.o build/obj/lib/cache.o build/obj/lib/tile.o build/obj/lib/png.o build/obj/lib/error.o \ build/obj/shared/util.o build/obj/shared/log.o lib/pypngtile.so : \ lib/libpngtile.so -bin/util: \ - lib/libpngtile.so \ - build/obj/shared/log.o +bin/pngtile : \ + build/obj/pngtile/main.o \ + lib/libpngtile.so build/obj/shared/log.o + +bin/pngtile-static : \ + build/obj/pngtile/main.o \ + lib/libpngtile.a SRC_PATHS = $(wildcard src/*/*.c) SRC_NAMES = $(patsubst src/%,%,$(SRC_PATHS)) @@ -37,15 +51,18 @@ .PHONY : dirs clean depend dist +dist-clean : clean dirs + dirs: mkdir -p bin lib dist mkdir -p $(SRC_DIRS:%=build/deps/%) - mkdir -p $(SRC_DIRS:%=build/obj/%) + mkdir -p $(SRC_DIRS:%=build/obj/%) build/obj/python clean: rm -f build/obj/*/*.o build/deps/*/*.d - rm -f bin/* lib/*.so run/* - rm -rf dist/* + rm -f bin/pngtile bin/pngtile-static lib/*.so lib/*.a + rm -f pngtile/*.pyc + rm -f */.*.swp */*/.*.swp # .h dependencies depend: $(SRC_NAMES:%.c=build/deps/%.d) @@ -72,27 +89,30 @@ $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ # output binaries -bin/% : build/obj/%/main.o +bin/% : $(CC) $(LDFLAGS) $+ $(LOADLIBES) $(LDLIBS) -o $@ # output libraries lib/lib%.so : $(CC) -shared $(LDFLAGS) $+ $(LOADLIBES) $(LDLIBS) -o $@ -build/pyx/%.c : src/py/%.pyx +lib/lib%.a : + $(AR) rc $@ $+ + +python/%.c : python/%.pyx cython -o $@ $< -build/obj/py/%.o : build/pyx/%.c - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ +build/obj/python/%.o : python/%.c + $(CC) -c -fPIC -I/usr/include/python2.5 $(CPPFLAGS) $(CFLAGS) $< -o $@ -lib/py%.so : build/obj/py/%.o +lib/py%.so : build/obj/python/py%.o $(CC) -shared $(LDFLAGS) $+ $(LOADLIBES) $(LDLIBS) -o $@ -dist: +dist: $(DIST_DEPS) + rm -rf dist/$(DIST_NAME) mkdir -p dist/$(DIST_NAME) - cp -rv Makefile $(DIST_RESOURCES) src/ include/ dist/$(DIST_NAME)/ - rm dist/$(DIST_NAME)/src/*/.*.sw[op] - make -C dist/$(DIST_NAME) dirs + cp -rv Makefile $(DIST_RESOURCES) src/ include/ dist/$(DIST_NAME)/ + make -C dist/$(DIST_NAME) dist-clean tar -C dist -czvf dist/$(DIST_NAME).tar.gz $(DIST_NAME) @echo "*** Output at dist/$(DIST_NAME).tar.gz" diff -r aaae02944832 -r 4e6e067b3472 README --- a/README Sun Sep 14 15:19:59 2014 +0300 +++ b/README Sun Sep 14 15:24:58 2014 +0300 @@ -9,8 +9,6 @@ For this purpose, the library linearly decodes the PNG image to an uncompressed memory-mapped file, which can then be later used to encode a portion of this raw pixel data back into a PNG image. - Additionally, the library contains a thread pool for doing asynchronous tile render operations in parallel. - NOTES: The command-line utility is mainly intended for maintining the image caches and testing, primary usage is expected @@ -20,7 +18,12 @@ There is a separate project that provides a web-based tile viewer using Javascript (implemented in Python as a WSGI application). - The .cache files are not portable across different architectures. + The .cache files are not portable across different architectures, nor are they compatible across different cache + format versions. + + The library supports sparse cache files. A pixel-format byte pattern can be provided with --background using + hexadecimal notation (--background 0xFFFFFF - for 24bpp RGB white), and consecutive regions of that color will + be omitted in the cache file, which may provide significant gains in space efficiency. COMPILING: @@ -29,13 +32,13 @@ * libpng 1.2.15~beta5-3ubuntu0.1 * NPTL 2.7 (glibc 2.7-10ubuntu5) - The code was verified to compile and run on cc.hut.fi's Ubuntu computers, e.g. asterix.hut.fi. - - To compile, simply execute + To compile dist versions, simply execute make - The libpngtile.so will be placed under lib/, and the 'util' binary under bin/. + The libpngtile.so and pypngtile.so libraries will be placed under lib/, and the 'pngtile' binary under bin/. + + XXX: If compiling the pypngtile.so library with make fails, then `setup.py build_ext -i` should build it. USAGE: @@ -45,37 +48,29 @@ Provide any number of *.png paths as arguments to the ./bin/util command. Each will be opened, and automatically updated if the cache doesn't exist yet, or is stale: - ./bin/util -v data/*.png + pngtile -v data/*.png + + Use -v/--verbose for more detailed output. - To render a tile from some image, provide appropriate -W/-H and -x/-y options to ./bin/util: + + To render a tile from some image, provide appropriate -W/-H and -x/-y options to pngtile: - ./bin/util data/*.png -W 1024 -H 1024 -x 8000 -y 4000 + pngtile data/*.png -W 1024 -H 1024 -x 8000 -y 4000 The output PNG tiles will be written to temporary files, the names of which are shown in the [INFO] output. - To force-update an image's cache, use the -U/--force-update option: - ./bin/util --force-update data/*.png + pngtile --force-update data/*.png - To change the number of threads used for tile-render operations, use -j/--threads: - - time ./bin/util -q data/* -W 4096 -H 4096 -x 8000 -y 4000 -j 1 - > real 0m3.866s - - time ./bin/util -q data/* -W 4096 -H 4096 -x 8000 -y 4000 -j 4 - > real 0m1.463s + Alternatively, to not update an image's cache, use the -N/--no-update option. - (measured on an Intel Core 2 Duo, compiled without optimizations) - TODO/BUGS: At this stage, the library is primarily designed to handle a specific set of PNG images, and hence does not support all aspects of the PNG format, nor any other image formats. - Cache updated operations are not executed using the thread pool. - The pt_images opened by main() are not cleaned up before process exit, due to the asynchronous nature of the tile render operation's accesses to the underlying pt_cache object. diff -r aaae02944832 -r 4e6e067b3472 bin/dev-server --- a/bin/dev-server Sun Sep 14 15:19:59 2014 +0300 +++ b/bin/dev-server Sun Sep 14 15:24:58 2014 +0300 @@ -3,21 +3,30 @@ import wsgiref.simple_server import werkzeug from werkzeug.exceptions import NotFound +import memcache import pngtile.wsgi -# dispatch on URL -app = werkzeug.DispatcherMiddleware(pngtile.wsgi.application, { - '/static': werkzeug.SharedDataMiddleware(NotFound(), { - '/': 'static', - }), -}) +def main (host='0.0.0.0', port=8000, memcache_host='localhost:11211') : + print "Using memcache server at %s" % memcache_host -def main (host='0.0.0.0', port=8000) : + # cache + cache = memcache.Client([memcache_host]) + + # original app + app = pngtile.wsgi.WSGIApplication(cache) + + # serve up static content as well + app = werkzeug.SharedDataMiddleware(app, { + '/static': 'static', + }) + + # http server httpd = wsgiref.simple_server.make_server(host, port, app) print "Listening on %s:%d" % (host, port) - + + # go httpd.serve_forever() if __name__ == '__main__' : diff -r aaae02944832 -r 4e6e067b3472 bin/pngtile.fcgi --- a/bin/pngtile.fcgi Sun Sep 14 15:19:59 2014 +0300 +++ b/bin/pngtile.fcgi Sun Sep 14 15:24:58 2014 +0300 @@ -1,10 +1,10 @@ +#!/usr/bin/python + import flup.server.fcgi -def main (app, bind=None) : - """ - Run as a non-threaded single-process non-multiplexed FastCGI server - """ +import memcache +def run_fastcgi (app, bind=None) : # create WSGIServer server = flup.server.fcgi.WSGIServer(app, # try to supress threading @@ -25,8 +25,22 @@ # run... threads :( server.run() +def main (bind=None) : + """ + Run as a non-threaded single-process non-multiplexed FastCGI server + """ + + # open cache + cache = memcache.Client(['localhost:11211']) + + # build app + app = pngtile.wsgi.WSGIApplication(cache) + + # server + run_fastcgi(app, bind) + if __name__ == '__main__' : import pngtile.wsgi - main(pngtile.wsgi.application) + main() diff -r aaae02944832 -r 4e6e067b3472 bin/pypngtile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/pypngtile Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,202 @@ +#!/usr/bin/env python + +""" + Python clone of the pngtile binary. +""" + +import optparse, sys + +import pypngtile as pt + +# CLI options +options = None + +def log (fmt, args) : + print >>sys.stderr, fmt % args + +def log_debug (fmt, *args) : + if options.debug or options.verbose : + log(fmt, args) + +def log_info (fmt, *args) : + if not options.quiet : + log(fmt, args) + +def log_warn (fmt, *args) : + log(fmt, args) + + +def parse_hex (hex) : + """ + Parse a 0xHH.. style string as a raw bytestring + """ + + if not hex.startswith("0x") : + raise ValueError(hex) + + return hex[2:].decode("hex") + +def parse_args (args) : + """ + Parse given argv[1:] + """ + + global options + + parser = optparse.OptionParser( + usage = "Usage: %prog [options] [...]", + ) + + # build opts list + parser.add_option('-q', "--quiet", help="Supress informational output", action='store_true') + parser.add_option('-v', "--verbose", help="Display more output", action='store_true') + parser.add_option('-D', "--debug", help="Equivalent to -v", action='store_true') + parser.add_option('-U', "--force-update", help="Unconditionally update the image caches", action='store_true') + parser.add_option('-N', "--no-update", help="Do not update the image caches, even if unusable", action='store_true') + parser.add_option('-B', "--background", help="Background pattern for sparse cache file", metavar="0xHH..") + parser.add_option('-W', "--width", help="Output tile width", metavar="PX", type='int', default=800) + parser.add_option('-H', "--height", help="Output tile height", metavar="PX", type='int', default=600) + parser.add_option('-x', "--x", help="Tile x offset", metavar="PX", type='int', default=0) + parser.add_option('-y', "--y", help="Tile y offset", metavar="PX", type='int', default=0) + parser.add_option('-z', "--zoom", help="Tile zoom level, -n", metavar="ZL", type='int', default=0) + parser.add_option('-o', "--out", help="Render tile to output file, or -", metavar="FILE") + + # parse + options, args = parser.parse_args(args=args) + + # decode + if options.background : + try : + options.background = parse_hex(options.background) + + except ValueError : + raise ValueError("Invalid value for --background: %s" % options.background) + + return args + + +def process_tile (image) : + """ + Process output tile for given image + """ + + # parse out + if options.out == '-' : + log_debug("\tUsing stdout as output...") + + # use stdout + out = sys.stdout + + else : + log_debug("\tOpening file for output: %s", options.out) + + # open file + out = open(options.out, "wb") + + log_info("\tRender tile %dx%d@(%d:%d)@%d -> %s...", options.width, options.height, options.x, options.y, options.zoom, options.out) + + # render + image.tile_file(options.width, options.height, options.x, options.y, options.zoom, out) + + log_debug("Rendered tile: %s", options.out) + + +def process_image (image) : + """ + Process given image + """ + + # check cache status + status = image.status() + + # update if stale? + if status != pt.CACHE_FRESH or options.force_update : + # decode status + if status == pt.CACHE_NONE : + log_info("\tImage cache is missing") + + elif status == pt.CACHE_STALE : + log_info("\tImage cache is stale") + + elif status == pt.CACHE_INCOMPAT : + log_info("\tImage cache is incompatible") + + elif status == pt.CACHE_FRESH : + log_info("\tImage cache is fresh") + + else : + log_warn("\tImage cache status unknown: %d", status) + + + # update unless supressed + if not options.no_update : + log_info("\tUpdating image cache...") + + # update with optional background color + image.update(background_color=options.background) + + log_debug("\tUpdated image cache") + + else : + # warn + log_warn("\tSupressing cache update even though it is needed") + + else: + log_debug("\tImage cache is fresh") + + # open it + image.open() + + # show info + info = image.info() + + log_info("\tImage dimensions: %d x %d (%d bpp)", info['img_width'], info['img_height'], info['img_bpp']) + log_info("\tImage mtime=%d, bytes=%d", info['image_mtime'], info['image_bytes']) + log_info("\tCache mtime=%d, bytes=%d, blocks=%d (%d bytes), version=%d", + info['cache_mtime'], info['cache_bytes'], info['cache_blocks'], info['cache_blocks'] * 512, info['cache_version'] + ) + + + # render tile? + if options.out : + log_debug("\tRendering output tile") + + process_tile(image) + + else : + log_debug("\tNot rendering output tile") + +def process_images (image_paths) : + """ + Open up each image_path and process using process_image + """ + + for image_path in image_paths : + log_debug("Loading image: %s", image_path) + + # open up + image = pt.Image(image_path, pt.OPEN_UPDATE) + + log_info("Opened image: %s", image_path); + + # process + process_image(image) + + +def main (args) : + """ + Operate on given argv[1:] + """ + + # parse opts/args + args = parse_args(args) + + # handle each image + process_images(args) + + +if __name__ == '__main__' : + from sys import argv + + main(argv[1:]) + diff -r aaae02944832 -r 4e6e067b3472 bin/tornado-server --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/tornado-server Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,9 @@ +#!/usr/bin/python +""" + Script for running the Tornado server + handler +""" + +from pngtile import tornado_handler + +tornado_handler.main() + diff -r aaae02944832 -r 4e6e067b3472 include/pngtile.h --- a/include/pngtile.h Sun Sep 14 15:19:59 2014 +0300 +++ b/include/pngtile.h Sun Sep 14 15:24:58 2014 +0300 @@ -8,6 +8,8 @@ */ #include #include // for FILE* +#include +#include // for time_t /** * "Global" context shared between images @@ -21,7 +23,10 @@ /** Bitmask for pt_image_open modes */ enum pt_open_mode { - /** Update cache if needed */ + /** Open cache for read*/ + PT_OPEN_READ = 0x00, + + /** Open cache for update */ PT_OPEN_UPDATE = 0x01, /** Accept stale cache */ @@ -43,12 +48,44 @@ /** Cache exists, but is stale */ PT_CACHE_STALE = 2, + + /** Cache exists, but it was generated using an incompatible version of this library */ + PT_CACHE_INCOMPAT = 3, }; -/** Metadata info for image */ +/** Metadata info for image. Values will be set to zero if not available */ struct pt_image_info { - /** Dimensions of image */ - size_t width, height; + /** Dimensions of image. Only available if the cache is open */ + size_t img_width, img_height; + + /** Bits per pixel */ + size_t img_bpp; + + /** Last update of image file */ + time_t image_mtime; + + /** Size of image file in bytes */ + size_t image_bytes; + + /** Cache format version or -err */ + int cache_version; + + /** Last update of cache file */ + time_t cache_mtime; + + /** Size of cache file in bytes */ + size_t cache_bytes; + + /** Size of cache file in blocks (for sparse cache files) - 512 bytes / block? */ + size_t cache_blocks; +}; + +/** + * Modifyable params for update + */ +struct pt_image_params { + /** Don't write out any contiguous regions of this color. Left-aligned in whatever format the source image is in */ + uint8_t background_color[4]; }; /** @@ -88,15 +125,19 @@ /** * Open a new pt_image for use. * + * The pt_ctx is optional, but required for pt_image_tile_async. + * * @param img_ptr returned pt_image handle - * @param ctx global state to use + * @param ctx global state to use (optional) * @param path filesystem path to .png file * @param mode combination of PT_OPEN_* flags */ int pt_image_open (struct pt_image **image_ptr, struct pt_ctx *ctx, const char *png_path, int cache_mode); /** - * Get the image's metadata + * Get the image's metadata. + * + * XXX: return void, this never fails, just returns partial info */ int pt_image_info (struct pt_image *image, const struct pt_image_info **info_ptr); @@ -109,11 +150,15 @@ /** * Update the given image's cache. + * + * @param params optional parameters to use for the update process */ -int pt_image_update (struct pt_image *image); +int pt_image_update (struct pt_image *image, const struct pt_image_params *params); /** * Load the image's cache in read-only mode without trying to update it. + * + * Fails if the cache doesn't exist. */ // XXX: rename to pt_image_open? int pt_image_load (struct pt_image *image); @@ -122,6 +167,8 @@ * Render a PNG tile to a FILE*. * * The PNG data will be written to the given stream, which will be flushed, but not closed. + * + * Tile render operations are threadsafe as long as the pt_image is not modified during execution. */ int pt_image_tile_file (struct pt_image *image, const struct pt_tile_info *info, FILE *out); @@ -130,6 +177,8 @@ * * The PNG data will be written to a malloc'd buffer. * + * Tile render operations are threadsafe as long as the pt_image is not modified during execution. + * * @param image render from image's cache * @param info tile parameters * @param buf_ptr returned heap buffer @@ -144,6 +193,8 @@ * * This function may return before the PNG has been rendered. * + * Fails with PT_ERR if not pt_ctx was given to pt_image_open. + * * @param image render from image's cache. The cache must have been opened previously! * @param info tile parameters * @param out IO stream to write PNG data to, and close once done @@ -159,14 +210,20 @@ * Error codes returned */ enum pt_error { + /** No error */ PT_SUCCESS = 0, + + /** Generic error */ + PT_ERR = 1, + PT_ERR_MEM, PT_ERR_PATH, PT_ERR_OPEN_MODE, PT_ERR_IMG_STAT, - PT_ERR_IMG_FOPEN, + PT_ERR_IMG_OPEN, + PT_ERR_IMG_FORMAT, PT_ERR_PNG_CREATE, PT_ERR_PNG, @@ -180,13 +237,17 @@ PT_ERR_CACHE_TRUNC, PT_ERR_CACHE_MMAP, PT_ERR_CACHE_RENAME_TMP, - + PT_ERR_CACHE_VERSION, + PT_ERR_CACHE_MUNMAP, + PT_ERR_CACHE_CLOSE, + + PT_ERR_TILE_DIM, PT_ERR_TILE_CLIP, + PT_ERR_TILE_ZOOM, PT_ERR_PTHREAD_CREATE, PT_ERR_CTX_SHUTDOWN, - PT_ERR_ZOOM, PT_ERR_MAX, }; diff -r aaae02944832 -r 4e6e067b3472 pngtile/handlers.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pngtile/handlers.py Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,172 @@ +import os, os.path +import pypngtile as pt + +from werkzeug import Response, exceptions + +from pngtile import render + +# path to images +DATA_ROOT = os.environ.get("PNGTILE_DATA_PATH") or os.path.abspath('data/') + +# only open each image once +IMAGE_CACHE = {} + +### Parse request data +def get_path (req_path) : + """ + Returns the name and path requested + """ + + # check DATA_ROOT exists.. + if not os.path.isdir(DATA_ROOT) : + raise exceptions.InternalServerError("Missing DATA_ROOT") + + # path to image + image_name = req_path.lstrip('/') + + # build absolute path + image_path = os.path.abspath(os.path.join(DATA_ROOT, image_name)) + + # ensure the path points inside the data root + if not image_path.startswith(DATA_ROOT) : + raise exceptions.NotFound(image_name) + + return image_name, image_path + +def get_image (name, path) : + """ + Gets an Image object from the cache, ensuring that the cached is available + """ + + # get Image object + if path in IMAGE_CACHE : + # get from cache + image = IMAGE_CACHE[path] + + else : + # open + image = pt.Image(path) + + # check + if image.status() not in (pt.CACHE_FRESH, pt.CACHE_STALE) : + raise exceptions.InternalServerError("Image cache not available: %s" % name) + + # load + image.open() + + # cache + IMAGE_CACHE[path] = image + + return image + + +### Handle werkzeug.Request objects -> werkzeug.Response +def handle_dir (req, name, path) : + """ + Handle request for a directory + """ + + prefix = req.script_root + + return Response(render.dir_html(prefix, name, path), content_type="text/html") + + + +def handle_img_viewport (req, image, name) : + """ + Handle request for image viewport + """ + + prefix = req.script_root + + # viewport + return Response(render.img_html(prefix, name, image), content_type="text/html") + + +def handle_img_region (req, image, cache) : + """ + Handle request for an image region + """ + + # specific image + width = int(req.args['w']) + height = int(req.args['h']) + cx = int(req.args['cx']) + cy = int(req.args['cy']) + zoom = int(req.args.get('zl', "0")) + + try : + # yay full render + return Response(render.img_png_region(image, cx, cy, zoom, width, height, cache), content_type="image/png") + + except ValueError, ex : + # too large + raise exceptions.Forbidden(str(ex)) + + +def handle_img_tile (req, image, cache) : + """ + Handle request for image tile + """ + + # tile + x = int(req.args['x']) + y = int(req.args['y']) + zoom = int(req.args.get('zl', "0")) + + # cache? + + # yay render + return Response(render.img_png_tile(image, x, y, zoom, cache), content_type="image/png") + +## Dispatch req to handle_img_* +def handle_img (req, name, path, cache) : + """ + Handle request for an image + """ + + # get image object + image = get_image(name, path) + + # what view? + if not req.args : + return handle_img_viewport(req, image, name) + + elif 'w' in req.args and 'h' in req.args and 'cx' in req.args and 'cy' in req.args : + return handle_img_region(req, image, cache) + + elif 'x' in req.args and 'y' in req.args : + return handle_img_tile(req, image, cache) + + else : + raise exceptions.BadRequest("Unknown args") + + + +## Dispatch request to handle_* +def handle_req (req, cache) : + """ + Main request handler + """ + + # decode req + name, path = get_path(req.path) + + # determine dir/image + if os.path.isdir(path) : + # directory + return handle_dir(req, name, path) + + elif not os.path.exists(path) : + # no such file + raise exceptions.NotFound(name) + + elif not name or not name.endswith('.png') : + # invalid file + raise exceptions.BadRequest("Not a PNG file") + + else : + # image + return handle_img(req, name, path, cache) + + diff -r aaae02944832 -r 4e6e067b3472 pngtile/render.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pngtile/render.py Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,245 @@ +""" + Rendering output +""" + +import os, os.path + +## Settings +# width of a tile +TILE_WIDTH = 256 +TILE_HEIGHT = 256 + +# max. output resolution to allow +MAX_PIXELS = 1920 * 1200 + +def dir_url (prefix, name, item) : + """ + Join together an absolute URL prefix, an optional directory name, and a directory item + """ + + url = prefix + + if name : + url += '/' + name + + url += '/' + item + + return url + +def dir_list (dir_path) : + """ + Yield a series of directory items to show for the given dir + """ + + # link to parent + yield '..' + + for item in os.listdir(dir_path) : + path = os.path.join(dir_path, item) + + # skip dotfiles + if item.startswith('.') : + continue + + # show dirs + if os.path.isdir(path) : + yield item + + # examine ext + base, ext = os.path.splitext(path) + + # show .png files with a .cache file + if ext == '.png' and os.path.exists(base + '.cache') : + yield item + + +### Render HTML data +def dir_html (prefix, name, path) : + """ + Directory index + """ + + name = name.rstrip('/') + + return """\ + + + Index of %(dir)s + + + +

Index of %(dir)s

+ +
    +%(listing)s +
+ +""" % dict( + prefix = prefix, + dir = '/' + name, + + listing = "\n".join( + #
  • link + """
  • %(name)s
  • """ % dict( + # URL to dir + url = dir_url(prefix, name, item), + + # item name + name = item, + ) for item in dir_list(path) + ), + ) + +def img_html (prefix, name, image) : + """ + HTML for image + """ + + # a little slow, but not so bad - two stats(), heh + info = image.info() + img_width, img_height = info['img_width'], info['img_height'] + + return """\ + + + %(title)s + + + + + + + +
    +
    +
    + + + +
    + +
    + +
    + Loading... +
    +
    +
    + + + +""" % dict( + title = name, + prefix = prefix, + tile_url = prefix + '/' + name, + + tile_width = TILE_WIDTH, + tile_height = TILE_HEIGHT, + + img_width = img_width, + img_height = img_height, + ) + + +# threshold to cache images on - only images with a source data region *larger* than this are cached +CACHE_THRESHOLD = 512 * 512 + +def scale_by_zoom (val, zoom) : + """ + Scale dimension by zoom factor + + zl > 0 -> bigger + zl < 0 -> smaller + """ + + if zoom > 0 : + return val << zoom + + elif zoom > 0 : + return val >> -zoom + + else : + return val + +### Image caching +def check_cache_threshold (width, height, zl) : + """ + Checks if a tile with the given dimensions should be cached + """ + + return (scale_by_zoom(width, zl) * scale_by_zoom(height, zl)) > CACHE_THRESHOLD + +def render_raw (image, width, height, x, y, zl) : + """ + Render and return tile + """ + + return image.tile_mem( + width, height, + x, y, zl + ) + +def render_cache (cache, image, width, height, x, y, zl) : + """ + Perform a cached render of the given tile + """ + + if cache : + # cache key + key = "tl_%d:%d_%d:%d:%d_%s" % (x, y, width, height, zl, image.path) + + # lookup + data = cache.get(key) + + else : + # oops, no cache + data = None + + if not data : + # cache miss, render + data = render_raw(image, width, height, x, y, zl) + + if cache : + # store + cache.add(key, data) + + # ok + return data + +### Render PNG Data +def img_png_tile (image, x, y, zoom, cache) : + """ + Render given tile, returning PNG data + """ + + # remap coordinates by zoom + x = scale_by_zoom(x, zoom) + y = scale_by_zoom(y, zoom) + + # do we want to cache this? + if check_cache_threshold(TILE_WIDTH, TILE_HEIGHT, zoom) : + # go via the cache + return render_cache(cache, image, TILE_WIDTH, TILE_HEIGHT, x, y, zoom) + + else : + # just go raw + return render_raw(image, TILE_WIDTH, TILE_HEIGHT, x, y, zoom) + +def img_png_region (image, cx, cy, zoom, width, height, cache) : + """ + Render arbitrary tile, returning PNG data + """ + + x = scale_by_zoom(cx - width / 2, zoom) + y = scale_by_zoom(cy - height / 2, zoom) + + # safely limit + if width * height > MAX_PIXELS : + raise ValueError("Image size: %d * %d > %d" % (width, height, MAX_PIXELS)) + + # these images are always cached + return render_cache(cache, image, width, height, x, y, zoom) + diff -r aaae02944832 -r 4e6e067b3472 pngtile/tornado_handler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pngtile/tornado_handler.py Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,108 @@ +""" + A tornado-based HTTP app +""" + +import tornado.web +import tornado.httpserver +import tornado.wsgi +import werkzeug + +from pngtile import handlers + +class MainHandler (tornado.web.RequestHandler) : + """ + Main handler for the / URL, pass off requests to werkzeug-based handlers... + """ + + def build_environ (self, path) : + """ + Yield a series of (key, value) pairs suitable for use with WSGI + """ + + request = self.request + + hostport = request.host.split(":") + + if len(hostport) == 2: + host = hostport[0] + port = int(hostport[1]) + else: + host = request.host + port = 443 if request.protocol == "https" else 80 + + yield "REQUEST_METHOD", request.method + yield "SCRIPT_NAME", "" + yield "PATH_INFO", path + yield "QUERY_STRING", request.query + yield "SERVER_NAME", host + yield "SERVER_PORT", port + + yield "wsgi.version", (1, 0) + yield "wsgi.url_scheme", request.protocol + + yield "CONTENT_TYPE", request.headers.get("Content-Type") + yield "CONTENT_LENGTH", request.headers.get("Content-Length") + + for key, value in request.headers.iteritems(): + yield "HTTP_" + key.replace("-", "_").upper(), value + + def get (self, path) : + environ = dict(self.build_environ(path)) + + # build Request + request = werkzeug.Request(environ) + + # handle + try : + response = handlers.handle_req(request) + + except werkzeug.exceptions.HTTPException, ex : + response = ex + + # return + def start_response (_status, _headers) : + status = int(_status.split()[0]) + headers = _headers + + self.set_status(status) + + for name, value in headers : + self.set_header(name, value) + + # invoke Response + data = response(environ, start_response) + + # output data + for chunk in data : + self.write(chunk) + +def build_app () : + return tornado.web.Application([ + # static, from $CWD/static/ + (r"/static/(.*)", tornado.web.StaticFileHandler, dict(path = "static/")), + + # dir listings, image html, PNG tiles + (r"(/.*)", MainHandler), + ]) + +def build_httpserver (app, port) : + server = tornado.httpserver.HTTPServer(app) + server.listen(port) + + return server + +def main (port=8000) : + """ + Build the app, http server and run the main loop + """ + + import logging + + logging.basicConfig(level=logging.DEBUG) + + app = build_app() + server = build_httpserver(app, port) + + tornado.ioloop.IOLoop.instance().start() + + diff -r aaae02944832 -r 4e6e067b3472 pngtile/wsgi.py --- a/pngtile/wsgi.py Sun Sep 14 15:19:59 2014 +0300 +++ b/pngtile/wsgi.py Sun Sep 14 15:24:58 2014 +0300 @@ -2,190 +2,37 @@ Our WSGI web interface, which can serve the JS UI and any .png tiles via HTTP. """ -from werkzeug import Request, Response, responder +from werkzeug import Request, responder from werkzeug import exceptions -import os.path - -import pypngtile as pt - -DATA_ROOT = os.path.abspath('data') -IMAGE_CACHE = {} - -TILE_WIDTH = 256 -TILE_HEIGHT = 256 - -def dir_view (req, name, path) : - prefix = os.path.dirname(req.script_root).rstrip('/') - name = name.rstrip('/') +from pngtile import handlers - return """\ - - - Index of %(dir)s - - - -

    Index of %(dir)s

    - -
      -%(listing)s -
    - -""" % dict( - prefix = prefix, - dir = name, - - listing = "\n".join( - """
  • %(name)s
  • """ % dict( - url = '/'.join((prefix, name, item)), - name = item, - ) for item in ['..'] + os.listdir(path) - ), - ) - -def image_view (req, image_path, image) : - image_name = os.path.basename(image_path) - - return """\ - - - %(title)s - - - - - - - -
    -
    -
    - - - -
    - -
    -
    -
    - - - -""" % dict( - title = image_name, - prefix = os.path.dirname(req.script_root).rstrip('/'), - tile_url = req.url, - - tile_width = TILE_WIDTH, - tile_height = TILE_HEIGHT, - ) - -def scale_by_zoom (val, zoom) : - if zoom > 0 : - return val << zoom - - elif zoom > 0 : - return val >> -zoom - - else : - return val - -def render_tile (image, x, y, zoom, width=TILE_WIDTH, height=TILE_HEIGHT) : - return image.tile_mem( - width, height, - scale_by_zoom(x, -zoom), scale_by_zoom(y, -zoom), - zoom - ) +class WSGIApplication (object) : + """ + Simple WSGI application invoking the werkzeug handlers + """ -def render_image (image, cx, cy, zoom, width, height) : - x = scale_by_zoom(cx - width / 2, -zoom) - y = scale_by_zoom(cy - height / 2, -zoom) - - return image.tile_mem( - width, height, - x, y, - zoom - ) - -def handle_main (req) : - # path to image - image_name = req.path.lstrip('/') - - # build absolute path - image_path = os.path.abspath(os.path.join(DATA_ROOT, image_name)) - - # ensure the path points inside the data root - if not image_path.startswith(DATA_ROOT) : - raise exceptions.NotFound(image_name) - - - if os.path.isdir(image_path) : - return Response(dir_view(req, image_name, image_path), content_type="text/html") - - elif not os.path.exists(image_path) : - raise exceptions.NotFound(image_name) - - elif not image_name or not image_name.endswith('.png') : - raise exceptions.BadRequest("Not a PNG file") - - - # get Image object - if image_path in IMAGE_CACHE : - # get from cache - image = IMAGE_CACHE[image_path] - - else : - # ensure exists - if not os.path.exists(image_path) : - raise exceptions.NotFound(image_name) + def __init__ (self, cache=None) : + """ + Use given cache if any + """ - # cache - image = IMAGE_CACHE[image_path] = pt.Image(image_path) - - if image.status() == pt.CACHE_NONE : - raise exceptions.InternalServerError("Image not cached: " + image_name) - - # what view? - if not req.args : - # viewport - return Response(image_view(req, image_path, image), content_type="text/html") - - elif 'w' in req.args and 'h' in req.args and 'cx' in req.args and 'cy' in req.args : - # specific image - width = int(req.args['w']) - height = int(req.args['h']) - cx = int(req.args['cx']) - cy = int(req.args['cy']) - zoom = int(req.args.get('zl', "0")) - - # yay full render - return Response(render_image(image, cx, cy, zoom, width, height), content_type="image/png") + self.cache = cache - elif 'x' in req.args and 'y' in req.args : - # tile - x = int(req.args['x']) - y = int(req.args['y']) - zoom = int(req.args.get('zl', "0")) + @responder + def __call__ (self, env, start_response) : + """ + Main WSGI entry point. + + This is wrapped with werkzeug, so we can return a Response object + """ + + req = Request(env, start_response) - # yay render - return Response(render_tile(image, x, y, zoom), content_type="image/png") - - else : - raise exceptions.BadRequest("Unknown args") - + try : + return handlers.handle_req(req, self.cache) -@responder -def application (env, start_response) : - req = Request(env, start_response) - - try : - return handle_main(req) + except exceptions.HTTPException, e : + return e - except exceptions.HTTPException, e : - return e - diff -r aaae02944832 -r 4e6e067b3472 python/pypngtile.pyx --- a/python/pypngtile.pyx Sun Sep 14 15:19:59 2014 +0300 +++ b/python/pypngtile.pyx Sun Sep 14 15:24:58 2014 +0300 @@ -4,6 +4,9 @@ cdef extern from "string.h" : char* strerror (int err) + void* memset (void *, int, size_t) + void* memcpy (void *, void *, size_t) + cimport stdio cimport stdlib cimport python_string @@ -22,79 +25,217 @@ pass enum pt_open_mode : + PT_OPEN_READ # 0 PT_OPEN_UPDATE enum pt_cache_status : - PT_CACHE_ERROR + PT_CACHE_ERROR # -1 PT_CACHE_FRESH PT_CACHE_NONE PT_CACHE_STALE + PT_CACHE_INCOMPAT struct pt_image_info : - size_t width, height + size_t img_width, img_height, img_bpp + int image_mtime, cache_mtime, cache_version + size_t image_bytes, cache_bytes + size_t cache_blocks + struct pt_image_params : + int background_color[4] + struct pt_tile_info : size_t width, height size_t x, y int zoom - - int pt_image_open (pt_image **image_ptr, pt_ctx *ctx, char *png_path, int cache_mode) - int pt_image_info_func "pt_image_info" (pt_image *image, pt_image_info **info_ptr) - int pt_image_status (pt_image *image) - int pt_image_update (pt_image *image) - int pt_image_tile_file (pt_image *image, pt_tile_info *info, stdio.FILE *out) - int pt_image_tile_mem (pt_image *image, pt_tile_info *info, char **buf_ptr, size_t *len_ptr) - void pt_image_destroy (pt_image *image) + ctypedef pt_image_info* const_image_info_ptr "const struct pt_image_info *" + + ## functions + int pt_image_open (pt_image **image_ptr, pt_ctx *ctx, char *png_path, int cache_mode) nogil + int pt_image_info_ "pt_image_info" (pt_image *image, pt_image_info **info_ptr) nogil + int pt_image_status (pt_image *image) nogil + int pt_image_load (pt_image *image) nogil + int pt_image_update (pt_image *image, pt_image_params *params) nogil + int pt_image_tile_file (pt_image *image, pt_tile_info *info, stdio.FILE *out) nogil + int pt_image_tile_mem (pt_image *image, pt_tile_info *info, char **buf_ptr, size_t *len_ptr) nogil + void pt_image_destroy (pt_image *image) nogil + + # error code -> name char* pt_strerror (int err) -OPEN_UPDATE = PT_OPEN_UPDATE -CACHE_ERROR = PT_CACHE_ERROR -CACHE_FRESH = PT_CACHE_FRESH -CACHE_NONE = PT_CACHE_NONE -CACHE_STALE = PT_CACHE_STALE +## constants +# Image() +OPEN_READ = PT_OPEN_READ +OPEN_UPDATE = PT_OPEN_UPDATE + +# Image.status -> ... +CACHE_FRESH = PT_CACHE_FRESH +CACHE_NONE = PT_CACHE_NONE +CACHE_STALE = PT_CACHE_STALE +CACHE_INCOMPAT = PT_CACHE_INCOMPAT class Error (BaseException) : - pass + """ + Base class for errors raised by pypngtile. + """ -cdef int trap_err (char *op, int ret) except -1 : - if ret < 0 : - raise Error("%s: %s: %s" % (op, pt_strerror(ret), strerror(errno))) - - else : - return ret + def __init__ (self, func, err) : + super(Error, self).__init__("%s: %s: %s" % (func, pt_strerror(err), strerror(errno))) cdef class Image : + """ + An image file on disk (.png) and an associated .cache file. + + Open the .png file at the given path using the given mode. + + path - filesystem path to .png file + mode - mode to operate cache in + OPEN_READ - read-only access to cache + OPEN_UPDATE - allow .update() + """ + cdef pt_image *image - def __cinit__ (self, char *png_path, int cache_mode = 0) : - trap_err("pt_image_open", - pt_image_open(&self.image, NULL, png_path, cache_mode) - ) + # XXX: should really be a pt_image property... + cdef readonly object path + - def info (self) : - cdef pt_image_info *image_info + # open the pt_image + def __cinit__ (self, char *path, int mode = 0) : + cdef int err + + # store + self.path = path - trap_err("pt_image_info", - pt_image_info_func(self.image, &image_info) - ) + # open + with nogil : + # XXX: I hope use of path doesn't break... + err = pt_image_open(&self.image, NULL, path, mode) - return (image_info.width, image_info.height) - + if err : + raise Error("pt_image_open", err) + + + def info (self) : + """ + Return a dictionary containing various information about the image. + + img_width - pixel dimensions of the source image + img_height only available if the cache was opened + img_bpp - bits per pixel for the source image + + image_mtime - last modification timestamp for source image + image_bytes - size of source image file in bytes + + cache_version - version of cache file available + cache_mtime - last modification timestamp for cache file + cache_bytes - size of cache file in bytes + cache_blocks - size of cache file in disk blocks - 512 bytes / block + """ + + cdef const_image_info_ptr infop + cdef int err + + with nogil : + err = pt_image_info_(self.image, &infop) + + if err : + raise Error("pt_image_info", err) + + # return as a struct + return infop[0] + + def status (self) : - return trap_err("pt_image_status", - pt_image_status(self.image) - ) + """ + Return a code describing the status of the underlying cache file. + + CACHE_FRESH - the cache file exists and is up-to-date + CACHE_NONE - the cache file does not exist + CACHE_STALE - the cache file exists, but is older than the source image + CACHE_INCOMPAT - the cache file exists, but is incompatible with this version of the library + """ + + cdef int ret + + with nogil : + ret = pt_image_status(self.image) + + if ret : + raise Error("pt_image_status", ret) + + return ret + + def open (self) : + """ + Open the underlying cache file for reading, if available. + """ + + cdef int err + + with nogil : + err = pt_image_load(self.image) + + if err : + raise Error("pt_image_load", err) + + + def update (self, background_color = None) : + """ + Update the underlying cache file from the source image. + + background_color - skip consecutive pixels that match this byte pattern in output + + Requires that the Image was opened using OPEN_UPDATE. + """ + + cdef pt_image_params params + cdef char *bgcolor + cdef int err + + memset(¶ms, 0, sizeof(params)) + + # params + if background_color : + # cast + bgcolor = background_color + + if 0 >= len(bgcolor) > 4 : + raise ValueError("background_color must be a str of between 1 and 4 bytes") + + # decode + memcpy(params.background_color, bgcolor, len(bgcolor)) - def update (self) : - trap_err("pt_image_update", - pt_image_update(self.image) - ) + # run update + with nogil : + err = pt_image_update(self.image, ¶ms) + + if err : + raise Error("pt_image_update", err) + def tile_file (self, size_t width, size_t height, size_t x, size_t y, int zoom, object out) : + """ + Render a region of the source image as a PNG tile to the given output file. + + width - dimensions of the output tile in px + height + x - coordinates in the source file + y + zoom - zoom level: out = 2**(-zoom) * in + out - output file + + Note that the given file object MUST be a *real* stdio FILE*, not a fake Python object. + """ + cdef stdio.FILE *outf cdef pt_tile_info ti + cdef int err + memset(&ti, 0, sizeof(ti)) + + # convert to FILE if not PyFile_Check(out) : raise TypeError("out: must be a file object") @@ -102,32 +243,53 @@ if not outf : raise TypeError("out: must have a FILE*") - + + # pack params ti.width = width ti.height = height ti.x = x ti.y = y ti.zoom = zoom - trap_err("pt_image_tile_file", - pt_image_tile_file(self.image, &ti, outf) - ) + # render + with nogil : + err = pt_image_tile_file(self.image, &ti, outf) + + if err : + raise Error("pt_image_tile_file", err) + def tile_mem (self, size_t width, size_t height, size_t x, size_t y, int zoom) : + """ + Render a region of the source image as a PNG tile, and return the PNG data a a string. + + width - dimensions of the output tile in px + height + x - coordinates in the source file + y + zoom - zoom level: out = 2**(-zoom) * in + """ + cdef pt_tile_info ti cdef char *buf cdef size_t len - + cdef int err + + memset(&ti, 0, sizeof(ti)) + + # pack params ti.width = width ti.height = height ti.x = x ti.y = y ti.zoom = zoom - # render and return ptr to buffer - trap_err("pt_image_tile_mem", - pt_image_tile_mem(self.image, &ti, &buf, &len) - ) + # render and return via buf/len + with nogil : + err = pt_image_tile_mem(self.image, &ti, &buf, &len) + + if err : + raise Error("pt_image_tile_mem", err) # copy buffer as str... data = python_string.PyString_FromStringAndSize(buf, len) @@ -137,7 +299,10 @@ return data + # release the pt_image def __dealloc__ (self) : if self.image : pt_image_destroy(self.image) + self.image = NULL + diff -r aaae02944832 -r 4e6e067b3472 setup.py --- a/setup.py Sun Sep 14 15:19:59 2014 +0300 +++ b/setup.py Sun Sep 14 15:24:58 2014 +0300 @@ -1,12 +1,37 @@ from distutils.core import setup from distutils.extension import Extension -from Cython.Distutils import build_ext + +import os.path + +build_root = os.path.abspath(os.path.dirname(__file__)) + +pypngtile_c = "python/pypngtile.c" +pypngtile_name = "python/pypngtile.pyx" + +cmdclass = dict() + +try : + from Cython.Distutils import build_ext + + cmdclass['build_ext'] = build_ext + +except ImportError : + path = os.path.join(build_root, pypngtile_c) + + if os.path.exists(path) : + print "Warning: falling back from .pyx -> .c due to missing Cython" + # just use the .c + pypngtile_name = pypngtile_c + + else : + # fail + raise setup( name = 'pngtiles', - cmdclass = {'build_ext': build_ext}, + cmdclass = cmdclass, ext_modules = [ - Extension("pypngtile", ["python/pypngtile.pyx"], + Extension("pypngtile", [pypngtile_name], include_dirs = ['include'], library_dirs = ['lib'], libraries = ['pngtile'], diff -r aaae02944832 -r 4e6e067b3472 src/lib/cache.c --- a/src/lib/cache.c Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/cache.c Sun Sep 14 15:24:58 2014 +0300 @@ -42,9 +42,107 @@ return err; } +/** + * Force-clean pt_cache, warn on errors + */ +static void pt_cache_abort (struct pt_cache *cache) +{ + if (cache->file != NULL) { + if (munmap(cache->file, sizeof(struct pt_cache_file) + cache->file->header.data_size)) + log_warn_errno("munmap: %p, %zu", cache->file, sizeof(struct pt_cache_file) + cache->file->header.data_size); + + cache->file = NULL; + } + + if (cache->fd >= 0) { + if (close(cache->fd)) + log_warn_errno("close: %d", cache->fd); + + cache->fd = -1; + } +} + +/** + * Open the cache file as an fd for reading + */ +static int pt_cache_open_read_fd (struct pt_cache *cache, int *fd_ptr) +{ + int fd; + + // actual open() + if ((fd = open(cache->path, O_RDONLY)) < 0) + RETURN_ERROR_ERRNO(PT_ERR_OPEN_MODE, EACCES); + + // ok + *fd_ptr = fd; + + return 0; +} + +/** + * Read in the cache header from the open file + */ +static int pt_cache_read_header (int fd, struct pt_cache_header *header) +{ + size_t len = sizeof(*header); + char *buf = (char *) header; + + // seek to start + if (lseek(fd, 0, SEEK_SET) != 0) + RETURN_ERROR(PT_ERR_CACHE_SEEK); + + // write out full header + while (len) { + ssize_t ret; + + // try and write out the header + if ((ret = read(fd, buf, len)) <= 0) + RETURN_ERROR(PT_ERR_CACHE_READ); + + // update offset + buf += ret; + len -= ret; + } + + // done + return 0; +} + +/** + * Read and return the version number from the cache file, temporarily opening it if needed + */ +static int pt_cache_version (struct pt_cache *cache) +{ + int fd; + struct pt_cache_header header; + int ret; + + // already open? + if (cache->file) + return cache->file->header.version; + + // temp. open + if ((ret = pt_cache_open_read_fd(cache, &fd))) + return ret; + + // read header + if ((ret = pt_cache_read_header(fd, &header))) + JUMP_ERROR(ret); + + // ok + ret = header.version; + +error: + // close + close(fd); + + return ret; +} + int pt_cache_status (struct pt_cache *cache, const char *img_path) { struct stat st_img, st_cache; + int ver; // test original file if (stat(img_path, &st_img) < 0) @@ -62,61 +160,61 @@ // compare mtime if (st_img.st_mtime > st_cache.st_mtime) return PT_CACHE_STALE; + + // read version + if ((ver = pt_cache_version(cache)) < 0) + // fail + return ver; - else - return PT_CACHE_FRESH; + // compare version + if (ver != PT_CACHE_VERSION) + return PT_CACHE_INCOMPAT; + + // ok, should be in order + return PT_CACHE_FRESH; } -int pt_cache_info (struct pt_cache *cache, struct pt_image_info *info) +void pt_cache_info (struct pt_cache *cache, struct pt_image_info *info) { - int err; + struct stat st; - // ensure open - if ((err = pt_cache_open(cache))) - return err; + if (cache->file) + // img info + pt_png_info(&cache->file->header.png, info); - info->width = cache->header->width; - info->height = cache->header->height; + // stat + if (stat(cache->path, &st) < 0) { + // unknown + info->cache_mtime = 0; + info->cache_bytes = 0; + info->cache_blocks = 0; + + } else { + // store + info->cache_version = pt_cache_version(cache); + info->cache_mtime = st.st_mtime; + info->cache_bytes = st.st_size; + info->cache_blocks = st.st_blocks; + } +} + +static int pt_cache_tmp_name (struct pt_cache *cache, char tmp_path[], size_t tmp_len) +{ + // get .tmp path + if (path_with_fext(cache->path, tmp_path, tmp_len, ".tmp")) + RETURN_ERROR(PT_ERR_PATH); return 0; } /** - * Abort any incomplete open operation, cleaning up + * Compute and return the full size of the .cache file */ -static void pt_cache_abort (struct pt_cache *cache) +static size_t pt_cache_size (size_t data_size) { - if (cache->header != NULL) { - munmap(cache->header, PT_CACHE_HEADER_SIZE + cache->size); - - cache->header = NULL; - cache->data = NULL; - } - - if (cache->fd >= 0) { - close(cache->fd); + assert(sizeof(struct pt_cache_file) == PT_CACHE_HEADER_SIZE); - cache->fd = -1; - } -} - -/** - * Open the cache file as an fd for reading - * - * XXX: needs locking - */ -static int pt_cache_open_read_fd (struct pt_cache *cache, int *fd_ptr) -{ - int fd; - - // actual open() - if ((fd = open(cache->path, O_RDONLY)) < 0) - RETURN_ERROR_ERRNO(PT_ERR_OPEN_MODE, EACCES); - - // ok - *fd_ptr = fd; - - return 0; + return sizeof(struct pt_cache_file) + data_size; } /** @@ -126,14 +224,14 @@ { int fd; char tmp_path[1024]; - + int err; + // get .tmp path - if (path_with_fext(cache->path, tmp_path, sizeof(tmp_path), ".tmp")) - RETURN_ERROR(PT_ERR_PATH); + if ((err = pt_cache_tmp_name(cache, tmp_path, sizeof(tmp_path)))) + return err; - // open for write, create - // XXX: locking? - if ((fd = open(tmp_path, O_RDWR | O_CREAT, 0644)) < 0) + // open for write, create, fail if someone else already opened it for update + if ((fd = open(tmp_path, O_RDWR | O_CREAT | O_EXCL, 0644)) < 0) RETURN_ERROR(PT_ERR_CACHE_OPEN_TMP); // ok @@ -144,9 +242,9 @@ /** - * Mmap the opened cache file using PT_CACHE_HEADER_SIZE plus the calculated size stored in cache->size + * Mmap the pt_cache_file using pt_cache_size(data_size) */ -static int pt_cache_open_mmap (struct pt_cache *cache, void **addr_ptr, bool readonly) +static int pt_cache_open_mmap (struct pt_cache *cache, struct pt_cache_file **file_ptr, size_t data_size, bool readonly) { int prot = 0; void *addr; @@ -160,43 +258,49 @@ prot |= PROT_WRITE; } - // perform mmap() from second page on - if ((addr = mmap(NULL, PT_CACHE_HEADER_SIZE + cache->size, prot, MAP_SHARED, cache->fd, 0)) == MAP_FAILED) + // mmap() the full file including header + if ((addr = mmap(NULL, pt_cache_size(data_size), prot, MAP_SHARED, cache->fd, 0)) == MAP_FAILED) RETURN_ERROR(PT_ERR_CACHE_MMAP); // ok - *addr_ptr = addr; + *file_ptr = addr; return 0; } -/** - * Read in the cache header from the open file - */ -static int pt_cache_read_header (struct pt_cache *cache, struct pt_cache_header *header) +int pt_cache_open (struct pt_cache *cache) { - size_t len = sizeof(*header); - char *buf = (char *) header; - - // seek to start - if (lseek(cache->fd, 0, SEEK_SET) != 0) - RETURN_ERROR(PT_ERR_CACHE_SEEK); + struct pt_cache_header header; + int err; - // write out full header - while (len) { - ssize_t ret; - - // try and write out the header - if ((ret = read(cache->fd, buf, len)) <= 0) - RETURN_ERROR(PT_ERR_CACHE_READ); + // ignore if already open + if (cache->file) + return 0; - // update offset - buf += ret; - len -= ret; - } + // open the .cache in readonly mode + if ((err = pt_cache_open_read_fd(cache, &cache->fd))) + return err; + + // read in header + if ((err = pt_cache_read_header(cache->fd, &header))) + JUMP_ERROR(err); + + // check version + if (header.version != PT_CACHE_VERSION) + JUMP_SET_ERROR(err, PT_ERR_CACHE_VERSION); + + // mmap the header + data + if ((err = pt_cache_open_mmap(cache, &cache->file, header.data_size, true))) + JUMP_ERROR(err); // done return 0; + +error: + // cleanup + pt_cache_abort(cache); + + return err; } /** @@ -233,34 +337,24 @@ */ static int pt_cache_create (struct pt_cache *cache, struct pt_cache_header *header) { - void *base; int err; - // no access - if (!(cache->mode & PT_OPEN_UPDATE)) - RETURN_ERROR(PT_ERR_OPEN_MODE); + assert(cache->mode & PT_OPEN_UPDATE); // open as .tmp if ((err = pt_cache_open_tmp_fd(cache, &cache->fd))) return err; - // calculate data size - cache->size = sizeof(*header) + header->height * header->row_bytes; + // write header + if ((err = pt_cache_write_header(cache, header))) + JUMP_ERROR(err); // grow file - if (ftruncate(cache->fd, PT_CACHE_HEADER_SIZE + cache->size) < 0) + if (ftruncate(cache->fd, pt_cache_size(header->data_size)) < 0) JUMP_SET_ERROR(err, PT_ERR_CACHE_TRUNC); // mmap header and data - if ((err = pt_cache_open_mmap(cache, &base, false))) - JUMP_ERROR(err); - - cache->header = base; - cache->data = base + PT_CACHE_HEADER_SIZE; - - // write header - // XXX: should just mmap... - if ((err = pt_cache_write_header(cache, header))) + if ((err = pt_cache_open_mmap(cache, &cache->file, header->data_size, false))) JUMP_ERROR(err); // done @@ -279,10 +373,11 @@ static int pt_cache_create_done (struct pt_cache *cache) { char tmp_path[1024]; + int err; // get .tmp path - if (path_with_fext(cache->path, tmp_path, sizeof(tmp_path), ".tmp")) - RETURN_ERROR(PT_ERR_PATH); + if ((err = pt_cache_tmp_name(cache, tmp_path, sizeof(tmp_path)))) + return err; // rename if (rename(tmp_path, cache->path) < 0) @@ -292,355 +387,76 @@ return 0; } -int pt_cache_open (struct pt_cache *cache) +/** + * Abort a failed cache update after cache_create + */ +static void pt_cache_create_abort (struct pt_cache *cache) +{ + char tmp_path[1024]; + int err; + + // close open stuff + pt_cache_abort(cache); + + // get .tmp path + if ((err = pt_cache_tmp_name(cache, tmp_path, sizeof(tmp_path)))) { + log_warn_errno("pt_cache_tmp_name: %s: %s", cache->path, pt_strerror(err)); + + return; + } + + // remove .tmp + if (unlink(tmp_path)) + log_warn_errno("unlink: %s", tmp_path); +} + +int pt_cache_update (struct pt_cache *cache, struct pt_png_img *img, const struct pt_image_params *params) { struct pt_cache_header header; - void *base; int err; - // ignore if already open - if (cache->header && cache->data) - return 0; + // check mode + if (!(cache->mode & PT_OPEN_UPDATE)) + RETURN_ERROR(PT_ERR_OPEN_MODE); - // open the .cache - if ((err = pt_cache_open_read_fd(cache, &cache->fd))) + // close if open + if ((err = pt_cache_close(cache))) + return err; + + // prep header + header.version = PT_CACHE_VERSION; + header.format = PT_IMG_PNG; + + // read img header + if ((err = pt_png_read_header(img, &header.png, &header.data_size))) return err; - // read in header - if ((err = pt_cache_read_header(cache, &header))) - JUMP_ERROR(err); - - // calculate data size - cache->size = sizeof(header) + header.height * header.row_bytes; + // save any params + if (params) + header.params = *params; - // mmap header and data - if ((err = pt_cache_open_mmap(cache, &base, true))) - JUMP_ERROR(err); + // create/open .tmp and write out header + if ((err = pt_cache_create(cache, &header))) + return err; - cache->header = base; - cache->data = base + PT_CACHE_HEADER_SIZE; + // decode to disk + if ((err = pt_png_decode(img, &cache->file->header.png, &cache->file->header.params, cache->file->data))) + goto error; - // done + // done, commit .tmp + if ((err = pt_cache_create_done(cache))) + goto error; + return 0; error: - // cleanup - pt_cache_abort(cache); + // cleanup .tmp + pt_cache_create_abort(cache); return err; } -int pt_cache_update_png (struct pt_cache *cache, png_structp png, png_infop info) -{ - struct pt_cache_header header; - int err; - - // XXX: check cache_mode - // XXX: check image doesn't use any options we don't handle - // XXX: close any already-opened cache file - - memset(&header, 0, sizeof(header)); - - // fill in basic info - header.width = png_get_image_width(png, info); - header.height = png_get_image_height(png, info); - header.bit_depth = png_get_bit_depth(png, info); - header.color_type = png_get_color_type(png, info); - - log_debug("width=%u, height=%u, bit_depth=%u, color_type=%u", - header.width, header.height, header.bit_depth, header.color_type - ); - - // only pack 1 pixel per byte, changes rowbytes - if (header.bit_depth < 8) - png_set_packing(png); - - // fill in other info - header.row_bytes = png_get_rowbytes(png, info); - - // calculate bpp as num_channels * bpc - // XXX: this assumes the packed bit depth will be either 8 or 16 - header.col_bytes = png_get_channels(png, info) * (header.bit_depth == 16 ? 2 : 1); - - log_debug("row_bytes=%u, col_bytes=%u", header.row_bytes, header.col_bytes); - - // palette etc. - if (header.color_type == PNG_COLOR_TYPE_PALETTE) { - int num_palette; - png_colorp palette; - - if (png_get_PLTE(png, info, &palette, &num_palette) == 0) - // XXX: PLTE chunk not read? - RETURN_ERROR(PT_ERR_PNG); - - // should only be 256 of them at most - assert(num_palette <= PNG_MAX_PALETTE_LENGTH); - - // copy - header.num_palette = num_palette; - memcpy(&header.palette, palette, num_palette * sizeof(*palette)); - - log_debug("num_palette=%u", num_palette); - } - - // create .tmp and write out header - if ((err = pt_cache_create(cache, &header))) - return err; - - - // write out raw image data a row at a time - for (size_t row = 0; row < header.height; row++) { - // read row data, non-interlaced - png_read_row(png, cache->data + row * header.row_bytes, NULL); - } - - - // move from .tmp to .cache - if ((err = pt_cache_create_done(cache))) - // XXX: pt_cache_abort? - return err; - - // done! - return 0; -} - -/** - * Return a pointer to the pixel data on \a row, starting at \a col. - */ -static inline void* tile_row_col (struct pt_cache *cache, size_t row, size_t col) -{ - return cache->data + (row * cache->header->row_bytes) + (col * cache->header->col_bytes); -} - -/** - * Fill in a clipped region of \a width_px pixels at the given row segment - */ -static inline void tile_row_fill_clip (struct pt_cache *cache, png_byte *row, size_t width_px) -{ - // XXX: use a configureable background color, or full transparency? - memset(row, /* 0xd7 */ 0x00, width_px * cache->header->col_bytes); -} - -/** - * Write raw tile image data, directly from the cache - */ -static int write_png_data_direct (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti) -{ - for (size_t row = ti->y; row < ti->y + ti->height; row++) - // write data directly - png_write_row(png, tile_row_col(cache, row, ti->x)); - - return 0; -} - -/** - * Write clipped tile image data (a tile that goes over the edge of the actual image) by aligning the data from the cache as needed - */ -static int write_png_data_clipped (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti) -{ - png_byte *rowbuf; - size_t row; - - // image data goes from (ti->x ... clip_x, ti->y ... clip_y), remaining region is filled - size_t clip_x, clip_y; - - - // figure out if the tile clips over the right edge - // XXX: use min() - if (ti->x + ti->width > cache->header->width) - clip_x = cache->header->width; - else - clip_x = ti->x + ti->width; - - // figure out if the tile clips over the bottom edge - // XXX: use min() - if (ti->y + ti->height > cache->header->height) - clip_y = cache->header->height; - else - clip_y = ti->y + ti->height; - - - // allocate buffer for a single row of image data - if ((rowbuf = malloc(ti->width * cache->header->col_bytes)) == NULL) - RETURN_ERROR(PT_ERR_MEM); - - // how much data we actually have for each row, in px and bytes - // from [(tile x)---](clip x) - size_t row_px = clip_x - ti->x; - size_t row_bytes = row_px * cache->header->col_bytes; - - // write the rows that we have - // from [(tile y]---](clip y) - for (row = ti->y; row < clip_y; row++) { - // copy in the actual tile data... - memcpy(rowbuf, tile_row_col(cache, row, ti->x), row_bytes); - - // generate the data for the remaining, clipped, columns - tile_row_fill_clip(cache, rowbuf + row_bytes, (ti->width - row_px)); - - // write - png_write_row(png, rowbuf); - } - - // generate the data for the remaining, clipped, rows - tile_row_fill_clip(cache, rowbuf, ti->width); - - // write out the remaining rows as clipped data - for (; row < ti->y + ti->height; row++) - png_write_row(png, rowbuf); - - // ok - return 0; -} - -static size_t scale_by_zoom_factor (size_t value, int z) -{ - if (z > 0) - return value << z; - - else if (z < 0) - return value >> -z; - - else - return value; -} - -#define ADD_AVG(l, r) (l) = ((l) + (r)) / 2 - -static int png_pixel_data (png_color *out, struct pt_cache *cache, size_t row, size_t col) -{ - if (cache->header->color_type == PNG_COLOR_TYPE_PALETTE) { - // palette entry number - int p; - - if (cache->header->bit_depth == 8) - p = *((uint8_t *) tile_row_col(cache, row, col)); - else - return -1; - - if (p >= cache->header->num_palette) - return -1; - - // reference data from palette - *out = cache->header->palette[p]; - - return 0; - - } else { - return -1; - } -} - -/** - * Write unscaled tile data - */ -static int write_png_data_unzoomed (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti) -{ - int err; - - // set basic info - png_set_IHDR(png, info, ti->width, ti->height, cache->header->bit_depth, cache->header->color_type, - PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT - ); - - // set palette? - if (cache->header->color_type == PNG_COLOR_TYPE_PALETTE) - png_set_PLTE(png, info, cache->header->palette, cache->header->num_palette); - - // write meta-info - png_write_info(png, info); - - // our pixel data is packed into 1 pixel per byte (8bpp or 16bpp) - png_set_packing(png); - - // figure out if the tile clips - if (ti->x + ti->width <= cache->header->width && ti->y + ti->height <= cache->header->height) - // doesn't clip, just use the raw data - err = write_png_data_direct(cache, png, info, ti); - - else - // fill in clipped regions - err = write_png_data_clipped(cache, png, info, ti); - - return err; -} - -/** - * Write scaled tile data - */ -static int write_png_data_zoomed (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti) -{ - // size of the image data in px - size_t data_width = scale_by_zoom_factor(ti->width, -ti->zoom); - size_t data_height = scale_by_zoom_factor(ti->height, -ti->zoom); - - // input pixels per output pixel - size_t pixel_size = scale_by_zoom_factor(1, -ti->zoom); - - // bytes per output pixel - size_t pixel_bytes = 3; - - // size of the output tile in px - size_t row_width = ti->width; - - // size of an output row in bytes (RGB) - size_t row_bytes = row_width * 3; - - // buffer to hold output rows - uint8_t *row_buf; - - // XXX: only supports zooming out... - if (ti->zoom >= 0) - RETURN_ERROR(PT_ERR_ZOOM); - - if ((row_buf = malloc(row_bytes)) == NULL) - RETURN_ERROR(PT_ERR_MEM); - - - // define pixel format: 8bpp RGB - png_set_IHDR(png, info, ti->width, ti->height, 8, PNG_COLOR_TYPE_RGB, - PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT - ); - - // write meta-info - png_write_info(png, info); - - // ...each output row - for (size_t out_row = 0; out_row < ti->height; out_row++) { - memset(row_buf, 0, row_bytes); - - // ...includes pixels starting from this row. - size_t in_row_offset = ti->y + scale_by_zoom_factor(out_row, -ti->zoom); - - // ...each out row includes pixel_size in rows - for (size_t in_row = in_row_offset; in_row < in_row_offset + pixel_size && in_row < cache->header->height; in_row++) { - // and includes each input pixel - for (size_t in_col = ti->x; in_col < ti->x + data_width && in_col < cache->header->width; in_col++) { - png_color c; - - // ...for this output pixel - size_t out_col = scale_by_zoom_factor(in_col - ti->x, ti->zoom); - - // get pixel RGB data - if (png_pixel_data(&c, cache, in_row, in_col)) - return -1; - - // average the RGB data - ADD_AVG(row_buf[out_col * pixel_bytes + 0], c.red); - ADD_AVG(row_buf[out_col * pixel_bytes + 1], c.green); - ADD_AVG(row_buf[out_col * pixel_bytes + 2], c.blue); - } - } - - // output - png_write_row(png, row_buf); - } - - // done - return 0; -} - -int pt_cache_tile_png (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti) +int pt_cache_tile (struct pt_cache *cache, struct pt_tile *tile) { int err; @@ -648,34 +464,38 @@ if ((err = pt_cache_open(cache))) return err; - // check within bounds - if (ti->x >= cache->header->width || ti->y >= cache->header->height) - // completely outside - RETURN_ERROR(PT_ERR_TILE_CLIP); - - // unscaled or scaled? - if (ti->zoom) - err = write_png_data_zoomed(cache, png, info, ti); + // render + if ((err = pt_png_tile(&cache->file->header.png, cache->file->data, tile))) + return err; - else - err = write_png_data_unzoomed(cache, png, info, ti); + return 0; +} - if (err) - return err; - - // done, flush remaining output - png_write_flush(png); +int pt_cache_close (struct pt_cache *cache) +{ + if (cache->file != NULL) { + if (munmap(cache->file, sizeof(struct pt_cache_file) + cache->file->header.data_size)) + RETURN_ERROR(PT_ERR_CACHE_MUNMAP); - // ok + cache->file = NULL; + } + + if (cache->fd >= 0) { + if (close(cache->fd)) + RETURN_ERROR(PT_ERR_CACHE_CLOSE); + + cache->fd = -1; + } + return 0; } void pt_cache_destroy (struct pt_cache *cache) { - free(cache->path); - + // cleanup pt_cache_abort(cache); + free(cache->path); free(cache); } diff -r aaae02944832 -r 4e6e067b3472 src/lib/cache.h --- a/src/lib/cache.h Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/cache.h Sun Sep 14 15:24:58 2014 +0300 @@ -7,14 +7,61 @@ * Internal image cache implementation */ #include "image.h" +#include "png.h" #include #include -#include +/** + * Cache format version + */ +#define PT_CACHE_VERSION 2 /** - * State for cache access + * Size used to store the cache header + */ +#define PT_CACHE_HEADER_SIZE 4096 + +/** + * On-disk header + */ +struct pt_cache_header { + /** Set to PT_CACHE_VERSION */ + uint16_t version; + + /** Image format */ + enum pt_img_format { + PT_IMG_PNG, ///< @see pt_png + } format; + + /** Data header by format */ + union { + struct pt_png_header png; + }; + + /** Parameters used */ + struct pt_image_params params; + + /** Size of the data segment */ + size_t data_size; +}; + +/** + * On-disk data format. This struct is always exactly PT_CACHE_HEADER_SIZE long + */ +struct pt_cache_file { + /** Header */ + struct pt_cache_header header; + + /** Padding for data */ + uint8_t padding[PT_CACHE_HEADER_SIZE - sizeof(struct pt_cache_header)]; + + /** Data follows, header.data_size bytes */ + uint8_t data[]; +}; + +/** + * Cache state */ struct pt_cache { /** Filesystem path to cache file */ @@ -26,42 +73,11 @@ /** Opened file */ int fd; - /** The mmap'd header */ - struct pt_cache_header *header; - - /** Memory-mapped file data, starting at PT_CACHE_HEADER_SIZE */ - uint8_t *data; - /** Size of the data segment in bytes, starting at PT_CACHE_HEADER_SIZE */ - size_t size; -}; - -/** - * Size used to store the cache header - */ -#define PT_CACHE_HEADER_SIZE 4096 + size_t data_size; -/** - * On-disk header - */ -struct pt_cache_header { - /** Pixel dimensions of image */ - uint32_t width, height; - - /** Pixel format */ - uint8_t bit_depth, color_type; - - /** Number of png_color entries that follow */ - uint16_t num_palette; - - /** Number of bytes per row */ - uint32_t row_bytes; - - /** Number of bytes per pixel */ - uint8_t col_bytes; - - /** Palette entries, up to 256 entries used */ - png_color palette[PNG_MAX_PALETTE_LENGTH]; + /** The mmap'd file */ + struct pt_cache_file *file; }; /** @@ -77,14 +93,16 @@ int pt_cache_status (struct pt_cache *cache, const char *img_path); /** - * Get info for the cached image, open it if not already open. + * Get info for the cached image. + * + * Does not open it if not yet opened. */ -int pt_cache_info (struct pt_cache *cache, struct pt_image_info *info); +void pt_cache_info (struct pt_cache *cache, struct pt_image_info *info); /** - * Update the cache data from the given PNG image. + * Update the cache data from the given image data */ -int pt_cache_update_png (struct pt_cache *cache, png_structp png, png_infop info); +int pt_cache_update (struct pt_cache *cache, struct pt_png_img *img, const struct pt_image_params *params); /** * Open the existing .cache for use. If already opened, does nothing. @@ -92,11 +110,16 @@ int pt_cache_open (struct pt_cache *cache); /** - * Render out a PNG tile as given, into the established png object, up to (but not including) the png_write_end. + * Render out the given tile * * If the cache is not yet open, this will open it */ -int pt_cache_tile_png (struct pt_cache *cache, png_structp png, png_infop info, const struct pt_tile_info *ti); +int pt_cache_tile (struct pt_cache *cache, struct pt_tile *tile); + +/** + * Close the cache, if opened + */ +int pt_cache_close (struct pt_cache *cache); /** * Release all resources associated with the given cache object without any cleanup. diff -r aaae02944832 -r 4e6e067b3472 src/lib/error.c --- a/src/lib/error.c Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/error.c Sun Sep 14 15:24:58 2014 +0300 @@ -5,13 +5,15 @@ */ const char *error_names[PT_ERR_MAX] = { [PT_SUCCESS] = "Success", + [PT_ERR] = "Unspecified error", [PT_ERR_MEM] = "malloc()", [PT_ERR_PATH] = "path", [PT_ERR_OPEN_MODE] = "open_mode", [PT_ERR_IMG_STAT] = "stat(.png)", - [PT_ERR_IMG_FOPEN] = "fopen(.png)", + [PT_ERR_IMG_OPEN] = "open(.png)", + [PT_ERR_IMG_FORMAT] = "Unknown image format", [PT_ERR_PNG_CREATE] = "png_create()", [PT_ERR_PNG] = "png_*()", @@ -25,12 +27,16 @@ [PT_ERR_CACHE_TRUNC] = "truncate(.cache)", [PT_ERR_CACHE_MMAP] = "mmap(.cache)", [PT_ERR_CACHE_RENAME_TMP] = "rename(.tmp, .cache)", + [PT_ERR_CACHE_VERSION] = "Incompatible .cache version", + [PT_ERR_CACHE_MUNMAP] = "munmap(cache->file)", + [PT_ERR_CACHE_CLOSE] = "close(cache->fd)", [PT_ERR_PTHREAD_CREATE] = "pthread_create", [PT_ERR_CTX_SHUTDOWN] = "pt_ctx is shutting down", + [PT_ERR_TILE_DIM] = "Invalid tile dimensions", [PT_ERR_TILE_CLIP] = "Tile outside of image", - [PT_ERR_ZOOM] = "Invalid zoom level", + [PT_ERR_TILE_ZOOM] = "Invalid zoom level", }; const char *pt_strerror (int err) @@ -39,7 +45,11 @@ err = -err; if (err < PT_SUCCESS || err >= PT_ERR_MAX) - return "Unknown error"; + return "Invalid error code"; + + else if (!error_names[err]) + return "Missing string for error code"; + else return error_names[err]; } diff -r aaae02944832 -r 4e6e067b3472 src/lib/image.c --- a/src/lib/image.c Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/image.c Sun Sep 14 15:24:58 2014 +0300 @@ -1,5 +1,6 @@ #include "image.h" #include "ctx.h" +#include "png.h" #include "cache.h" #include "tile.h" #include "error.h" @@ -7,10 +8,10 @@ #include "shared/log.h" #include +#include +#include #include -#include - static int pt_image_new (struct pt_image **image_ptr, struct pt_ctx *ctx, const char *path) { struct pt_image *image; @@ -38,128 +39,6 @@ } /** - * Open the image's FILE - */ -static int pt_image_open_file (struct pt_image *image, FILE **file_ptr) -{ - FILE *fp; - - // open - if ((fp = fopen(image->path, "rb")) == NULL) - RETURN_ERROR(PT_ERR_IMG_FOPEN); - - // ok - *file_ptr = fp; - - return 0; -} - -/** - * Open the PNG image, setting up the I/O and returning the png_structp and png_infop - */ -static int pt_image_open_png (struct pt_image *image, png_structp *png_ptr, png_infop *info_ptr) -{ - FILE *fp = NULL; - png_structp png = NULL; - png_infop info = NULL; - int err; - - // open I/O - if ((err = pt_image_open_file(image, &fp))) - JUMP_ERROR(err); - - // create the struct - if ((png = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)) == NULL) - JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); - - // create the info - if ((info = png_create_info_struct(png)) == NULL) - JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); - - // setup error trap for the I/O - if (setjmp(png_jmpbuf(png))) - JUMP_SET_ERROR(err, PT_ERR_PNG); - - // setup I/O to FILE - png_init_io(png, fp); - - // ok - // XXX: what to do with fp? Should fclose() when done? - *png_ptr = png; - *info_ptr = info; - - return 0; - -error: - // cleanup file - if (fp) fclose(fp); - - // cleanup PNG state - png_destroy_read_struct(&png, &info, NULL); - - return err; -} - -/** - * Update the image_info field from the given png object. - * - * Must be called under libpng-error-trap! - * - * XXX: currently this info is not used, pulled from the cache instead - */ -static int pt_image_update_info (struct pt_image *image, png_structp png, png_infop info) -{ - // query png_get_* - image->info.width = png_get_image_width(png, info); - image->info.height = png_get_image_height(png, info); - - return 0; -} - -/** - * Open the PNG image, and write out to the cache - */ -static int pt_image_update_cache (struct pt_image *image) -{ - png_structp png; - png_infop info; - int err = 0; - - // pre-check enabled - if (!(image->cache->mode & PT_OPEN_UPDATE)) - RETURN_ERROR_ERRNO(PT_ERR_OPEN_MODE, EACCES); - - // open .png - if ((err = pt_image_open_png(image, &png, &info))) - return err; - - // setup error trap - if (setjmp(png_jmpbuf(png))) - JUMP_SET_ERROR(err, PT_ERR_PNG); - - // read meta-info - png_read_info(png, info); - - // update our meta-info - if ((err = pt_image_update_info(image, png, info))) - JUMP_ERROR(err); - - // pass to cache object - if ((err = pt_cache_update_png(image->cache, png, info))) - JUMP_ERROR(err); - - // finish off, ignore trailing data - png_read_end(png, NULL); - -error: - // clean up - // XXX: we need to close the fopen'd .png - png_destroy_read_struct(&png, &info, NULL); - - return err; -} - -/** * Build a filesystem path representing the appropriate path for this image's cache entry, and store it in the given * buffer. */ @@ -177,7 +56,13 @@ char cache_path[1024]; int err; - // XXX: verify that the path exists and looks like a PNG file + // verify that the path exists and looks like a PNG file + if ((err = pt_png_check(path)) < 0) + return err; + + if (err) + // fail, not a PNG + RETURN_ERROR(PT_ERR_IMG_FORMAT); // alloc if ((err = pt_image_new(&image, ctx, path))) @@ -202,13 +87,66 @@ return err; } +int pt_image_open_file (struct pt_image *image, FILE **file_ptr) +{ + FILE *fp; + + // open + if ((fp = fopen(image->path, "rb")) == NULL) + RETURN_ERROR(PT_ERR_IMG_OPEN); + + // ok + *file_ptr = fp; + + return 0; +} + +/** + * Open the PNG image, and write out to the cache + */ +int pt_image_update (struct pt_image *image, const struct pt_image_params *params) +{ + struct pt_png_img img; + int err = 0; + + // pre-check enabled + if (!(image->cache->mode & PT_OPEN_UPDATE)) + RETURN_ERROR_ERRNO(PT_ERR_OPEN_MODE, EACCES); + + // open .png + if ((err = pt_png_open(image, &img))) + return err; + + // pass to cache object + if ((err = pt_cache_update(image->cache, &img, params))) + JUMP_ERROR(err); + +error: + // clean up + pt_png_release_read(&img); + + return err; +} + + int pt_image_info (struct pt_image *image, const struct pt_image_info **info_ptr) { - int err; + struct stat st; - // update info - if ((err = pt_cache_info(image->cache, &image->info))) - return err; + // update info from cache + pt_cache_info(image->cache, &image->info); + + // stat our info + if (stat(image->path, &st) < 0) { + // unknown + image->info.image_mtime = 0; + image->info.image_bytes = 0; + + } else { + // store + image->info.image_mtime = st.st_mtime; + image->info.image_bytes = st.st_size; + } // return pointer *info_ptr = &image->info; @@ -221,10 +159,6 @@ return pt_cache_status(image->cache, image->path); } -int pt_image_update (struct pt_image *image) -{ - return pt_image_update_cache(image); -} int pt_image_load (struct pt_image *image) { @@ -278,6 +212,9 @@ return err; } +/** + * Async work func for pt_image_tile_async + */ static void _pt_image_tile_async (void *arg) { struct pt_tile *tile = arg; @@ -300,6 +237,10 @@ struct pt_tile *tile; int err; + // need a ctx for this + if (!image->ctx) + return -1; + // alloc if ((err = pt_tile_new(&tile))) return err; diff -r aaae02944832 -r 4e6e067b3472 src/lib/image.h --- a/src/lib/image.h Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/image.h Sun Sep 14 15:24:58 2014 +0300 @@ -22,5 +22,9 @@ struct pt_image_info info; }; +/** + * Open the image's FILE + */ +int pt_image_open_file (struct pt_image *image, FILE **file_ptr); #endif diff -r aaae02944832 -r 4e6e067b3472 src/lib/png.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/png.c Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,610 @@ +#include "png.h" // pt_png header +#include "error.h" +#include "shared/log.h" // debug only + +#include // sysmtem libpng header +#include + + +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +int pt_png_check (const char *path) +{ + FILE *fp; + uint8_t header[8]; + int ret; + + // fopen + if ((fp = fopen(path, "rb")) == NULL) + RETURN_ERROR(PT_ERR_IMG_OPEN); + + // read + if (fread(header, 1, sizeof(header), fp) != sizeof(header)) + JUMP_SET_ERROR(ret, PT_ERR_IMG_FORMAT); + + // compare signature + if (png_sig_cmp(header, 0, sizeof(header))) + // not a PNG file + ret = 1; + + else + // valid PNG file + ret = 0; + +error: + // cleanup + fclose(fp); + + return ret; +} + +int pt_png_open (struct pt_image *image, struct pt_png_img *img) +{ + int err; + + // init + memset(img, 0, sizeof(*img)); + + // open I/O + if ((err = pt_image_open_file(image, &img->fh))) + JUMP_ERROR(err); + + // create the struct + if ((img->png = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)) == NULL) + JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); + + // create the info + if ((img->info = png_create_info_struct(img->png)) == NULL) + JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); + + // setup error trap for the I/O + if (setjmp(png_jmpbuf(img->png))) + JUMP_SET_ERROR(err, PT_ERR_PNG); + + // setup error trap + if (setjmp(png_jmpbuf(img->png))) + JUMP_SET_ERROR(err, PT_ERR_PNG); + + + // setup I/O to FILE + png_init_io(img->png, img->fh); + + // read meta-info + png_read_info(img->png, img->info); + + + // img->fh will be closed by pt_png_release_read + return 0; + +error: + // cleanup + pt_png_release_read(img); + + return err; +} + +int pt_png_read_header (struct pt_png_img *img, struct pt_png_header *header, size_t *data_size) +{ + // check image doesn't use any options we don't handle + if (png_get_interlace_type(img->png, img->info) != PNG_INTERLACE_NONE) { + log_warn("Can't handle interlaced PNG"); + + RETURN_ERROR(PT_ERR_IMG_FORMAT); + } + + + // initialize + memset(header, 0, sizeof(*header)); + + // fill in basic info + header->width = png_get_image_width(img->png, img->info); + header->height = png_get_image_height(img->png, img->info); + header->bit_depth = png_get_bit_depth(img->png, img->info); + header->color_type = png_get_color_type(img->png, img->info); + + log_debug("width=%u, height=%u, bit_depth=%u, color_type=%u", + header->width, header->height, header->bit_depth, header->color_type + ); + + // only pack 1 pixel per byte, changes rowbytes + if (header->bit_depth < 8) + png_set_packing(img->png); + + // fill in other info + header->row_bytes = png_get_rowbytes(img->png, img->info); + + // calculate bpp as num_channels * bpc + // this assumes the packed bit depth will be either 8 or 16 + header->col_bytes = png_get_channels(img->png, img->info) * (header->bit_depth == 16 ? 2 : 1); + + log_debug("row_bytes=%u, col_bytes=%u", header->row_bytes, header->col_bytes); + + // palette etc. + if (header->color_type == PNG_COLOR_TYPE_PALETTE) { + int num_palette; + png_colorp palette; + + if (png_get_PLTE(img->png, img->info, &palette, &num_palette) == 0) + // PLTE chunk not read? + RETURN_ERROR(PT_ERR_PNG); + + // should only be 256 of them at most + assert(num_palette <= PNG_MAX_PALETTE_LENGTH); + + // copy + header->num_palette = num_palette; + memcpy(&header->palette, palette, num_palette * sizeof(*palette)); + + log_debug("num_palette=%u", num_palette); + } + + // calculate data size + *data_size = header->height * header->row_bytes; + + return 0; +} + +/** + * Decode the PNG data directly to memory - not good for sparse backgrounds + */ +static int pt_png_decode_direct (struct pt_png_img *img, const struct pt_png_header *header, const struct pt_image_params *params, uint8_t *out) +{ + // write out raw image data a row at a time + for (size_t row = 0; row < header->height; row++) { + // read row data, non-interlaced + png_read_row(img->png, out + row * header->row_bytes, NULL); + } + + return 0; +} + +/** + * Decode the PNG data, filtering it for sparse regions + */ +static int pt_png_decode_sparse (struct pt_png_img *img, const struct pt_png_header *header, const struct pt_image_params *params, uint8_t *out) +{ + // one row of pixel data + uint8_t *row_buf; + + // alloc + if ((row_buf = malloc(header->row_bytes)) == NULL) + RETURN_ERROR(PT_ERR_MEM); + + // decode each row at a time + for (size_t row = 0; row < header->height; row++) { + // read row data, non-interlaced + png_read_row(img->png, row_buf, NULL); + + // skip background-colored regions to keep the cache file sparse + // ...in blocks of PT_CACHE_BLOCK_SIZE bytes + for (size_t col_base = 0; col_base < header->width; col_base += PT_IMG_BLOCK_SIZE) { + // size of this block in bytes + size_t block_size = min(PT_IMG_BLOCK_SIZE * header->col_bytes, header->row_bytes - col_base); + + // ...each pixel + for ( + size_t col = col_base; + + // BLOCK_SIZE * col_bytes wide, don't go over the edge + col < col_base + block_size; + + col += header->col_bytes + ) { + // test this pixel + if (bcmp(row_buf + col, params->background_color, header->col_bytes)) { + // differs + memcpy( + out + row * header->row_bytes + col_base, + row_buf + col_base, + block_size + ); + + // skip to next block + break; + } + } + + // skip this block + continue; + } + } + + return 0; +} + +int pt_png_decode (struct pt_png_img *img, const struct pt_png_header *header, const struct pt_image_params *params, uint8_t *out) +{ + int err; + + // decode + // XXX: it's an array, you silly + if (params->background_color) + err = pt_png_decode_sparse(img, header, params, out); + + else + err = pt_png_decode_direct(img, header, params, out); + + if (err) + return err; + + // finish off, ignore trailing data + png_read_end(img->png, NULL); + + return 0; +} + +int pt_png_info (struct pt_png_header *header, struct pt_image_info *info) +{ + // fill in info from header + info->img_width = header->width; + info->img_height = header->height; + info->img_bpp = header->bit_depth; + + return 0; +} + +/** + * libpng I/O callback: write out data + */ +static void pt_png_mem_write (png_structp png, png_bytep data, png_size_t length) +{ + struct pt_tile_mem *buf = png_get_io_ptr(png); + int err; + + // write to buffer + if ((err = pt_tile_mem_write(buf, data, length))) + // drop err, because png_error doesn't do formatted output + png_error(png, "pt_tile_mem_write: ..."); +} + +/** + * libpng I/O callback: flush buffered data + */ +static void pt_png_mem_flush (png_structp png_ptr) +{ + // no-op +} + + +/** + * Return a pointer to the pixel data on \a row, starting at \a col. + */ +static inline const void* tile_row_col (const struct pt_png_header *header, const uint8_t *data, size_t row, size_t col) +{ + return data + (row * header->row_bytes) + (col * header->col_bytes); +} + +/** + * Write raw tile image data, directly from the cache + */ +static int pt_png_encode_direct (struct pt_png_img *img, const struct pt_png_header *header, const uint8_t *data, const struct pt_tile_info *ti) +{ + for (size_t row = ti->y; row < ti->y + ti->height; row++) + // write data directly + // missing const... + png_write_row(img->png, (const png_bytep) tile_row_col(header, data, row, ti->x)); + + return 0; +} + +/** + * Fill in a clipped region of \a width_px pixels at the given row segment + */ +static inline void tile_row_fill_clip (const struct pt_png_header *header, png_byte *row, size_t width_px) +{ + // XXX: use a configureable background color, or full transparency? + memset(row, /* 0xd7 */ 0x00, width_px * header->col_bytes); +} + +/** + * Write clipped tile image data (a tile that goes over the edge of the actual image) by aligning the data from the cache as needed + */ +static int pt_png_encode_clipped (struct pt_png_img *img, const struct pt_png_header *header, const uint8_t *data, const struct pt_tile_info *ti) +{ + png_byte *rowbuf; + size_t row; + + // image data goes from (ti->x ... clip_x, ti->y ... clip_y), remaining region is filled + size_t clip_x, clip_y; + + + // fit the left/bottom edge against the image dimensions + clip_x = min(ti->x + ti->width, header->width); + clip_y = min(ti->y + ti->height, header->height); + + + // allocate buffer for a single row of image data + if ((rowbuf = malloc(ti->width * header->col_bytes)) == NULL) + RETURN_ERROR(PT_ERR_MEM); + + // how much data we actually have for each row, in px and bytes + // from [(tile x)---](clip x) + size_t row_px = clip_x - ti->x; + size_t row_bytes = row_px * header->col_bytes; + + // write the rows that we have + // from [(tile y]---](clip y) + for (row = ti->y; row < clip_y; row++) { + // copy in the actual tile data... + memcpy(rowbuf, tile_row_col(header, data, row, ti->x), row_bytes); + + // generate the data for the remaining, clipped, columns + tile_row_fill_clip(header, rowbuf + row_bytes, (ti->width - row_px)); + + // write + png_write_row(img->png, rowbuf); + } + + // generate the data for the remaining, clipped, rows + tile_row_fill_clip(header, rowbuf, ti->width); + + // write out the remaining rows as clipped data + for (; row < ti->y + ti->height; row++) + png_write_row(img->png, rowbuf); + + // ok + return 0; +} + +/** + * Write unscaled tile data + */ +static int pt_png_encode_unzoomed (struct pt_png_img *img, const struct pt_png_header *header, const uint8_t *data, const struct pt_tile_info *ti) +{ + int err; + + // set basic info + png_set_IHDR(img->png, img->info, ti->width, ti->height, header->bit_depth, header->color_type, + PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT + ); + + // set palette? + if (header->color_type == PNG_COLOR_TYPE_PALETTE) + // oops... missing const + png_set_PLTE(img->png, img->info, (png_colorp) header->palette, header->num_palette); + + // write meta-info + png_write_info(img->png, img->info); + + // our pixel data is packed into 1 pixel per byte (8bpp or 16bpp) + png_set_packing(img->png); + + // figure out if the tile clips + if (ti->x + ti->width <= header->width && ti->y + ti->height <= header->height) + // doesn't clip, just use the raw data + err = pt_png_encode_direct(img, header, data, ti); + + else + // fill in clipped regions + err = pt_png_encode_clipped(img, header, data, ti); + + return err; +} + +/** + * Manipulate powers of two + */ +static inline size_t scale_by_zoom_factor (size_t value, int z) +{ + if (z > 0) + return value << z; + + else if (z < 0) + return value >> -z; + + else + return value; +} + +#define ADD_AVG(l, r) (l) = ((l) + (r)) / 2 + +/** + * Converts a pixel's data into a png_color + */ +static inline void png_pixel_data (const png_color **outp, const struct pt_png_header *header, const uint8_t *data, size_t row, size_t col) +{ + // palette entry number + int p; + + switch (header->color_type) { + case PNG_COLOR_TYPE_PALETTE: + switch (header->bit_depth) { + case 8: + // 8bpp palette + p = *((uint8_t *) tile_row_col(header, data, row, col)); + + break; + + default : + // unknown + return; + } + + // hrhr - assume our working data is valid (or we have 255 palette entries, so it doesn't matter...) + assert(p < header->num_palette); + + // reference data from palette + *outp = &header->palette[p]; + + return; + + default : + // unknown pixel format + return; + } +} + +/** + * Write scaled tile data + */ +static int pt_png_encode_zoomed (struct pt_png_img *img, const struct pt_png_header *header, const uint8_t *data, const struct pt_tile_info *ti) +{ + // size of the image data in px + size_t data_width = scale_by_zoom_factor(ti->width, ti->zoom); + size_t data_height = scale_by_zoom_factor(ti->height, ti->zoom); + + // input pixels per output pixel + size_t pixel_size = scale_by_zoom_factor(1, ti->zoom); + + // bytes per output pixel + size_t pixel_bytes = 3; + + // size of the output tile in px + size_t row_width = ti->width; + + // size of an output row in bytes (RGB) + size_t row_bytes = row_width * 3; + + // buffer to hold output rows + uint8_t *row_buf; + + // color entry for pixel + const png_color *c = &header->palette[0]; + + // only supports zooming out... + if (ti->zoom < 0) + RETURN_ERROR(PT_ERR_TILE_ZOOM); + + if ((row_buf = malloc(row_bytes)) == NULL) + RETURN_ERROR(PT_ERR_MEM); + + // suppress warning... + (void) data_height; + + // define pixel format: 8bpp RGB + png_set_IHDR(img->png, img->info, ti->width, ti->height, 8, PNG_COLOR_TYPE_RGB, + PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT + ); + + // write meta-info + png_write_info(img->png, img->info); + + // ...each output row + for (size_t out_row = 0; out_row < ti->height; out_row++) { + memset(row_buf, 0, row_bytes); + + // ...includes pixels starting from this row. + size_t in_row_offset = ti->y + scale_by_zoom_factor(out_row, ti->zoom); + + // ...each out row includes pixel_size in rows + for (size_t in_row = in_row_offset; in_row < in_row_offset + pixel_size && in_row < header->height; in_row++) { + // and includes each input pixel + for (size_t in_col = ti->x; in_col < ti->x + data_width && in_col < header->width; in_col++) { + + // ...for this output pixel + size_t out_col = scale_by_zoom_factor(in_col - ti->x, -ti->zoom); + + // get pixel RGB data + png_pixel_data(&c, header, data, in_row, in_col); + + // average the RGB data + ADD_AVG(row_buf[out_col * pixel_bytes + 0], c->red); + ADD_AVG(row_buf[out_col * pixel_bytes + 1], c->green); + ADD_AVG(row_buf[out_col * pixel_bytes + 2], c->blue); + } + } + + // output + png_write_row(img->png, row_buf); + } + + // done + return 0; +} + +int pt_png_tile (const struct pt_png_header *header, const uint8_t *data, struct pt_tile *tile) +{ + struct pt_png_img _img, *img = &_img; + struct pt_tile_info *ti = &tile->info; + int err; + + // init img + memset(img, 0, sizeof(*img)); + + // check within bounds + if (ti->x >= header->width || ti->y >= header->height) + // completely outside + RETURN_ERROR(PT_ERR_TILE_CLIP); + + // open PNG writer + if ((img->png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)) == NULL) + JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); + + if ((img->info = png_create_info_struct(img->png)) == NULL) + JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); + + // libpng error trap + if (setjmp(png_jmpbuf(img->png))) + JUMP_SET_ERROR(err, PT_ERR_PNG); + + + + // setup output I/O + switch (tile->out_type) { + case PT_TILE_OUT_FILE: + // use default FILE* operation + // do NOT store in img->fh + png_init_io(img->png, tile->out.file); + + break; + + case PT_TILE_OUT_MEM: + // use pt_tile_mem struct via pt_png_mem_* callbacks + png_set_write_fn(img->png, &tile->out.mem, pt_png_mem_write, pt_png_mem_flush); + + break; + + default: + FATAL("tile->out_type: %d", tile->out_type); + } + + + + // unscaled or scaled? + if (ti->zoom) + err = pt_png_encode_zoomed(img, header, data, ti); + + else + err = pt_png_encode_unzoomed(img, header, data, ti); + + if (err) + goto error; + + + // flush remaining output + png_write_flush(img->png); + + // done + png_write_end(img->png, img->info); + +error: + // cleanup + pt_png_release_write(img); + + return err; +} + + +void pt_png_release_read (struct pt_png_img *img) +{ + png_destroy_read_struct(&img->png, &img->info, NULL); + + // close possible filehandle + if (img->fh) { + if (fclose(img->fh)) + log_warn_errno("fclose"); + } +} + +void pt_png_release_write (struct pt_png_img *img) +{ + png_destroy_write_struct(&img->png, &img->info); + + // close possible filehandle + if (img->fh) { + if (fclose(img->fh)) + log_warn_errno("fclose"); + } + +} + diff -r aaae02944832 -r 4e6e067b3472 src/lib/png.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/png.h Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,97 @@ +#ifndef PNGTILE_PNG_H +#define PNGTILE_PNG_H + +/** + * @file + * PNG-specific handling + */ +#include +#include + +/** + * Handle sparse data at this granularity (pixels) + */ +#define PT_IMG_BLOCK_SIZE 64 + +/** + * PNG img state + */ +struct pt_png_img { + /** libpng state */ + png_struct *png; + png_info *info; + + /** Possible opened I/O file */ + FILE *fh; +}; + +/** + * Cache header layout for PNG-format images + */ +struct pt_png_header { + /** Pixel dimensions of image */ + uint32_t width, height; + + /** Pixel format */ + uint8_t bit_depth, color_type; + + /** Number of png_color entries that follow */ + uint16_t num_palette; + + /** Number of bytes per row */ + uint32_t row_bytes; + + /** Number of bytes per pixel */ + uint8_t col_bytes; + + /** Palette entries, up to 256 entries used */ + png_color palette[PNG_MAX_PALETTE_LENGTH]; +}; + + +#include "image.h" +#include "tile.h" + +/** + * Check if the given path looks like a .png file. + * + * Returns 0 if ok, 1 if non-png file, -1 on error + */ +int pt_png_check (const char *path); + +/** + * Open the given .png image, and read info + */ +int pt_png_open (struct pt_image *image, struct pt_png_img *img); + +/** + * Fill in the PNG header and return the size of the pixel data + */ +int pt_png_read_header (struct pt_png_img *img, struct pt_png_header *header, size_t *data_size); + +/** + * Decode the PNG data into the given data segment, using the header as decoded by pt_png_read_header + */ +int pt_png_decode (struct pt_png_img *img, const struct pt_png_header *header, const struct pt_image_params *params, uint8_t *out); + +/** + * Fill in img_* fields of pt_image_info from header + */ +int pt_png_info (struct pt_png_header *header, struct pt_image_info *info); + +/** + * Render out a tile + */ +int pt_png_tile (const struct pt_png_header *header, const uint8_t *data, struct pt_tile *tile); + +/** + * Release pt_png_ctx resources as allocated by pt_png_open + */ +void pt_png_release_read (struct pt_png_img *img); + +/** + * Release pt_png_ctx resources as allocated by pt_png_... + */ +void pt_png_release_write (struct pt_png_img *img); + +#endif diff -r aaae02944832 -r 4e6e067b3472 src/lib/tile.c --- a/src/lib/tile.c Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/tile.c Sun Sep 14 15:24:58 2014 +0300 @@ -3,6 +3,34 @@ #include "shared/log.h" // only FATAL #include +#include + +int pt_tile_mem_write (struct pt_tile_mem *buf, void *data, size_t len) +{ + size_t buf_len = buf->len; + + // grow? + while (buf->off + len > buf_len) + buf_len *= 2; + + if (buf_len != buf->len) { + char *tmp; + + if ((tmp = realloc(buf->base, buf_len)) == NULL) + RETURN_ERROR(PT_ERR_MEM); + + buf->base = tmp; + buf->len = buf_len; + } + + // copy + memcpy(buf->base + buf->off, data, len); + + buf->off += len; + + return 0; +} + int pt_tile_new (struct pt_tile **tile_ptr) { @@ -47,84 +75,14 @@ return 0; } -static void pt_tile_mem_write (png_structp png, png_bytep data, png_size_t length) -{ - struct pt_tile_mem *buf = png_get_io_ptr(png); - size_t buf_len = buf->len; - - // grow? - while (buf->off + length > buf_len) - buf_len *= 2; - - if (buf_len != buf->len) { - char *tmp; - - if ((tmp = realloc(buf->base, buf_len)) == NULL) - png_error(png, "pt_tile_buf_write - realloc failed"); - - buf->base = tmp; - buf->len = buf_len; - } - - // copy - memcpy(buf->base + buf->off, data, length); - - buf->off += length; -} - -static void pt_tile_mem_flush (png_structp png_ptr) -{ - // no-op -} - int pt_tile_render (struct pt_tile *tile) { - png_structp png = NULL; - png_infop info = NULL; - int err = 0; - - // open PNG writer - if ((png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)) == NULL) - JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); - - if ((info = png_create_info_struct(png)) == NULL) - JUMP_SET_ERROR(err, PT_ERR_PNG_CREATE); - - // libpng error trap - if (setjmp(png_jmpbuf(png))) - JUMP_SET_ERROR(err, PT_ERR_PNG); - - // setup output I/O - switch (tile->out_type) { - case PT_TILE_OUT_FILE: - // use default FILE* operation - png_init_io(png, tile->out.file); + // validate dimensions + if (!tile->info.width || !tile->info.height) + RETURN_ERROR(PT_ERR_TILE_DIM); - break; - - case PT_TILE_OUT_MEM: - // use pt_tile_mem struct via pt_tile_mem_* callbacks - png_set_write_fn(png, &tile->out.mem, pt_tile_mem_write, pt_tile_mem_flush); - - break; - - default: - FATAL("tile->out_type: %d", tile->out_type); - } - - // render tile - if ((err = pt_cache_tile_png(tile->cache, png, info, &tile->info))) - JUMP_ERROR(err); - - // done - png_write_end(png, info); - -error: - // cleanup - png_destroy_write_struct(&png, &info); - - return err; + return pt_cache_tile(tile->cache, tile); } void pt_tile_abort (struct pt_tile *tile) diff -r aaae02944832 -r 4e6e067b3472 src/lib/tile.h --- a/src/lib/tile.h Sun Sep 14 15:19:59 2014 +0300 +++ b/src/lib/tile.h Sun Sep 14 15:24:58 2014 +0300 @@ -2,8 +2,12 @@ #define PNGTILE_TILE_H /** + * @file * Generating PNG tiles from a cache */ + +struct pt_tile; + #include "pngtile.h" #include "cache.h" @@ -42,6 +46,12 @@ }; /** + * Write to the tile's output buffer + */ +int pt_tile_mem_write (struct pt_tile_mem *buf, void *data, size_t len); + + +/** * Alloc a new pt_tile, which must be initialized using pt_tile_init_* */ int pt_tile_new (struct pt_tile **tile_ptr); diff -r aaae02944832 -r 4e6e067b3472 src/pngtile/main.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pngtile/main.c Sun Sep 14 15:24:58 2014 +0300 @@ -0,0 +1,435 @@ +#include "shared/log.h" + +#include "pngtile.h" + +#include +#include +#include +#include +#include + +enum option_names { + + _OPT_LONGONLY = 255, + + OPT_BENCHMARK, + OPT_RANDOMIZE, +}; + +/** + * Command-line options + */ +static const struct option options[] = { + { "help", false, NULL, 'h' }, + { "quiet", false, NULL, 'q' }, + { "verbose", false, NULL, 'v' }, + { "debug", false, NULL, 'D' }, + { "force-update", false, NULL, 'U' }, + { "no-update", false, NULL, 'N' }, + { "background", true, NULL, 'B' }, + { "width", true, NULL, 'W' }, + { "height", true, NULL, 'H' }, + { "x", true, NULL, 'x' }, + { "y", true, NULL, 'y' }, + { "zoom", true, NULL, 'z' }, + { "out", true, NULL, 'o' }, + { "threads", true, NULL, 'j' }, + + // --long-only options + { "benchmark", true, NULL, OPT_BENCHMARK }, + { "randomize", false, NULL, OPT_RANDOMIZE }, + { 0, 0, 0, 0 } +}; + +/** + * Print usage/help info on stderr + */ +void help (const char *argv0) +{ + fprintf(stderr, "Usage: %s [options] [...]\n", argv0); + fprintf(stderr, + "Open each of the given image files, check cache status, optionally update their cache, display image info, and\n" + "optionally render a tile of each.\n" + "\n" + "\t-h, --help show this help and exit\n" + "\t-q, --quiet supress informational output\n" + "\t-v, --verbose display more informational output\n" + "\t-D, --debug equivalent to -v\n" + "\t-U, --force-update unconditionally update image caches\n" + "\t-N, --no-update do not update the image cache\n" + "\t-B, --background set background pattern for sparse cache file: 0xHH..\n" + "\t-W, --width PX set tile width\n" + "\t-H, --height PX set tile height\n" + "\t-x, --x PX set tile x offset\n" + "\t-y, --y PX set tile y offset\n" + "\t-z, --zoom ZL set zoom factor (<0)\n" + "\t-o, --out FILE set tile output file\n" + "\t-j, --threads N number of threads\n" + "\t--benchmark N do N tile renders\n" + "\t--randomize randomize tile x/y coords\n" + ); +} + +unsigned long parse_uint (const char *val, const char *name) +{ + char *endptr; + long int out; + + // decode + out = strtol(val, &endptr, 0); + + // validate + if (*endptr || out < 0) + EXIT_ERROR(EXIT_FAILURE, "Invalid value for %s: %s", name, val); + + // ok + return out; +} + +signed long parse_sint (const char *val, const char *name) +{ + char *endptr; + long int out; + + // decode + out = strtol(val, &endptr, 0); + + // validate + if (*endptr) + EXIT_ERROR(EXIT_FAILURE, "Invalid value for %s: %s", name, val); + + // ok + return out; +} + +long randrange (long start, long end) +{ + return start + (rand() * (end - start) / RAND_MAX); +} + +/** + * Randomize tile x/y with given image info + */ +void randomize_tile (struct pt_tile_info *ti, const struct pt_image_info *info) +{ + ti->x = randrange(0, info->img_width - ti->width); + ti->y = randrange(0, info->img_height - ti->height); +} + +/** + * Render a tile + */ +int do_tile (struct pt_ctx *ctx, struct pt_image *image, const struct pt_tile_info *ti, const char *out_path) +{ + FILE *out_file = NULL; + char tmp_name[] = "pt-tile-XXXXXX"; + int err = 0; + + if (!out_path) { + int fd; + + // temporary file for output + if ((fd = mkstemp(tmp_name)) < 0) { + log_errno("mkstemp"); + goto error; + } + + out_path = tmp_name; + + // open out + if ((out_file = fdopen(fd, "wb")) == NULL) { + log_errno("fdopen"); + goto error; + } + + } else if (strcmp(out_path, "-") == 0) { + // use stdout + if ((out_file = fdopen(STDOUT_FILENO, "wb")) == NULL) { + log_errno("fdopen: STDOUT_FILENO"); + goto error; + } + + } else { + // use file + if ((out_file = fopen(out_path, "wb")) == NULL) { + log_errno("fopen: %s", out_path); + goto error; + } + + } + + + if (ctx) { + // render + log_info("\tAsync render tile %zux%zu@(%zu,%zu) -> %s", ti->width, ti->height, ti->x, ti->y, out_path); + + if ((err = pt_image_tile_async(image, ti, out_file))) { + log_errno("pt_image_tile_async: %s", pt_strerror(err)); + goto error; + } + + // will close it itself + out_file = NULL; + + } else { + // render + log_info("\tRender tile %zux%zu@(%zu,%zu) -> %s", ti->width, ti->height, ti->x, ti->y, out_path); + + if ((err = pt_image_tile_file(image, ti, out_file))) { + log_errno("pt_image_tile_file: %s", pt_strerror(err)); + goto error; + } + } + +error: + // cleanup + if (out_file && fclose(out_file)) + log_warn_errno("fclose: out_file"); + + return err; +} + +int main (int argc, char **argv) +{ + int opt; + bool force_update = false, no_update = false, randomize = false; + struct pt_tile_info ti = { + .width = 800, + .height = 600, + .x = 0, + .y = 0, + .zoom = 0 + }; + struct pt_image_params update_params = { }; + const char *out_path = NULL; + int threads = 0, benchmark = 0; + int err; + + // parse arguments + while ((opt = getopt_long(argc, argv, "hqvDUNB:W:H:x:y:z:o:j:", options, NULL)) != -1) { + switch (opt) { + case 'h': + // display help + help(argv[0]); + + return EXIT_SUCCESS; + + case 'q': + // supress excess log output + set_log_level(LOG_WARN); break; + + case 'v': + case 'D': + // display additional output + set_log_level(LOG_DEBUG); break; + + case 'U': + // force update of image caches + force_update = true; break; + + case 'N': + // supress update of image caches + no_update = true; break; + + case 'B': + // background pattern + { + unsigned int b1 = 0, b2 = 0, b3 = 0, b4 = 0; + + // parse 0xXXXXXXXX + if (sscanf(optarg, "0x%02x%02x%02x%02x", &b1, &b2, &b3, &b4) < 1) + FATAL("Invalid hex value for -B/--background: %s", optarg); + + // store + update_params.background_color[0] = b1; + update_params.background_color[1] = b2; + update_params.background_color[2] = b3; + update_params.background_color[3] = b4; + + } break; + + case 'W': + ti.width = parse_uint(optarg, "--width"); break; + + case 'H': + ti.height = parse_uint(optarg, "--height"); break; + + case 'x': + ti.x = parse_uint(optarg, "--x"); break; + + case 'y': + ti.y = parse_uint(optarg, "--y"); break; + + case 'z': + ti.zoom = parse_sint(optarg, "--zoom"); break; + + case 'o': + // output file + out_path = optarg; break; + + case 'j': + threads = parse_uint(optarg, "--threads"); break; + + case OPT_BENCHMARK: + benchmark = parse_uint(optarg, "--benchmark"); break; + + case OPT_RANDOMIZE: + randomize = true; break; + + case '?': + // useage error + help(argv[0]); + + return EXIT_FAILURE; + + default: + // getopt??? + FATAL("getopt_long returned unknown code %d", opt); + } + } + + // end-of-arguments? + if (!argv[optind]) + EXIT_WARN(EXIT_FAILURE, "No images given"); + + + + struct pt_ctx *ctx = NULL; + struct pt_image *image = NULL; + enum pt_cache_status status; + + if (threads) { + // build ctx + log_debug("Construct pt_ctx with %d threads", threads); + + if ((err = pt_ctx_new(&ctx, threads))) + EXIT_ERROR(EXIT_FAILURE, "pt_ctx_new: threads=%d", threads); + } + + // process each image in turn + log_debug("Processing %d images...", argc - optind); + + for (int i = optind; i < argc; i++) { + const char *img_path = argv[i]; + + log_debug("Loading image from: %s...", img_path); + + // open + if ((err = pt_image_open(&image, ctx, img_path, PT_OPEN_UPDATE))) { + log_errno("pt_image_open: %s: %s", img_path, pt_strerror(err)); + continue; + } + + log_info("Opened image at: %s", img_path); + + // check if stale + if ((status = pt_image_status(image)) < 0) { + log_errno("pt_image_status: %s: %s", img_path, pt_strerror(status)); + goto error; + } + + // update if stale + if (status != PT_CACHE_FRESH || force_update) { + if (status == PT_CACHE_NONE) + log_info("\tImage cache is missing"); + + else if (status == PT_CACHE_STALE) + log_info("\tImage cache is stale"); + + else if (status == PT_CACHE_INCOMPAT) + log_info("\tImage cache is incompatible"); + + else if (status == PT_CACHE_FRESH) + log_info("\tImage cache is fresh"); + + else + log_warn("\tImage cache status is unknown"); + + if (!no_update) { + log_info("\tUpdating image cache..."); + + if ((err = pt_image_update(image, &update_params))) { + log_error("pt_image_update: %s: %s", img_path, pt_strerror(err)); + goto error; + } + + log_debug("\tUpdated image cache"); + + } else { + log_warn("\tSupressing cache update"); + } + + } else { + log_debug("\tImage cache is fresh"); + + // ensure it's loaded + log_info("\tLoad image cache..."); + + if ((err = pt_image_load(image))) { + log_errno("pt_image_load: %s", pt_strerror(err)); + goto error; + } + } + + // show info + const struct pt_image_info *info; + + if ((err = pt_image_info(image, &info))) { + log_warn_errno("pt_image_info: %s: %s", img_path, pt_strerror(err)); + + } else { + log_info("\tImage dimensions: %zux%zu (%zu bpp)", info->img_width, info->img_height, info->img_bpp); + log_info("\tImage mtime=%ld, bytes=%zu", (long) info->image_mtime, info->image_bytes); + log_info("\tCache mtime=%ld, bytes=%zu, blocks=%zu (%zu bytes), version=%d", + (long) info->cache_mtime, info->cache_bytes, info->cache_blocks, info->cache_blocks * 512, info->cache_version + ); + } + + // render tile? + if (benchmark) { + log_info("\tRunning %d %stile renders...", benchmark, randomize ? "randomized " : ""); + + // n times + for (int i = 0; i < benchmark; i++) { + // randomize x, y + if (randomize) + randomize_tile(&ti, info); + + if (do_tile(ctx, image, &ti, out_path)) + goto error; + } + + } else if (out_path) { + // randomize x, y + if (randomize) + randomize_tile(&ti, info); + + // just once + if (do_tile(ctx, image, &ti, out_path)) + goto error; + + } + // cleanup + // XXX: leak because of async + if (!ctx) + pt_image_destroy(image); + + continue; + +error: + // quit + EXIT_ERROR(EXIT_FAILURE, "Processing image failed: %s", img_path); + } + + if (ctx) { + log_info("Waiting for images to finish..."); + + // wait for tile operations to finish... + pt_ctx_shutdown(ctx); + } + + log_info("Done"); + + return 0; +} + diff -r aaae02944832 -r 4e6e067b3472 src/shared/log.c --- a/src/shared/log.c Sun Sep 14 15:19:59 2014 +0300 +++ b/src/shared/log.c Sun Sep 14 15:24:58 2014 +0300 @@ -45,7 +45,7 @@ size_t str_append_fmt_va (char *buf_ptr, size_t *buf_size, const char *fmt, va_list args) { - int ret; + int ret = 0; if (*buf_size && (ret = vsnprintf(buf_ptr, *buf_size, fmt, args)) < 0) return 0; diff -r aaae02944832 -r 4e6e067b3472 src/util/main.c --- a/src/util/main.c Sun Sep 14 15:19:59 2014 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,239 +0,0 @@ -#include "shared/log.h" - -#include "pngtile.h" - -#include -#include -#include - -/** - * Command-line options - */ -static const struct option options[] = { - { "help", false, NULL, 'h' }, - { "quiet", false, NULL, 'q' }, - { "verbose", false, NULL, 'v' }, - { "debug", false, NULL, 'D' }, - { "force-update", false, NULL, 'U' }, - { "width", true, NULL, 'W' }, - { "height", true, NULL, 'H' }, - { "x", true, NULL, 'x' }, - { "y", true, NULL, 'y' }, - { "zoom", true, NULL, 'z' }, - { "threads", true, NULL, 'j' }, - { 0, 0, 0, 0 } -}; - -/** - * Print usage/help info on stderr - */ -void help (const char *argv0) -{ - fprintf(stderr, "Usage: %s [options] [...]\n", argv0); - fprintf(stderr, - "XXX: Process some image files.\n" - "\n" - "\t-h, --help show this help and exit\n" - "\t-q, --quiet supress informational output\n" - "\t-v, --verbose display more informational output\n" - "\t-D, --debug equivalent to -v\n" - "\t-U, --force-update unconditionally update image caches\n" - "\t-W, --width set tile width\n" - "\t-H, --height set tile height\n" - "\t-x, --x set tile x offset\n" - "\t-y, --y set tile z offset\n" - "\t-z, --zoom set zoom factor (<0)\n" - "\t-j, --threads number of threads\n" - ); -} - -int main (int argc, char **argv) -{ - int opt; - bool force_update = false; - struct pt_tile_info ti = {0, 0, 0, 0, 0}; - int threads = 2; - int tmp, err; - - // parse arguments - while ((opt = getopt_long(argc, argv, "hqvDUW:H:x:y:z:j:", options, NULL)) != -1) { - switch (opt) { - case 'h': - // display help - help(argv[0]); - - return EXIT_SUCCESS; - - case 'q': - // supress excess log output - set_log_level(LOG_WARN); - - break; - - case 'v': - case 'D': - // display additional output - set_log_level(LOG_DEBUG); - - break; - - case 'U': - // force update of image caches - force_update = true; - - break; - - case 'W': - ti.width = strtol(optarg, NULL, 0); break; - - case 'H': - ti.height = strtol(optarg, NULL, 0); break; - - case 'x': - ti.x = strtol(optarg, NULL, 0); break; - - case 'y': - ti.y = strtol(optarg, NULL, 0); break; - - case 'z': - ti.zoom = strtol(optarg, NULL, 0); break; - - case 'j': - if ((tmp = strtol(optarg, NULL, 0)) < 1) - FATAL("Invalid value for -j/--threads"); - - threads = tmp; break; - - case '?': - // useage error - help(argv[0]); - - return EXIT_FAILURE; - - default: - // getopt??? - FATAL("getopt_long returned unknown code %d", opt); - } - } - - // end-of-arguments? - if (!argv[optind]) - EXIT_WARN(EXIT_FAILURE, "No images given"); - - - - struct pt_ctx *ctx = NULL; - struct pt_image *image = NULL; - enum pt_cache_status status; - - // build ctx - log_debug("Construct pt_ctx with %d threads", threads); - - if ((err = pt_ctx_new(&ctx, threads))) - EXIT_ERROR(EXIT_FAILURE, "pt_ctx_new: threads=%d", threads); - - - // process each image in turn - log_debug("Processing %d images...", argc - optind); - - for (int i = optind; i < argc; i++) { - const char *img_path = argv[i]; - - log_debug("Loading image from: %s...", img_path); - - // open - if ((err = pt_image_open(&image, ctx, img_path, PT_OPEN_UPDATE))) { - log_errno("pt_image_open: %s: %s", img_path, pt_strerror(err)); - continue; - } - - log_info("Opened image at: %s", img_path); - - // check if stale - if ((status = pt_image_status(image)) < 0) { - log_errno("pt_image_status: %s: %s", img_path, pt_strerror(status)); - goto error; - } - - // update if stale - if (status != PT_CACHE_FRESH || force_update) { - if (status == PT_CACHE_NONE) - log_debug("\tImage cache is missing"); - - else if (status == PT_CACHE_STALE) - log_debug("\tImage cache is stale"); - - else if (status == PT_CACHE_FRESH) - log_debug("\tImage cache is fresh"); - - log_debug("\tUpdating image cache..."); - - if ((err = pt_image_update(image))) { - log_warn_errno("pt_image_update: %s: %s", img_path, pt_strerror(err)); - } - - log_info("\tUpdated image cache"); - - } else { - log_debug("\tImage cache is fresh"); - } - - // show info - const struct pt_image_info *img_info; - - if ((err = pt_image_info(image, &img_info))) - log_warn_errno("pt_image_info: %s: %s", img_path, pt_strerror(err)); - - else - log_info("\tImage dimensions: %zux%zu", img_info->width, img_info->height); - - // render tile? - if (ti.width && ti.height) { - char tmp_name[] = "pt-tile-XXXXXX"; - int fd; - FILE *out; - - // temporary file for output - if ((fd = mkstemp(tmp_name)) < 0) { - log_errno("mkstemp"); - - continue; - } - - // open out - if ((out = fdopen(fd, "w")) == NULL) { - log_errno("fdopen"); - - continue; - } - - // ensure it's loaded - log_debug("\tLoad image cache..."); - - if ((err = pt_image_load(image))) - log_errno("pt_image_load: %s", pt_strerror(err)); - - // render - log_info("\tAsync render tile %zux%zu@(%zu,%zu) -> %s", ti.width, ti.height, ti.x, ti.y, tmp_name); - - - if ((err = pt_image_tile_async(image, &ti, out))) - log_errno("pt_image_tile: %s: %s", img_path, pt_strerror(err)); - } - -error: - // cleanup - // XXX: leak because of async: pt_image_destroy(image); - ; - } - - log_info("Waiting for images to finish..."); - - // wait for tile operations to finish... - pt_ctx_shutdown(ctx); - - log_info("Done"); - - return 0; -} - diff -r aaae02944832 -r 4e6e067b3472 static/dragdrop.js --- a/static/dragdrop.js Sun Sep 14 15:19:59 2014 +0300 +++ b/static/dragdrop.js Sun Sep 14 15:24:58 2014 +0300 @@ -240,8 +240,7 @@ style.top = p[1] + "px"; if(style.visibility=="hidden") style.visibility = ""; // fix gecko rendering - }, - + } }); Draggable._dragging = { }; diff -r aaae02944832 -r 4e6e067b3472 static/index.html --- a/static/index.html Sun Sep 14 15:19:59 2014 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ - - - Mandelbrot - - - - - - - -
    -
    -
    -
    -
    - - - - diff -r aaae02944832 -r 4e6e067b3472 static/style.css --- a/static/style.css Sun Sep 14 15:19:59 2014 +0300 +++ b/static/style.css Sun Sep 14 15:24:58 2014 +0300 @@ -45,3 +45,13 @@ background-color: #FFFFFF; } + +.background { + z-index: 1; + + font-size: xx-large; + + text-align: center; + + margin: 30%; +} diff -r aaae02944832 -r 4e6e067b3472 static/tiles2.js --- a/static/tiles2.js Sun Sep 14 15:19:59 2014 +0300 +++ b/static/tiles2.js Sun Sep 14 15:24:58 2014 +0300 @@ -1,36 +1,60 @@ // A source of tile images of a specific width/height, zoom level range, and some other attributes + +/** + * A source of image tiles. + * + * The image source is expected to serve fixed-size tiles (tile_width × tile_height) of image data based on the + * x, y, zl URL query parameters. + * + * x, y - the pixel coordinates of the top-left corner + * XXX: these are scaled from the image coordinates by the zoom level + * + * zl - the zoom level used, in < zl < out. + * The image pixels are scaled by powers-of-two, so a 256x256 tile at zl=1 shows a 512x512 area of the + * 1:1 image. + */ var Source = Class.create({ - initialize: function (path, tile_width, tile_height, zoom_min, zoom_max) { + initialize: function (path, tile_width, tile_height, zoom_min, zoom_max, img_width, img_height) { this.path = path; this.tile_width = tile_width; this.tile_height = tile_height; this.zoom_min = zoom_min; this.zoom_max = zoom_max; + this.img_width = img_width; + this.img_height = img_height; this.refresh = false; this.opt_key = this.opt_value = null; }, - // build a URL for the given tile image - build_url: function (col, row, zl, sw, sh) { - // two-bit hash (0-4) based on the (col, row) - var hash = ( (col % 2) << 1 | (row % 2) ) + 1; - - // the subdomain to use - var subdomain = ""; - - if (0) + /** + * Return an URL representing the tile at the given viewport (row, col) at the given zl. + * + * XXX: sw/sh = screen-width/height, used to choose an appropriate output size for dynamic image sources... + */ + build_url: function (col, row, zl /*, sw, sh */) { + // XXX: distribute tile requests across tile*.foo.bar + if (0) { + // two-bit hash (0-4) based on the (col, row) + var hash = ( (col % 2) << 1 | (row % 2) ) + 1; + + // the subdomain to use + var subdomain = ""; + subdomain = "tile" + hash + "."; + } // the (x, y) co-ordinates var x = col * this.tile_width; var y = row * this.tile_height; var url = this.path + "?x=" + x + "&y=" + y + "&zl=" + zl; // + "&sw=" + sw + "&sh=" + sh; - + + // refresh the tile each time it is loaded? if (this.refresh) url += "&ts=" + new Date().getTime(); - + + // XXX: additional parameters, not used if (this.opt_key && this.opt_value) url += "&" + this.opt_key + "=" + this.opt_value; @@ -43,40 +67,38 @@ } }); -// a viewport that contains a substrate which contains several zoom layers which contain many tiles +/** + * Viewport implements the tiles-UI. It consists of a draggable substrate, which in turn consists of several + * ZoomLayers, which then contain the actual tile images. + * + * Vars: + * scroll_x/y - the visible pixel offset of the top-left corner + */ var Viewport = Class.create({ initialize: function (source, viewport_id) { this.source = source; - + + // get a handle on the UI elements this.id = viewport_id; this.div = $(viewport_id); this.substrate = this.div.down("div.substrate"); - - // the stack of zoom levels - this.zoom_layers = []; - - // pre-populate the stack - for (var zoom_level = source.zoom_min; zoom_level <= source.zoom_max; zoom_level++) { - var zoom_layer = new ZoomLayer(source, zoom_level); - - this.substrate.appendChild(zoom_layer.div); - this.zoom_layers[zoom_level] = zoom_layer; - } // make the substrate draggable this.draggable = new Draggable(this.substrate, { onStart: this.on_scroll_start.bind(this), onDrag: this.on_scroll_move.bind(this), onEnd: this.on_scroll_end.bind(this), + + zindex: false }); - // event handlers + // register event handlers for other UI functions Event.observe(this.substrate, "dblclick", this.on_dblclick.bindAsEventListener(this)); Event.observe(this.substrate, "mousewheel", this.on_mousewheel.bindAsEventListener(this)); Event.observe(this.substrate, "DOMMouseScroll", this.on_mousewheel.bindAsEventListener(this)); // mozilla Event.observe(document, "resize", this.on_resize.bindAsEventListener(this)); - // zoom buttons + // init zoom UI buttons this.btn_zoom_in = $("btn-zoom-in"); this.btn_zoom_out = $("btn-zoom-out"); @@ -85,54 +107,98 @@ if (this.btn_zoom_out) Event.observe(this.btn_zoom_out, "click", this.zoom_out.bindAsEventListener(this)); - - // set viewport size - this.update_size(); + + // initial view location (centered) + var cx = this.source.img_width / 2; + var cy = this.source.img_height / 2; + var zl = 0; // XXX: would need to scale x/y for this: (this.source.zoom_min + this.source.zoom_max) / 2; - // this comes after update_size, since it must be updated once we have the size and zoom layer... - this.image_link = $("lnk-image"); - - // initial location? + // from link? if (document.location.hash) { - // x:y:z tuple + // parse x:y:z tuple var pt = document.location.hash.substr(1).split(":"); - // unpack - var cx = 0, cy = 0, z = 0; - - if (pt.length) cx = parseInt(pt.shift()); - if (pt.length) cy = parseInt(pt.shift()); - if (pt.length) z = parseInt(pt.shift()); + // unpack + if (pt.length) cx = parseInt(pt.shift()) || cx; + if (pt.length) cy = parseInt(pt.shift()) || cy; + if (pt.length) zl = parseInt(pt.shift()) || zl; + } - // initial view - this.zoom_scaled( - cx - this.center_offset_x, - cy - this.center_offset_y, - z - ); + // initialize zoom state to given zl + this._init_zoom(zl); + + // initialize viewport size + this.update_size(); + + // initialize scroll offset + this._init_scroll(cx, cy); + + // this comes after update_size, so that the initial update_size doesn't try and update the image_link, + // since this only works once we have the zoom layers set up... + this.image_link = $("lnk-image"); + this.update_image_link(); - } else { - // this sets the scroll offsets, zoom level, and loads the tiles - this.zoom_to(0, 0, 0); + // display tiles! + this.update_tiles(); + }, + +/* + * Initializers - only run once + */ + + /** Initialize the zoom state to show the given zoom level */ + _init_zoom: function (zl) { + // the stack of zoom levels + this.zoom_layers = []; + + // populate the zoom-layers stack based on the number of zoom levels defined for the source + for (var zoom_level = this.source.zoom_min; zoom_level <= this.source.zoom_max; zoom_level++) { + var zoom_layer = new ZoomLayer(this.source, zoom_level); + + this.substrate.appendChild(zoom_layer.div); + this.zoom_layers[zoom_level] = zoom_layer; } + + // is the new zoom level valid? + if (!this.zoom_layers[zl]) + // XXX: nasty, revert to something else? + return false; + + // set the zoom layyer + this.zoom_layer = this.zoom_layers[zl]; + + // enable it with initial z-index + this.zoom_layer.enable(11); + + // init the UI accordingly + this.update_zoom_ui(); }, + + /** Initialize the scroll state to show the given (scaled) centered coordinates */ + _init_scroll: function (cx, cy) { + // scroll center + this.scroll_center_to(cx, cy); + }, + - /* event handlers */ +/* + * Handle input events + */ - // window resized + /** Viewport resized */ on_resize: function (ev) { this.update_size(); this.update_tiles(); }, - // double-click handler + /** Double-click to zoom and center */ on_dblclick: function (ev) { var offset = this.event_offset(ev); - this.zoom_center_to( + // center view and zoom in + this.center_and_zoom_in( this.scroll_x + offset.x, - this.scroll_y + offset.y, - 1 // zoom in + this.scroll_y + offset.y ); }, @@ -168,7 +234,7 @@ // delta > 0 : scroll up, zoom in // delta < 0 : scroll down, zoom out - delta = delta < 0 ? -1 : 1; + delta = delta < 0 ? 1 : -1; // Firefox's DOMMouseEvent's pageX/Y attributes are broken. layerN is for mozilla, offsetN for IE, seems to work @@ -180,25 +246,25 @@ this.zoom_center_to(x, y, delta); }, - // substrate scroll was started + /** Substrate scroll was started */ on_scroll_start: function (ev) { }, - // substrate was scrolled, update scroll_{x,y}, and then update tiles after 100ms + /** Substrate was scrolled, update scroll_{x,y}, and then update tiles after 100ms */ on_scroll_move: function (ev) { this.update_scroll(); - this.update_after_timeout(); + + // fast-update at 100ms intervals + this.update_after_timeout(true); }, - // substrate scroll was ended, update tiles now + /** Substrate scroll was ended, update tiles now */ on_scroll_end: function (ev) { this.update_now(); }, - /* get state */ - - // return the absolute (x, y) coords of the given event inside the viewport + /** Calculate the absolute (x, y) coords of the given event inside the viewport */ event_offset: function (ev) { var offset = this.div.cumulativeOffset(); @@ -208,9 +274,11 @@ }; }, - /* modify state */ +/* + * Change view - scroll/zoom + */ - // scroll the view to place the given absolute (x, y) co-ordinate at the top left + /** Scroll the view to place the given absolute (x, y) co-ordinate at the top left */ scroll_to: function (x, y) { // update it via the style this.substrate.style.top = "-" + y + "px"; @@ -221,7 +289,7 @@ this.scroll_y = y; }, - // scroll the view to place the given absolute (x, y) co-ordinate at the center + /** Scroll the view to place the given absolute (x, y) co-ordinate at the center */ scroll_center_to: function (x, y) { return this.scroll_to( x - this.center_offset_x, @@ -229,40 +297,40 @@ ); }, - // zoom à la delta such that the given (zoomed) absolute (x, y) co-ordinates will be at the top left + /** Zoom à la delta such that the given (zoomed) absolute (x, y) co-ordinates will be at the top left */ zoom_scaled: function (x, y, delta) { if (!this.update_zoom(delta)) + // couldn't zoom, scaled coords are wrong return false; // scroll to the new position this.scroll_to(x, y); - // update view - // XXX: ... + // update view after 100ms - in case we zoom again? this.update_after_timeout(); return true; }, - // zoom à la delta such that the given (current) absolute (x, y) co-ordinates will be at the top left + /** Zoom à la delta such that the given (current) absolute (x, y) co-ordinates will be at the top left */ zoom_to: function (x, y, delta) { return this.zoom_scaled( - scaleByZoomDelta(x, delta), - scaleByZoomDelta(y, delta), + scaleByZoomDelta(x, -delta), + scaleByZoomDelta(y, -delta), delta ); }, - // zoom à la delta such that the given (current) absolute (x, y) co-ordinates will be at the center + /** Zoom à la delta such that the given (current) absolute (x, y) co-ordinates will be at the center */ zoom_center_to: function (x, y, delta) { return this.zoom_scaled( - scaleByZoomDelta(x, delta) - this.center_offset_x, - scaleByZoomDelta(y, delta) - this.center_offset_y, + scaleByZoomDelta(x, -delta) - this.center_offset_x, + scaleByZoomDelta(y, -delta) - this.center_offset_y, delta ); }, - // zoom à la delta, keeping the view centered + /** Zoom à la delta, keeping the view centered */ zoom_centered: function (delta) { return this.zoom_center_to( this.scroll_x + this.center_offset_x, @@ -271,105 +339,144 @@ ); }, - // zoom in one level, keeping the view centered + /** Zoom in one level, keeping the view centered */ zoom_in: function () { - return this.zoom_centered(+1); + return this.zoom_centered(-1); }, - // zoom out one leve, keeping the view centered + /** Zoom out one level, keeping the view centered */ zoom_out: function () { - return this.zoom_centered(-1); + return this.zoom_centered(+1); }, - /* update view/state to reflect reality */ + /** Center the view on the given coords, and zoom in, if possible */ + center_and_zoom_in: function (cx, cy) { + // try and zoom in + if (this.update_zoom(-1)) { + // scaled coords + cx = scaleByZoomDelta(cx, 1); + cy = scaleByZoomDelta(cy, 1); + } - // figure out the viewport dimensions + // re-center + this.scroll_center_to(cx, cy); + + // update view after 100ms - in case we zoom again? + this.update_after_timeout(); + }, + +/* + * Update view state + */ + + /** Update the view_* / center_offset_* vars, and any dependent items */ update_size: function () { this.view_width = this.div.getWidth(); this.view_height = this.div.getHeight(); this.center_offset_x = Math.floor(this.view_width / 2); this.center_offset_y = Math.floor(this.view_height / 2); - + + // the link-to-image uses the current view size this.update_image_link(); }, - - // figure out the scroll offset as absolute pixel co-ordinates at the top left + + /** + * Update the scroll_x/y state + */ update_scroll: function() { this.scroll_x = -parseInt(this.substrate.style.left); this.scroll_y = -parseInt(this.substrate.style.top); }, - - // wiggle the ZoomLevels around to match the current zoom level - update_zoom: function(delta) { - if (!this.zoom_layer) { - // first zoom operation - - // is the new zoom level valid? - if (!this.zoom_layers[delta]) - return false; - - // set the zoom layyer - this.zoom_layer = this.zoom_layers[delta]; - - // enable it - this.zoom_layer.enable(11); - - // no need to .update_tiles or anything like that - - } else { - // is the new zoom level valid? - if (!this.zoom_layers[this.zoom_layer.level + delta]) - return false; + + /** + * Switch zoom layers + */ + update_zoom: function (delta) { + // is the new zoom level valid? + if (!this.zoom_layers[this.zoom_layer.level + delta]) + return false; - var zoom_old = this.zoom_layer; - var zoom_new = this.zoom_layers[this.zoom_layer.level + delta]; - - // XXX: ugly hack - if (this.zoom_timer) { - clearTimeout(this.zoom_timer); - this.zoom_timer = null; - } - - // get other zoom layers out of the way - this.zoom_layers.each(function (zl) { - zl.disable(); - }); - - // update the zoom layer - this.zoom_layer = zoom_new; - - // apply new z-indexes, preferr the current one over the new one - zoom_new.enable(11); - zoom_old.enable(10); - - // resize the tiles in the two zoom layers - zoom_new.update_tiles(zoom_new.level); - zoom_old.update_tiles(zoom_old.level); - - // XXX: ugly hack - this.zoom_timer = setTimeout(function () { zoom_old.disable()}, 1000); + var zoom_old = this.zoom_layer; + var zoom_new = this.zoom_layers[this.zoom_layer.level + delta]; + + // XXX: clear hide-zoom-after-timeout + if (this.zoom_timer) { + clearTimeout(this.zoom_timer); + this.zoom_timer = null; } + + // get other zoom layers out of the way + // XXX: u + this.zoom_layers.each(function (zl) { + zl.disable(); + }); + + // update the current zoom layer + this.zoom_layer = zoom_new; + + // layer them such that the old on remains visible underneath the new one + zoom_new.enable(11); + zoom_old.enable(10); + + // resize the tiles in the two zoom layers + zoom_new.update_tiles(zoom_new.level); + zoom_old.update_tiles(zoom_new.level); + + // disable the old zoom layer after 1000ms - after the new zoom layer has loaded - not an optimal solution + this.zoom_timer = setTimeout(function () { zoom_old.disable()}, 1000); - // update UI + // update UI state this.update_zoom_ui(); return true; }, - // keep the zoom buttons, if any, updated - update_zoom_ui: function () { - if (this.btn_zoom_in) - (this.zoom_layer.level >= this.source.zoom_max) ? this.btn_zoom_in.disable() : this.btn_zoom_in.enable(); + /** Run update_tiles() at 500ms intervals */ + update_after_timeout: function (fast) { + // have not called update_tiles() yet + this._update_idle = false; - if (this.btn_zoom_out) - (this.zoom_layer.level <= this.source.zoom_min) ? this.btn_zoom_out.disable() : this.btn_zoom_out.enable(); - - this.update_image_link(); + // cancel old timeout + if (!fast && this._update_timeout) { + clearTimeout(this._update_timeout); + this._update_timeout = null; + } + + if (!this._update_timeout) + // trigger after delay + this._update_timeout = setTimeout(this._update_timeout_trigger.bind(this), fast ? 500 : 100); }, - // ensure that all tiles that are currently visible are loaded - update_tiles: function() { + _update_timeout_trigger: function () { + this._update_timeout = null; + + // have called update_tiles() + this._update_idle = true; + + this.update_tiles(); + }, + + /** + * Unschedule the call to update_tiles() and call it now, unless it's already been triggered by the previous call to + * update_after_timeout + */ + update_now: function () { + // abort trigger if active + if (this._update_timeout) { + clearTimeout(this._update_timeout); + this._update_timeout = null; + } + + // update now unless already done + if (!this._update_idle) + this.update_tiles(); + }, + + /** + * Determine the set of visible tiles, and ensure they are loaded + */ + update_tiles: function () { // short names for some vars... var x = this.scroll_x; var y = this.scroll_y; @@ -379,40 +486,41 @@ var th = this.source.tile_height; var zl = this.zoom_layer.level; - // figure out what set of columns are visible + // figure out which set of cols/rows are visible var start_col = Math.max(0, Math.floor(x / tw)); var start_row = Math.max(0, Math.floor(y / th)); var end_col = Math.floor((x + sw) / tw); var end_row = Math.floor((y + sh) / th); - // loop through all those tiles + // loop through all visible tiles for (var col = start_col; col <= end_col; col++) { for (var row = start_row; row <= end_row; row++) { // the tile's id - var id = "tile_" + this.zoom_layer.level + "_" + col + "_" + row; + var id = "tile_" + zl + "_" + col + "_" + row; // does the element exist already? + // XXX: use basic document.getElementById for perf? var t = $(id); if (!t) { // build a new tile t = Builder.node("img", { - src: this.source.build_url(col, row, zl, sw, sh), - id: id, - title: "(" + col + ", " + row + ")", + src: this.source.build_url(col, row, zl /* , sw, sh */), + id: id //, + // title: "(" + col + ", " + row + ")", // style set later } ); - // set the CSS style stuff + // position t.style.top = th * row; t.style.left = tw * col; t.style.display = "none"; - // wait for it to load + // display once loaded Event.observe(t, "load", _tile_loaded.bindAsEventListener(t)); - // store the col/row + // remember the col/row t.__col = col; t.__row = row; @@ -429,17 +537,41 @@ this.update_scroll_ui(); }, - - // update scroll-dependant UI elements + +/* + * UI state + */ + + /** + * Update any zl-dependant UI elements + */ + update_zoom_ui: function () { + // deactivate zoom-in button if zoomed in + if (this.btn_zoom_in) + (this.zoom_layer.level <= this.source.zoom_min) ? this.btn_zoom_in.disable() : this.btn_zoom_in.enable(); + + // deactivate zoom-out button if zoomed out + if (this.btn_zoom_out) + (this.zoom_layer.level >= this.source.zoom_max) ? this.btn_zoom_out.disable() : this.btn_zoom_out.enable(); + + // link-to-image + this.update_image_link(); + }, + + /** + * Update any position-dependant UI elements + */ update_scroll_ui: function () { + // update the link-to-this-page thing + document.location.hash = (this.scroll_x + this.center_offset_x) + ":" + (this.scroll_y + this.center_offset_y) + ":" + this.zoom_layer.level; + // update link-to-image this.update_image_link(); - - // update the link-to-this-page thing - document.location.hash = "#" + (this.scroll_x + this.center_offset_x) + ":" + (this.scroll_y + this.center_offset_y) + ":" + this.zoom_layer.level; }, - // update image link with size, zoom, pos + /** + * Update the link-to-image-of-this-view link with dimensions, zoom, position + */ update_image_link: function () { if (!this.image_link) return; @@ -452,41 +584,17 @@ ); this.image_link.innerHTML = this.view_width + "x" + this.view_height + "@" + this.zoom_layer.level; - }, - - // do update_tiles after 100ms, unless we are called again - update_after_timeout: function () { - this._update_idle = false; - - if (this._update_timeout) - clearTimeout(this._update_timeout); - - this._update_timeout = setTimeout(this._update_timeout_trigger.bind(this), 100); - }, - - _update_timeout_trigger: function () { - this._update_idle = true; - - this.update_tiles(); - }, - - // call update_tiles if it hasn't been called due to update_after_timeout - update_now: function () { - if (this._update_timeout) - clearTimeout(this._update_timeout); - - if (!this._update_idle) - this.update_tiles(); - }, - + } }); -// used by Viewport.update_tiles to make a tile visible after it has loaded +/** Used by Viewport.update_tiles to make a tile visible after it has loaded */ function _tile_loaded (ev) { this.style.display = "block"; } -// a zoom layer containing the tiles for one zoom level +/** + * A zoom layer contains a (col, row) grid of tiles at a specific zoom level. + */ var ZoomLayer = Class.create({ initialize: function (source, zoom_level) { this.source = source; @@ -497,28 +605,36 @@ this.tiles = []; }, - // add a tile to this zoom layer + /** Add a tile to this zoom layer's grid */ add_tile: function (tile) { this.div.appendChild(tile); this.tiles.push(tile); }, - // make this zoom layer visible with the given z-index + /** Make this zoom layer visible with the given z-index */ enable: function (z_index) { this.div.style.zIndex = z_index; - this.div.show(); + + // XXX: IE8 needs this for some reason + $(this.div).show(); }, - // hide this zoom layer - disable: function (z_index) { - this.div.hide(); + /** Hide this zoom layer */ + disable: function () { + // XXX: IE8 + $(this.div).hide(); }, - // update the tiles in this zoom layer so that they are in the correct position and of the correct size when - // viewed with the given zoom level + /** + * Update the tile grid in this zoom layer such the tiles are in the correct position and of the correct size + * when viewed with the given zoom level. + * + * For zoom levels different than this layer's level, this will resize the tiles! + */ update_tiles: function (zoom_level) { - var zd = zoom_level - this.level; - + var zd = this.level - zoom_level; + + // desired tile size var tw = scaleByZoomDelta(this.source.tile_width, zd); var th = scaleByZoomDelta(this.source.tile_height, zd); @@ -526,23 +642,22 @@ var tiles_len = tiles.length; var t, ts; + // XXX: *all* tiles? :/ for (var i = 0; i < tiles_len; i++) { t = tiles[i]; ts = t.style; - + + // reposition ts.width = tw; ts.height = th; - ts.top = th*t.__row; - ts.left = tw*t.__col; + ts.top = th * t.__row; + ts.left = tw * t.__col; } - }, - - - + } }); -// scale the given co-ordinate by a zoom delta. If we zoom in (dz > 0), n will become larger, and if we zoom -// out (dz < 0), n will become smaller. +// scale the given co-ordinate by a zoom delta. If we zoom out (dz > 0), n will become larger, and if we zoom +// in (dz < 0), n will become smaller. function scaleByZoomDelta (n, dz) { if (dz > 0) return n << dz;