--- a/Makefile	Fri Jun 06 23:37:45 2008 +0300
+++ b/Makefile	Sat Jun 07 05:05:18 2008 +0300
@@ -3,7 +3,7 @@
 
 EXECS = render_file web_main render_node
 
-all: render_file render_node render_web
+all: web_main file_main node_main
 
 common.o: common.c common.h
 http.o: http.c http.h
@@ -12,16 +12,18 @@
 render.o: render.c render.h
 render_remote.o: render_remote.c render_remote.h
 render_png.o: render_png.c render_png.h
+render_raw.o: render_raw.c render_raw.h
 render_local.o: render_local.c render_local.h
+render_multi.o: render_multi.c render_multi.h
 render_mandelbrot.o: render_mandelbrot.c render_mandelbrot.h
 
 file_main.o: file_main.c
 node_main.o: node_main.c
 web_main.o: web_main.c
 
-render_file: file_main.o common.o render.o render_png.o render_local.o render_mandelbrot.o
-render_node: node_main.o common.o render.o render_png.o render_local.o render_mandelbrot.o
-render_web: web_main.o common.o http.o render.o remote_node.o remote_pool.o render_remote.o
+file_main: file_main.o common.o render.o render_raw.o render_png.o render_local.o render_mandelbrot.o
+node_main: node_main.o common.o render.o render_raw.o render_png.o render_local.o render_mandelbrot.o
+web_main: web_main.o common.o http.o render.o render_png.o remote_node.o remote_pool.o render_remote.o render_multi.o
 
 clean :
 	rm *.o ${EXECS}

--- a/remote_node.h	Fri Jun 06 23:37:45 2008 +0300
+++ b/remote_node.h	Sat Jun 07 05:05:18 2008 +0300
@@ -24,7 +24,7 @@
      * Dynamic information
      */
 
-    // how many render requests this node is currently processing
+    // an estimate of how many render requests this node is currently processing
     int current_load;
 };
 

--- a/remote_pool.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/remote_pool.c	Sat Jun 07 05:05:18 2008 +0300
@@ -101,6 +101,11 @@
             node_info = &pool_info->nodes[i];
         }
     }
+    
+    if (node_info) {
+        // add one to its load, because that's probably correct, and works if we pick multiple nodes from the pool at the same time
+        node_info->current_load++;
+    }
 
     // either NULL or the right remote_info
     return node_info;

--- a/render.h	Fri Jun 06 23:37:45 2008 +0300
+++ b/render.h	Sat Jun 07 05:05:18 2008 +0300
@@ -33,6 +33,11 @@
 struct render *render_alloc ();
 
 /*
+ * Clear out the value of the given render context
+ */
+int render_init (struct render *ctx, int mode);
+
+/*
  * What kind of image to render, PNG or RAW?
  */
 int render_set_mode (struct render *ctx, int mode);

--- a/render_local.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_local.c	Sat Jun 07 05:05:18 2008 +0300
@@ -6,15 +6,14 @@
 #include "render_internal.h"
 #include "render_local.h"
 #include "render_png.h"
+#include "render_raw.h"
 #include "render_mandelbrot.h"
 
 int render_local (struct render *render, double *duration) {
-    assert(render->mode == RENDER_PNG);
-
     unsigned char *rowbuf = NULL;
     struct render_png *png_ctx = NULL;
+    struct render_raw *raw_ctx = NULL;
     clock_t t1, t2;
-
     
     if (duration)
         *duration = -1;
@@ -23,14 +22,34 @@
     if (!(rowbuf = malloc(render->img_w)))
         ERROR("malloc");
  
-    // the render_png stuff
-    if (!(png_ctx = render_png_init(render)))
-        ERROR("render_png_init");
-    
-    //  set render_* to use it
-    if (render_local_mem(render, &rowbuf, (int(*)(void *arg, unsigned char *)) &render_png_row, png_ctx))
-        ERROR("render_local_mem");
-   
+    // what mode?
+    switch (render->mode) {
+        case RENDER_PNG :
+            // the render_png stuff
+            if (!(png_ctx = render_png_init(render)))
+                ERROR("render_png_init");
+                
+            //  set render_* to use the render_png
+            if (render_local_mem(render, &rowbuf, (int(*)(void *arg, unsigned char *)) &render_png_row, png_ctx))
+                ERROR("render_local_mem");
+
+            break;
+
+        case RENDER_RAW :
+            // the render_raw stuff
+            if (!(raw_ctx = render_raw_init(render)))
+                ERROR("render_raw_init");
+
+            //  set render_* to use the render_raw
+            if (render_local_mem(render, &rowbuf, (int(*)(void *arg, unsigned char *)) &render_raw_row, raw_ctx))
+                ERROR("render_local_mem");
+
+            break;
+
+        default :
+            assert(0);
+    }
+  
     // then we can actually render
     t1 = clock();
 
@@ -48,7 +67,12 @@
 
 error:
     free(rowbuf);
-    free(png_ctx);
+
+    if (png_ctx)
+        render_png_abort(png_ctx);
+
+    if (raw_ctx)
+        render_raw_abort(raw_ctx);
 
     return -1;
 }

--- a/render_local.h	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_local.h	Sat Jun 07 05:05:18 2008 +0300
@@ -5,6 +5,10 @@
 
 /*
  * Renders the given struct render locally in one operation.
+ *
+ * How this operates depends on the render mode:
+ *  RENDER_PNG      - see render_png_init
+ *  RENDER_RAW      - see render_raw_init
  */
 int render_local (struct render *render, double *duration);
 

--- a/render_multi.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_multi.c	Sat Jun 07 05:05:18 2008 +0300
@@ -1,12 +1,16 @@
-
+#include <stdlib.h>
+#include <assert.h>
 
+#include "common.h"
+#include "render_internal.h"
 #include "render_multi.h"
+#include "render_remote.h"
 #include "remote_node.h"
-
+#include "render_png.h"
 
 struct render_multi {
     // these are used as arguments to render_remote
-    struct render_sub_ctx {
+    struct render_multi_sub {
         // our offset in the list
         int index;
 
@@ -16,13 +20,40 @@
         // a pointer to ourself
         struct render_multi *self;
 
+        // if the render connection is marked as done without us having read all the data, move it from render_remote's buffers into here
+        struct evbuffer *in_buf;
+
         // _render_sent called for this?
         int render_sent;
+
+        // our offset into the row, static
+        size_t row_offset;
+
+        // how wide our slice is, static
+        size_t slice_width;
+
+        // how many bytes we have already written into the current row
+        size_t col;
+
+        // is this render op done?
+        int render_done;
     } remote_renders[RENDER_MULTI_NODES_MAX];
     
     // how many remote_renders we have
     int remote_render_count;
 
+    // have we called cb_sent?
+    int have_sent;
+
+    // the png thing
+    struct render_png *png_info;
+
+    // our pixel data row
+    unsigned char *rowbuf;
+
+    // buffer render_png output in this
+    struct evbuffer *out_buf;
+
     // our own callbacks that we call
     void (*cb_sent)(void *arg);
     void (*cb_data)(struct evbuffer *buf, void *arg);
@@ -30,12 +61,149 @@
     void (*cb_fail)(void *arg);
 
     void *cb_arg;
-
 };
 
-// our render_remote callbacks
-void _render_sent (void *arg) {
-    struct render_sub_ctx *ctx, *ctx2 = arg;
+#define FAIL_PARTIAL 0x01
+#define FAIL_SILENT 0x02
+
+// prototypes
+static void _render_multi_do_free (struct render_multi *ctx);
+static void _render_multi_do_sent (struct render_multi *ctx);
+static void _render_multi_do_done (struct render_multi *ctx);
+static void _render_multi_do_fail (struct render_multi *ctx, int flags);
+
+/*
+ * Actually free the request. Should be de-initialized (either _render_multi_error, or _render_done when this is called
+ */
+static void _render_multi_do_free (struct render_multi *ctx) {
+    if (!ctx)
+        return;
+    
+    int i;
+    for (i = 0; i < ctx->remote_render_count; i++) {
+        if (ctx->remote_renders[i].in_buf) {
+            evbuffer_free(ctx->remote_renders[i].in_buf);
+            ctx->remote_renders[i].in_buf = NULL;
+        }
+    }
+
+    if (ctx->rowbuf) {
+        free(ctx->rowbuf);
+        ctx->rowbuf = NULL;
+    }
+
+    if (ctx->out_buf) {
+        evbuffer_free(ctx->out_buf);
+        ctx->out_buf = NULL;
+    }
+
+    free(ctx);
+}
+
+// the request has been sent
+static void _render_multi_do_sent (struct render_multi *ctx) {
+    int i;
+
+    // check that all the remote_renders have indeed been sent
+    for (i = 0; i < ctx->remote_render_count; i++) {
+        assert(ctx->remote_renders[i].render_sent);
+    }
+
+    // assert the callbacks are still valid
+    assert(ctx->cb_sent && ctx->cb_fail && ctx->cb_done);
+
+    // call cb_sent and then invalidate it
+    ctx->cb_sent(ctx->cb_arg);
+    ctx->cb_sent = NULL;
+    ctx->have_sent = 1;
+    
+    // we're going to always have the PNG header data buffered at this point, so give that to the user right away
+    assert(evbuffer_get_length(ctx->out_buf) > 0);
+
+    ctx->cb_data(ctx->out_buf, ctx->cb_arg);
+
+    // ok
+    return;
+}
+
+// the request compelted normally
+static void _render_multi_do_done (struct render_multi *ctx) {
+    int i;
+
+    // check that all the remote_renders are indeed complete
+    for (i = 0; i < ctx->remote_render_count; i++) {
+        assert(ctx->remote_renders[i].remote_render == NULL);
+        assert(ctx->remote_renders[i].col == 0);
+    }
+
+    // finish the png_info
+    if (render_png_done(ctx->png_info)) {
+        // don't free it twice, though...
+        ctx->png_info = NULL;
+
+        ERROR("render_png_done");
+    }
+
+    ctx->png_info = NULL;
+
+    // check that both callbacks are still valid
+    assert(ctx->cb_fail && ctx->cb_done);
+
+    // call cb_done and then invalidate it
+    ctx->cb_done(ctx->cb_arg);
+    ctx->cb_fail = NULL;
+
+    // free ourself
+    _render_multi_do_free(ctx);
+    
+    // ok
+    return;
+
+error:    
+    /* O_o */
+    _render_multi_do_fail(ctx, FAIL_PARTIAL);
+}
+
+// the request completed abnormally. Flags:
+//  FAIL_SILENT     - don't call cb_fail
+//  FAIL_PARTIAL    - assume png_info may be NULL
+static void _render_multi_do_fail (struct render_multi *ctx, int flags) {
+    int i;
+
+    // cancel any in-progress remote renders
+    for (i = 0; i < ctx->remote_render_count; i++)
+        if (ctx->remote_renders[i].remote_render) {
+            render_remote_cancel(ctx->remote_renders[i].remote_render);
+            ctx->remote_renders[i].remote_render = NULL;
+        }
+    
+    if (!(flags & FAIL_PARTIAL) || ctx->png_info) {
+        // abort the render_png
+        render_png_abort(ctx->png_info);
+        ctx->png_info = NULL;
+    }
+    
+    // check that both callbacks are still valid
+    assert(ctx->cb_fail && ctx->cb_done);
+    
+    if (!(flags & FAIL_SILENT)) {
+        // call cb_fail and then invalidate it
+        ctx->cb_fail(ctx->cb_arg);
+    }
+
+    ctx->cb_fail = NULL;
+
+    // free ourselves
+    _render_multi_do_free(ctx);
+}
+
+/*
+ * One of the remote render commands has succesfully been sent.
+ *
+ * Once all of these commands have been sent, invoke our cb_sent.
+ */
+static void _render_multi_sent (void *arg) {
+    struct render_multi_sub *ctx = arg;
 
     ctx->render_sent = 1;
 
@@ -46,22 +214,169 @@
             break;
     }
     
+    // did we loop through all of them?
     if (i == ctx->self->remote_render_count) {
-        // call cb_sent
-        ctx->self->cb_sent(ctx->self->cb_arg);
+        // tell our user
+        _render_multi_do_sent(ctx->self);
     }
 }
 
-void _render_data (struct evbuffer *buf, void *arg) {
+/*
+ * We have received data from a single render node.
+ *
+ * Move the data into the row buffer if there's space in it, otherwise
+ * leave it inside the buffer.
+ *
+ * If this fills up our slice, check and see if the entire row is now full. If
+ * it is, pass the row to render_png_row, clear out the col values, and call
+ * render_remote_shake for each remote render
+ */
+static void _render_multi_data (struct evbuffer *buf, void *arg) {
+    struct render_multi_sub *ctx = arg;
 
+    assert(ctx->col <= ctx->slice_width);
+
+    // are we being called from _render_multi_done?
+    if (ctx->render_done && ctx->in_buf == NULL) {
+        // move all the data into our own buffer
+        if (!(ctx->in_buf = evbuffer_new()))
+            ERROR("evbuffer_new");
+
+        if (evbuffer_add_buffer(ctx->in_buf, buf))
+            ERROR("evbuffer_add_buffer");
+        
+        // don't do anything else
+        return;
+    }
+
+    // is our slice full?
+    if (ctx->col == ctx->slice_width) {
+        // don't care for new data
+        return;
+    }
+    
+    // read new data into our slice
+//    printf("rowbuf + %4d : %d");
+    ctx->col += evbuffer_remove(buf, ctx->self->rowbuf + ctx->row_offset + ctx->col, ctx->slice_width - ctx->col);
+
+    // is our slice full now?
+    if (ctx->col == ctx->slice_width) {
+        // is the row complete now?
+        int i;
+        for (i = 0; i < ctx->self->remote_render_count; i++) {
+            if (ctx->self->remote_renders[i].col < ctx->self->remote_renders[i].slice_width)
+                break;
+        }
+        
+        // are they all full?
+        if (i == ctx->self->remote_render_count) {
+            // pass the data to render_png, this results in calls to _render_multi_png_data
+            if (render_png_row(ctx->self->png_info, ctx->self->rowbuf))
+                ERROR("render_png_row");
+
+            // clear the col values
+            for (i = 0; i < ctx->self->remote_render_count; i++) {
+                ctx->self->remote_renders[i].col = 0;
+            }
+            
+            // shake the buffers
+            // XXX: this will result in recursion
+            for (i = 0; i < ctx->self->remote_render_count; i++) {
+                if (ctx->self->remote_renders[i].remote_render) {
+                    if (render_remote_shake(ctx->self->remote_renders[i].remote_render))
+                        ERROR("render_remote_shake");
+                } else {
+                    // already disconnected, use in_buf instead
+                    assert(ctx->self->remote_renders[i].in_buf);
+
+                    // call it directly...
+                    _render_multi_data(ctx->self->remote_renders[i].in_buf, &ctx->self->remote_renders[i]);
+                }
+            }
+        }
+    }
+
+    return;
+
+error:
+    _render_multi_do_fail(ctx->self, 0);
 }
 
-void _render_done (void *arg) {
+/*
+ * We fed a row of pixels into render_png, and this PNG data came out.
+ *
+ * We need to pass it back to our caller
+ */
+static int _render_multi_png_data (const unsigned char *data, size_t length, void *arg) {
+    struct render_multi *ctx = arg;
 
+    // XXX: avoid these many data copies?
+    if (evbuffer_add(ctx->out_buf, data, length))
+        ERROR("evbuffer_add");
+    
+    // only call the data cb if we've already called cb_sent
+    if (ctx->have_sent)
+        ctx->cb_data(ctx->out_buf, ctx->cb_arg);
+    
+    // ok
+    return 0;
+
+error:
+    // don't do any failing here, this will return control to a _render_* function that will handle it
+    return -1;
 }
 
-void _render_fail (void *arg) {
+/*
+ * The render node has rendered everything that it needs to render. Our slice
+ * of the last row should be full (or empty) now. If all the remote_renders are
+ * done, we can call render_png_done and then cb_done.
+ */
+static void _render_multi_done (void *arg) {
+    struct render_multi_sub *ctx = arg;
+ 
+    // mark it as done
+    ctx->render_done = 1;
 
+    // shake out the rest of the data as needed, as render_multi won't keep it anymore
+    render_remote_shake(ctx->remote_render);
+   
+    // invalidate this ctx's remote render
+    ctx->remote_render = NULL;
+
+    // is the data incomplete?
+    if (!(ctx->col == ctx->slice_width || ctx->col == 0))
+        ERROR("incomplete data for slice %d: %d/%d bytes", ctx->index, ctx->col, ctx->slice_width);
+
+    // are all of them done?
+    int i;
+    for (i = 0; i < ctx->self->remote_render_count; i++) {
+        if (!ctx->self->remote_renders[i].render_done)
+            break;
+    }
+    
+    // are they all done?
+    if (i == ctx->self->remote_render_count) {
+        // finish it off
+        _render_multi_do_done(ctx->self);
+    }
+
+    // ok, wait for the rest to complete
+    return;
+    
+error:
+    _render_multi_do_fail(ctx->self, 0);
+}
+
+/*
+ * One render node failed, abort the whole thing
+ */
+static void _render_multi_fail (void *arg) {
+    struct render_multi_sub *ctx = arg;
+
+    // invalidate this ctx's remote render
+    ctx->remote_render = NULL;
+
+    _render_multi_do_fail(ctx->self, 0);
 }
 
 #define ROUND_DOWN(dividend, divisor) ((dividend) / (divisor))
@@ -70,8 +385,8 @@
 #define HALF(a, b) (( a + b) / 2)
 
 struct render_multi *render_multi (
-        render_t *render_ctx,               // what to render
-        struct remote_pool *render_pool,    // what render pool to use
+        struct render *render,               // what to render
+        struct remote_pool *pool_info,    // what render pool to use
         void (*cb_sent)(void *arg),
         void (*cb_data)(struct evbuffer *buf, void *arg),
         void (*cb_done)(void *arg),
@@ -79,27 +394,40 @@
         void *cb_arg
 ) {
     struct render_multi *ctx = NULL;
-    render_t r_left, r_right;
-
-    // for now, just split it in half into two render_ts
-    assert(RENDER_MULTI_NODES_MAX >= 2);
-
-    if (
-            render_init(&r_left, RENDER_RAW)
-         || render_init(&r_right, RENDER_RAW)
-         || render_set_size(&r_left, render_ctx->img_w / 2, render_ctx->img_h)
-         || render_set_size(&r_right, render_ctx->img_w / 2 +  render_ctx->img_w % 2, render_ctx->img_h)
-         || render_set_region_raw(&r_left, render_ctx->x1, render_ctx->y1, HALF(render_ctx->x1, render_ctx->x2), render_ctx->y2)
-         || render_set_region_raw(&r_right, HALF(render_ctx->x1, render_ctx->x2), render_ctx->y1, render_ctx->x2, render_ctx->y2)
-    )
-        ERROR("render_{init,set_size,set_region_raw}");
+    struct render r_left, r_right;
     
     // alloc the render_multi
     ctx = calloc(1, sizeof(struct render_multi));
 
     if (!ctx)
         ERROR("calloc");
-        
+
+    // init the remote_render
+    // for now, just split it in half into two render_ts
+    ctx->remote_renders[0].index = 0;
+    ctx->remote_renders[0].self = ctx;
+    ctx->remote_renders[0].slice_width = render->img_w / 2;
+    ctx->remote_renders[0].row_offset = 0;
+
+    ctx->remote_renders[1].index = 1;
+    ctx->remote_renders[1].self = ctx;
+    ctx->remote_renders[1].slice_width = render->img_w / 2 + render->img_w % 2;
+    ctx->remote_renders[1].row_offset = render->img_w / 2;
+
+    ctx->remote_render_count = 2;
+
+    assert(RENDER_MULTI_NODES_MAX >= 2);
+
+    if (
+            render_init(&r_left, RENDER_RAW)
+         || render_init(&r_right, RENDER_RAW)
+         || render_set_size(&r_left, ctx->remote_renders[0].slice_width, render->img_h)
+         || render_set_size(&r_right, ctx->remote_renders[1].slice_width, render->img_h)
+         || render_region_raw(&r_left, render->x1, render->y1, HALF(render->x1, render->x2), render->y2)
+         || render_region_raw(&r_right, HALF(render->x1, render->x2), render->y1, render->x2, render->y2)
+    )
+        ERROR("render_{init,set_size,set_region_raw}");
+            
     // store the provided callback functions
     ctx->cb_sent = cb_sent;
     ctx->cb_data = cb_data;
@@ -107,6 +435,22 @@
     ctx->cb_fail = cb_fail;
     ctx->cb_arg = cb_arg;
 
+    // our rowbuf
+    if (!(ctx->rowbuf = malloc(render->img_w)))
+        ERROR("malloc");
+
+    // store our render_png callbacks, must be before png_info
+    if (render_io_custom(render, &_render_multi_png_data, NULL, ctx))
+        ERROR("render_io_custom");
+
+    // evbuffer, must be before png_info
+    if (!(ctx->out_buf = evbuffer_new()))
+        ERROR("evbuffer_new");
+
+    // png info
+    if (!(ctx->png_info = render_png_init(render)))
+        ERROR("render_png_init");
+
     // pull two nodes from the pool
     struct remote_node *node_left, *node_right;
 
@@ -116,28 +460,35 @@
     )
         ERROR("remote_pool_get");
     
-    // init the remote_render
-    ctx->remote_renders[0].index = 0;
-    ctx->remote_renders[0].self = &ctx
-    ctx->remote_renders[1].index = 1;
-    ctx->remote_renders[1].self = &ctx
-    ctx->remote_render_count = 2;
-
     // the two render_remote calls
     if (
             !(ctx->remote_renders[0].remote_render = render_remote(&r_left, node_left, 
-                &_render_sent, &_render_data, &_render_done, &_render_fail, &ctx->remote_renders[0]))
+                &_render_multi_sent, &_render_multi_data, &_render_multi_done, &_render_multi_fail, &ctx->remote_renders[0]))
          || !(ctx->remote_renders[1].remote_render = render_remote(&r_right, node_right,
-                &_render_sent, &_render_data, &_render_done, &_render_fail, &ctx->remote_renders[1]))
+                &_render_multi_sent, &_render_multi_data, &_render_multi_done, &_render_multi_fail, &ctx->remote_renders[1]))
     )
         ERROR("render_remote");
     
     // I guess that's a succesfull start now
-    return 0;
+    return ctx;
 
 error:
-    free(ctx);
+    _render_multi_do_fail(ctx, FAIL_SILENT | FAIL_PARTIAL);
 
-    return -1;
+    return NULL;
 }
 
+void render_multi_cancel (struct render_multi *ctx) {
+    _render_multi_do_fail(ctx, FAIL_SILENT);
+}
+
+int render_multi_set_recv (struct render_multi *ctx, size_t recv_threshold, size_t unread_buffer) {
+    return 0;
+}
+
+int render_multi_shake (struct render_multi *ctx) {
+    // call the data cb
+    ctx->cb_data(ctx->out_buf, ctx->cb_arg);
+
+    return 0;
+}

--- a/render_multi.h	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_multi.h	Sat Jun 07 05:05:18 2008 +0300
@@ -1,6 +1,10 @@
 #ifndef RENDER_MULTI_H
 #define RENDER_MULTI_H
 
+#include <event2/util.h>
+#include <event2/buffer.h>
+
+#include "render.h"
 #include "remote_pool.h"
 
 /*
@@ -18,8 +22,8 @@
  * The behaviour of the callbacks is mostly the same as for render_remote.h
  */
 struct render_multi *render_multi (
-        render_t *render_ctx,               // what to render
-        struct remote_pool *render_pool,    // what render pool to use
+        struct render *render_ctx,               // what to render
+        struct remote_pool *pool_info,    // what render pool to use
         void (*cb_sent)(void *arg),
         void (*cb_data)(struct evbuffer *buf, void *arg),
         void (*cb_done)(void *arg),
@@ -27,4 +31,24 @@
         void *cb_arg
 );
 
+/*
+ * Cancel the given request. No more callbacks will be called, buffered data is
+ * discarded and the remote render processes will cancel ASAP.
+ */
+void render_multi_cancel (struct render_multi *ctx);
+
+/*
+ * Doesn't actually do anything yet
+ */
+int render_multi_set_recv (struct render_multi *ctx, size_t recv_threshold, size_t unread_buffer);
+
+/*
+ * Call cb_data with the current set of buffered input data immediately,
+ * regardless of whether or not the buffer contains any data, or any new
+ * data has been received.
+ *
+ * Only call this after cb_sent and before cb_done/cb_fail.
+ */
+int render_multi_shake (struct render_multi *ctx);
+
 #endif /* RENDER_MULTI_H */

--- a/render_png.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_png.c	Sat Jun 07 05:05:18 2008 +0300
@@ -11,31 +11,33 @@
     png_structp png_ptr;
     png_infop info_ptr;
 
-    // our render op, containing the I/O callbacks
-    struct render *render;
+    // some info that we need to keep from the struct render
+    render_ctx_write_cb io_write_fn;
+    render_ctx_flush_cb io_flush_fn;
+    void *cb_arg;
 };
 
-void _render_png_write(png_structp png_ptr, png_bytep data, png_size_t length) {
+static void _render_png_write(png_structp png_ptr, png_bytep data, png_size_t length) {
     struct render_png *ctx = png_get_io_ptr(png_ptr);
 
-    if (ctx->render->io_write_fn)
-        if (ctx->render->io_write_fn(data, length, ctx->render->cb_arg)) {
+    if (ctx->io_write_fn)
+        if (ctx->io_write_fn(data, length, ctx->cb_arg)) {
             // error, doesn't return
             png_error(png_ptr, "_render_png_write: io_write_fn");
         }
 }
 
-void _render_png_flush(png_structp png_ptr) {
+static void _render_png_flush(png_structp png_ptr) {
     struct render_png *ctx = png_get_io_ptr(png_ptr);
     
-    if (ctx->render->io_flush_fn)
-        if (ctx->render->io_flush_fn(ctx->render->cb_arg)) {
+    if (ctx->io_flush_fn)
+        if (ctx->io_flush_fn(ctx->cb_arg)) {
             // error, doesn't return
             png_error(png_ptr, "_render_png_flush: io_flush_fn");
         }
 }
 
-void _render_png_free (struct render_png *ctx) {
+static void _render_png_free (struct render_png *ctx) {
     if (ctx)
         png_destroy_write_struct(&ctx->png_ptr, &ctx->info_ptr);
 
@@ -51,8 +53,10 @@
     if (!(ctx = calloc(1, sizeof(struct render_png))))
         ERROR("calloc");
 
-    // store the struct render
-    ctx->render = render;
+    // store some info from the struct render
+    ctx->io_write_fn = render->io_write_fn;
+    ctx->io_flush_fn = render->io_flush_fn;
+    ctx->cb_arg = render->cb_arg;
 
     // libpng initialization
     if (!(ctx->png_ptr = png_create_write_struct (PNG_LIBPNG_VER_STRING, NULL, NULL, NULL)))
@@ -111,6 +115,9 @@
 
     // write end
     png_write_end(ctx->png_ptr, ctx->info_ptr);
+    
+    // free everything
+    _render_png_free(ctx);
 
     // success
     return 0;
@@ -124,7 +131,8 @@
     // libpng error handling
     if (setjmp(png_jmpbuf(ctx->png_ptr)))
         ERROR("libpng");
-
+    
+    // just free it
     _render_png_free(ctx);
 
     // success

--- a/render_png.h	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_png.h	Sat Jun 07 05:05:18 2008 +0300
@@ -11,7 +11,11 @@
 /*
  * Build and return a render_png that can be used to render a PNG of the given struct render.
  *
- * This inspects the size and io properties of the struct render.
+ * The dimensions of the PNG image will be read from render. If render's I/O 
+ * was set using render_io_stream, then the PNG data will be written to that
+ * stream. If it was set using render_io_custom, then the write_fn will be
+ * called with chunks of PNG data, and flush_fn from time to time as a hint to
+ * flush the data, but this may be omitted.
  *
  * returns NULL on failure.
  */
@@ -20,7 +24,7 @@
 /*
  * Feed a full row of raw pixel data into the PNG image.
  *
- * buf must be struct render.img_w bytes wide.
+ * buf must be render.img_w bytes wide.
  */
 int render_png_row (struct render_png *ctx, unsigned char *rowbuf);
 

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/render_raw.c	Sat Jun 07 05:05:18 2008 +0300
@@ -0,0 +1,98 @@
+#include <stdlib.h>
+
+#include "common.h"
+#include "render_internal.h"
+#include "render_raw.h"
+
+struct render_raw {
+    // some info that we need to keep from the struct render
+    u_int32_t img_w;
+
+    // cb or stream?
+    FILE *io_stream;
+    render_ctx_write_cb io_write_fn;
+    render_ctx_flush_cb io_flush_fn;
+    void *cb_arg;
+};
+
+static void _render_raw_free (struct render_raw *ctx) {
+    free(ctx);
+}
+
+struct render_raw *render_raw_init (struct render *render) {
+    struct render_raw *ctx = NULL;
+
+    // calloc the render_png
+    if (!(ctx = calloc(1, sizeof(struct render_raw))))
+        ERROR("calloc");
+
+    // store some info from the struct render
+    ctx->img_w = render->img_w;
+    ctx->io_stream = render->io_stream;
+    ctx->io_write_fn = render->io_write_fn;
+    ctx->io_flush_fn = render->io_flush_fn;
+    ctx->cb_arg = render->cb_arg;
+
+    // success
+    return ctx;
+
+error:
+    _render_raw_free(ctx);
+    return NULL;
+}
+
+int render_raw_row (struct render_raw *ctx, unsigned char *rowbuf) {
+    // write it out
+    if (ctx->io_stream) {
+        if (fwrite(rowbuf, ctx->img_w, 1, ctx->io_stream) != 1)
+            PERROR("fwrite");
+
+    } else if (ctx->io_write_fn) {
+        if (ctx->io_write_fn(rowbuf, ctx->img_w, ctx->cb_arg))
+            ERROR("io_write_fn");
+
+    } else {
+        // ignore
+    }
+
+    // success
+    return 0;
+
+error:
+    // don't free it yet, raw_{done,abort} does that!
+    return -1;
+}
+
+int render_raw_done (struct render_raw *ctx) {
+    // flush
+    if (ctx->io_stream) {
+        if (fflush(ctx->io_stream))
+            PERROR("fflush");
+
+    } else if (ctx->io_flush_fn) {
+        if (ctx->io_flush_fn(ctx->cb_arg))
+            ERROR("io_flush_fn");
+
+    } else {
+        // ignore
+    }
+
+    // free everything
+    _render_raw_free(ctx);
+
+    // success
+    return 0;
+
+error:
+    _render_raw_free(ctx);
+    return -1;
+}
+
+int render_raw_abort (struct render_raw *ctx) {
+    // just free it
+    _render_raw_free(ctx);
+
+    // success
+    return 0;
+}
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/render_raw.h	Sat Jun 07 05:05:18 2008 +0300
@@ -0,0 +1,41 @@
+#ifndef RENDER_RAW_H
+#define RENDER_RAW_H
+
+#include "render.h"
+
+/*
+ * For rendering raw pixel data
+ */
+struct render_raw;
+
+/*
+ * Build and return a render_raw that can be used to handle the raw rendered pixel data.
+ *
+ * If render's I/O was set up with render_io_stream, the raw pixels will simply
+ * be fwritten to that stream (and fflushed at the end). If I/O was set up with
+ * render_io_custom, write_fn will be called with a row of pixel data (a buffer
+ * img_w bytes long) for every row that's written. Flush_fn will be called once
+ * all the pixels have been "written".
+ *
+ * returns NULL on failure.
+ */
+struct render_raw *render_raw_init (struct render *render);
+
+/*
+ * Feed a full row of raw pixel data to be handled
+ *
+ * buf must be render.img_w bytes wide.
+ */
+int render_raw_row (struct render_raw *ctx, unsigned char *rowbuf);
+
+/*
+ * Mark the render as complete and free the render_raw, flushing out any remaining data.
+ */
+int render_raw_done (struct render_raw *ctx);
+
+/*
+ * Abort the render, free the render_raw, and discard any remaining data.
+ */
+int render_raw_abort (struct render_raw *ctx);
+
+#endif /* RENDER_RAW_H */

--- a/render_remote.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/render_remote.c	Sat Jun 07 05:05:18 2008 +0300
@@ -3,6 +3,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <errno.h>
+#include <assert.h>
 
 #include <event2/event.h>
 #include <event2/bufferevent.h>
@@ -31,6 +32,9 @@
     } render_cmd;
 
     #pragma pack(pop)
+    
+    // has cb_done/cb_fail already been called?
+    int alive;
 
     void (*cb_sent)(void *arg);
     void (*cb_data)(struct evbuffer *buf, void *arg);
@@ -40,31 +44,44 @@
     void *cb_arg;
 };
 
-void _remote_render_ctx_free (struct remote_render_ctx **ctx) {
+static void _remote_render_free (struct remote_render_ctx *ctx) {
     // free the data_bev
-    if ((*ctx)->data_bev) {
-        bufferevent_free((*ctx)->data_bev);
-        (*ctx)->data_bev = NULL;
+    if (ctx->data_bev) {
+        bufferevent_free(ctx->data_bev);
+        ctx->data_bev = NULL;
     }
+    
+    // close the socket (ctx->ev_conn remains valid even after we're done with it...)
+    close(event_get_fd(ctx->ev_conn));
 
     // and the event
-    event_free((*ctx)->ev_conn);
+    event_free(ctx->ev_conn);
     
     // free the context structure
-    free(*ctx);
-    
-    *ctx = NULL;
+    free(ctx);
 }
 
-#define RENDER_FAILED(ctx, desc) \
-    do {                                        \
-        perror(desc);                           \
-        ctx->cb_fail(ctx->cb_arg);              \
-        _remote_render_ctx_free(&ctx);          \
-        return;                                 \
-    } while (0)
+static void _remote_render_done (struct remote_render_ctx *ctx) {
+    assert(ctx->alive);
+    
+    ctx->alive = 0;
 
-void _remote_write (struct bufferevent *bev, void *arg) {
+    ctx->cb_done(ctx->cb_arg);
+
+    _remote_render_free(ctx);
+}
+
+static void _remote_render_fail (struct remote_render_ctx *ctx) {
+    assert(ctx->alive);
+    
+    ctx->alive = 0;
+
+    ctx->cb_fail(ctx->cb_arg);
+    
+    _remote_render_free(ctx);
+}
+
+static void _remote_write (struct bufferevent *bev, void *arg) {
     struct remote_render_ctx *ctx = arg;
 
     // the write buffer was drained, so the render command was sent
@@ -72,21 +89,25 @@
     
     // we don't care about EV_WRITE anymore
     if (bufferevent_disable(ctx->data_bev, EV_WRITE))
-        RENDER_FAILED(ctx, "render_remote: bufferevent_disable");
+        ERROR("bufferevent_disable");
 
     // start receiving data
     if (bufferevent_enable(ctx->data_bev, EV_READ))
-        RENDER_FAILED(ctx, "render_remote: bufferevent_enable");
+        ERROR("bufferevent_enable");
+    
+    return;
+error:
+    _remote_render_fail(ctx);
 }
 
-void _remote_read (struct bufferevent *bev, void *arg) {
+static void _remote_read (struct bufferevent *bev, void *arg) {
     struct remote_render_ctx *ctx = arg;
     
     // pass the bufferevent's input buffer to our callback - libevent doesn't provide any function to access this, but hopefully this works correctly
     ctx->cb_data(EVBUFFER_INPUT(bev), ctx->cb_arg);
 }
 
-void _remote_error (struct bufferevent *bev, short what, void *arg) {
+static void _remote_error (struct bufferevent *bev, short what, void *arg) {
     struct remote_render_ctx *ctx = arg;
 
     // OH NOES; WHAT DO WE DO!?
@@ -99,45 +120,48 @@
             ctx->cb_data(EVBUFFER_INPUT(bev), ctx->cb_arg);
 
         // signal completion
-        ctx->cb_done(ctx->cb_arg);
+        _remote_render_done(ctx);
+
+        return;
 
     } else if (what & EVBUFFER_ERROR) {
         // crap.
-        perr("render_remote");
-
-        ctx->cb_fail(ctx->cb_arg);
+        PWARNING("EVBUFFER_ERROR");
 
     } else if (what & EVBUFFER_TIMEOUT) {
         // ah well
-        error("render_remote: timeout");
-
-        ctx->cb_fail(ctx->cb_arg);
+        WARNING("render_remote: timeout");
 
     } else {
-        err_exit("weird bufferevent error code: 0x%02X", what);
+        FATAL("weird bufferevent error code: 0x%02X", what);
     }
-
-    // free resources
-    _remote_render_ctx_free(&ctx);
+    
+    // cb_fail + free
+    _remote_render_fail(ctx);
 }
 
-void _remote_connected (int fd, short event, void *arg) {
+static void _remote_connected (int fd, short event, void *arg) {
     struct remote_render_ctx *ctx = arg;
 
     // set up the read/write bufferevent
     if ((ctx->data_bev = bufferevent_new(fd, &_remote_read, &_remote_write, &_remote_error, ctx)) == NULL)
-        RENDER_FAILED(ctx, "render_remote: bufferevent_new");
+        ERROR("bufferevent_new");
 
     // write the render command
     if (bufferevent_write(ctx->data_bev, &ctx->render_cmd, sizeof(ctx->render_cmd)))
-        RENDER_FAILED(ctx, "render_remote: bufferevent_write");
+        ERROR("bufferevent_write");
 
     // wait for it to be written out
     if (bufferevent_enable(ctx->data_bev, EV_WRITE))
-        RENDER_FAILED(ctx, "render_remote: bufferevent_enable");
+        ERROR("bufferevent_enable");
+    
+    return;
+
+error:
+    _remote_render_fail(ctx);
 }
 
-void render_cmd_build (struct render *render, struct remote_render_ctx *ctx) {
+static void render_cmd_build (struct render *render, struct remote_render_ctx *ctx) {
     // just copy over the render params to the render_cmd
     ctx->render_cmd.mode = render->mode;
     ctx->render_cmd.img_w = htonl(render->img_w);
@@ -197,6 +221,9 @@
     if (event_add(ctx->ev_conn, NULL))
         ERROR("event_add");
     
+    // we are now alive
+    ctx->alive = 1;
+
     // success
     return ctx;
 
@@ -228,16 +255,14 @@
 }
 
 void render_remote_cancel (struct remote_render_ctx *ctx) {
-    // if it's still just connecting, cancel that
-    if (event_pending(ctx->ev_conn, EV_WRITE, NULL)) {
-        event_del(ctx->ev_conn);
+    // we must be alive for this..
+    assert(ctx->alive);
 
-    }
-    
-    // close the socket (ctx->ev_conn remains valid even after we're done with it...)
-    close(event_get_fd(ctx->ev_conn));
+    // if it's still just connecting, cancel that
+    if (event_pending(ctx->ev_conn, EV_WRITE, NULL))
+        event_del(ctx->ev_conn);
     
     // this takes care of the rest
-    _remote_render_ctx_free (&ctx);
+    _remote_render_free (ctx);
 }
 

--- a/web_main.c	Fri Jun 06 23:37:45 2008 +0300
+++ b/web_main.c	Sat Jun 07 05:05:18 2008 +0300
@@ -16,11 +16,15 @@
 
 #include "common.h"
 #include "http.h"
+#include "render_internal.h"
 #include "render.h"
-#include "render_remote.h"
 #include "remote_node.h"
 #include "remote_pool.h"
 
+// XXX: replace with a common one
+#include "render_remote.h"
+#include "render_multi.h"
+
 #define MIN_CHUNK_SIZE 4096
 #define OVERFLOW_BUFFER 4096
 
@@ -39,7 +43,7 @@
 
     int headers_sent;
     
-    struct remote_render_ctx *remote_ctx;
+    struct render_multi *render_info;
 
     size_t bytes_sent;
 
@@ -47,18 +51,18 @@
 };
 
 // cb func prototypes
-void _render_http_written (struct evhttp_request *request, void *arg);
+static void _render_http_written (struct evhttp_request *request, void *arg);
 
-void _render_cleanup (struct render_request *ctx) {
+static void _render_cleanup (struct render_request *ctx) {
     // clean up
     free(ctx);
 }
 
-void _render_sent (void *arg) {
+static void _render_sent (void *arg) {
     struct render_request *ctx = arg;
 
     // set chunk size
-    render_remote_set_recv(ctx->remote_ctx, MIN_CHUNK_SIZE, OVERFLOW_BUFFER);
+    render_multi_set_recv(ctx->render_info, MIN_CHUNK_SIZE, OVERFLOW_BUFFER);
 
     // send headers
     evhttp_add_header(evhttp_request_get_output_headers(ctx->http_request), "Content-Type", "image/png");
@@ -72,7 +76,7 @@
     printf("render [%p]: sent headers\n", ctx);
 }
 
-void _render_data (struct evbuffer *buf, void *arg) {
+static void _render_data (struct evbuffer *buf, void *arg) {
     struct render_request *ctx = arg;
 
     size_t buf_size = EVBUFFER_LENGTH(buf);
@@ -104,7 +108,7 @@
     ctx->bytes_sent += buf_size;
 }
 
-void _render_done (void *arg) {
+static void _render_done (void *arg) {
     struct render_request *ctx = arg;
 
     // if we are paused, just shove the data into the http buffers, they might become larger than they should be, but it's easier to just move the data there and let render_remote complete
@@ -112,7 +116,7 @@
         printf("render [%p]: done: flushing the rest of our data\n", ctx);
         ctx->paused = 0;
 
-        render_remote_shake(ctx->remote_ctx);
+        render_multi_shake(ctx->render_info);
     }
 
     // send end
@@ -124,7 +128,7 @@
     _render_cleanup(ctx);
 }
 
-void _render_fail (void *arg) {
+static void _render_fail (void *arg) {
     struct render_request *ctx = arg;
 
     if (ctx->headers_sent) {
@@ -139,18 +143,18 @@
     _render_cleanup(ctx);
 }
 
-void _render_http_lost (struct evhttp_request *req, void *arg) {
+static void _render_http_lost (struct evhttp_request *req, void *arg) {
     struct render_request *ctx = arg;
 
     printf("render [%p]: lost http connection\n", ctx);
 
     // cancel
-    render_remote_cancel(ctx->remote_ctx);
+    render_multi_cancel(ctx->render_info);
 
     _render_cleanup(ctx);
 }
 
-void _render_http_written (struct evhttp_request *request, void *arg) {
+static void _render_http_written (struct evhttp_request *request, void *arg) {
     struct render_request *ctx = arg;
     
     printf("render [%p]: http available for write\n", ctx);
@@ -159,10 +163,10 @@
     ctx->paused = 0;
 
     // shake out the buffers
-    render_remote_shake(ctx->remote_ctx);
+    render_multi_shake(ctx->render_info);
 }
 
-void _http_render_execute (struct evhttp_request *request, u_int32_t img_w, u_int32_t img_h) {
+static void _http_render_execute (struct evhttp_request *request, u_int32_t img_w, u_int32_t img_h) {
     // render request context
     struct render_request *req_ctx = calloc(1, sizeof(struct render_request));
 
@@ -174,35 +178,34 @@
     req_ctx->bytes_sent = 0;
     
     // render context
-    struct render *render;
+    struct render render;
     
-    if (!(render = render_alloc()))
+    if (render_init(&render, RENDER_PNG))
         ERROR("render_alloc");
 
-    if (render_set_mode(render, RENDER_PNG))
-        ERROR("render_set_mode");
-
-    if (render_set_size(render, img_w, img_h))
+    if (render_set_size(&render, img_w, img_h))
         ERROR("render_set_size");
 
-    if (render_region_full(render))
+    if (render_region_full(&render))
         ERROR("render_region_full");
 
+/*
     // pick a render_node
     struct remote_node *node_info;
 
     if ((node_info = remote_pool_get(&remote_pool)) == NULL)
         ERROR("remote_pool_get");
+*/
 
     // initiate the remote render operation
-    if ((req_ctx->remote_ctx = render_remote(render, node_info,
+    if ((req_ctx->render_info = render_multi(&render, &remote_pool,
         &_render_sent,
         &_render_data,
         &_render_done,
         &_render_fail,
         req_ctx
     )) == NULL)
-        ERROR("render_remote");
+        ERROR("render_multi");
 
     // set close cb
     evhttp_set_reply_abortcb(request, &_render_http_lost, req_ctx);
@@ -229,7 +232,7 @@
 
     evhttp_request_get_peer(request, &peer_address, &peer_port);
     
-    u_int32_t img_w = 256, img_h = 256;
+    unsigned long int img_w = 256, img_h = 256;
 
     // parse request arguments
     struct http_qarg qarg_spec[] = {
@@ -241,7 +244,7 @@
     http_qarg_parse(request, qarg_spec);
 
     // request log
-    printf("REQ: [%s:%d] method=%d, uri=%s, img_w=%d, img_h=%d\n", peer_address, peer_port, evhttp_request_get_type(request), uri, img_w, img_h);
+    printf("REQ: [%s:%d] method=%d, uri=%s, img_w=%lu, img_h=%lu\n", peer_address, peer_port, evhttp_request_get_type(request), uri, img_w, img_h);
     
     // do it
     _http_render_execute(request, img_w, img_h);