# HG changeset patch
# User terom@cl-543.hel-01.fi.sixxs.net
# Date 1218130086 -10800
# Node ID 33e464fd6773633eba0d77630c102603abdf3450
# Parent  605f4459a294a37011b31505a489e1e54303a240
my hg working dir managed to break itself somehow... my attempt to fix that, and add some cache code :)

diff -r 605f4459a294 -r 33e464fd6773 .hgignore
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgignore	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,9 @@
+syntax: glob
+
+*.o
+.*.swp
+lib/
+data/*
+*_main
+Makefile.*
+
diff -r 605f4459a294 -r 33e464fd6773 cache.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,157 @@
+#ifndef CACHE_H
+#define CACHE_H
+
+/*
+ * The interface to the internal caching mechanism.
+ *
+ * This presents a pretty generic API, which allows the actual implementation to change, or to compare multiple
+ * different implementations.
+ *
+ */
+
+
+/*
+ * The handle used to access the global cache.
+ */
+struct cache;
+
+/*
+ * A cache request/operation.
+ */
+struct cache_req;
+
+/*
+ * The actual implementation of the cache.
+ */
+struct cache_engine;
+
+/*
+ * What we use as keys in the cache. Key is a pointer to an arbitrary char buffer, length is the size of the key
+ * in bytes. If this is given as zero, it will be calcuated using strlen(). Zero-length keys are invalid.
+ */
+struct cache_key_t {
+    const char *buf;
+    size_t length;
+};
+
+/*
+ * The various states that a cache request can be in
+ */
+enum cache_req_state {
+    CACHE_STATE_INVALID,
+
+    CACHE_STATE_LOOKUP,
+    
+    CACHE_STATE_OPEN,
+
+    CACHE_STATE_WRITE,
+    CACHE_STATE_WRITE_PAUSE,
+    CACHE_STATE_READ,
+
+    CACHE_STATE_DONE,
+    CACHE_STATE_ERROR,
+};
+
+/*
+ * Transitions between states
+ */
+enum cache_req_event {
+    // LOOKUP -> OPEN
+    CACHE_EVENT_HIT,
+    CACHE_EVENT_MISS,
+    
+    // OPEN -> WRITE
+    CACHE_EVENT_BEGIN_WRITE,
+    
+    // OPEN -> READ
+    CACHE_EVENT_BEGIN_READ,
+    
+    // WRITE -> PAUSE_WRITE
+    CACHE_EVENT_PAUSE_WRITE,
+
+    // PAUSE_WRITE -> WRITE
+    CACHE_EVENT_RESUME_WRITE,
+
+    // READ -> READ
+    CACHE_EVENT_DATA_AVAILABLE,
+    
+    // READ -> DONE
+    CACHE_EVENT_DONE,
+
+    // * -> ERROR
+    CACHE_EVENT_ERROR,
+};
+
+/*
+ * The callback used for cache_reqs
+ */
+typedef (int) (*cache_callback) (struct cache_req *, enum cache_req_event, void *arg);
+
+
+
+
+
+
+/*
+ * Open up a cache using the given engine (which can be configured with engine-specific params).
+ */
+struct cache *cache_open (struct cache_engine *engine);
+
+/*
+ * Create a new request. The given callback function will be called at the various stages in the request, and can then
+ * drive the request forward.
+ */
+struct cache_req *cache_request (struct cache *cache, struct cache_key *key, cache_callback cb_func, void *cb_data);
+
+/*
+ * Get the request's state.
+ */ 
+enum cache_req_state cache_request_state (struct cache_req *req);
+
+/*
+ * Get information about the amount of data in this cache entry.
+ *      size        - the total size of the cache entry. -1 if unknown
+ *      offset      - how many bytes of data the cache entry contains
+ *      available   - how many unread bytes are available
+ */
+void cache_req_available (struct cache_req *req, ssize_t *size, ssize_t *offset, ssize_t *available);
+
+/*
+ * Add some data into this cache entry, reading from the given fd. This is only valid for cache_req's in
+ * CACHE_REQ_WRITE mode.
+ *
+ * If you know the amount of data that should be pushed, you can supply it in size. If size is given as 0, all
+ * available data will be consumed. Size will be updated to the number of bytes pushed into the cache.
+ *
+ * (size == 0) is valid if req is in the WRITE_PAUSE state (either before or after the call).
+ */
+int cache_req_push (struct cache_req *req, int fd, size_t *size);
+
+/*
+ * Get some data from this cache entry, writing it into the given fd. This is valid for all cache_req's in
+ * CACHE_REQ_READ and CACHE_REQ_WRITE mode.
+ *
+ * If the value of size is given (nonzero), then the given amount of data will be written to the fd from the cache.
+ * If the cache contains less data than given, this is an error. If the value of size is zero, as much data as possible
+ * will be written to the fd. Size will be updated to the number of bytes pulled from the cache.
+ */
+int cache_req_pull (struct cache_req *req, int fd, size_t *size);
+
+/*
+ * Abort the given cache request (external failure). This is only needed for CACHE_REQ_WRITE requests, and is invalid
+ * for other requests. Any dependant cache requests will fail.
+ */
+void cache_req_abort (struct cache_req *req);
+
+/*
+ * Indicate that the given cache write request is complete. This is only valid for CACHE_REQ_WRITE requests, and is
+ * invalid for other requests. Any dependant cache requests will complete.
+ */
+int cache_req_done (struct cache_req *req);
+
+/*
+ * Release the given cache request. Only valid after calling req_abort/req_done, or getting EVENT_DONE/EVENT_ERROR.
+ */
+void cache_req_release (struct cache_req *req);
+
+#endif /* CACHE_H */
diff -r 605f4459a294 -r 33e464fd6773 cache/cache.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/cache.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,24 @@
+
+#include "../cache.h"
+#include "cache.h"
+
+struct cache *cache_open (struct cache_engine *engine) {
+    struct cache *cache;
+
+    if (engine->fn_init(engine, &cache))
+        goto error;
+
+    return cache;
+
+error:
+    return NULL;
+}
+
+int cache_init (struct cache *cache, struct cache_engine *engine) {
+    cache->engine = engine;
+
+    LIST_INIT(&cache->op_list);
+
+    return 0;
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/cache.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/cache.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,19 @@
+#ifndef CACHE_CACHE_H
+#define CACHE_CACHE_H
+
+#include <sys/queue.h>
+
+#include "op.h"
+
+struct cache {
+    struct cache_engine *engine;
+    
+    /*
+     * List of currently active operations
+     */
+    LIST_HEAD(cache_op_list_head, cache_op) op_list;
+};
+
+int cache_init (struct cache *cache, struct cache_engine *engine);
+
+#endif /* CACHE_CACHE_H */
diff -r 605f4459a294 -r 33e464fd6773 cache/engine.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/engine.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,19 @@
+#ifndef CACHE_ENGINE_H
+#define CACHE_ENGINE_H
+
+struct cache_engine {
+    /*
+     * Allocate a `struct cache`-compatible struct and return it via cache_ptr.
+     */
+    int (fn_init*) (struct cache_engine *engine, struct cache **cache_ptr);
+    
+    /*
+     * Allocate a `struct cache_op`-compatible struct and return it via cache_ptr.
+     *
+     * Begin the index lookup.
+     */
+    int (fn_op_start*) (struct cache *cache, struct cache_op **op_ptr, struct cache_key *key);
+};
+
+#endif /* CACHE_ENGINE_H */
+
diff -r 605f4459a294 -r 33e464fd6773 cache/engines/fs.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/engines/fs.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,156 @@
+
+#include "../cache.h"
+#include "../engine.h"
+#include "../op.h"
+#include "../../common.h"
+
+#define FS_BLOCK_START 4096
+#define FS_BLOCK_GROW_FACTOR 2
+
+struct cache_engine_fs {
+    struct cache_engine base;
+    
+    // custom stuff
+    const char *cache_dir;
+};
+
+struct cache_fs {
+    struct cache base;
+
+    // custom stuff?
+};
+
+struct cache_op_fs {
+    struct cache_op base;
+
+    // custom
+    int fd;
+
+    off_t file_size;
+
+    void *mmap;
+};
+
+static int _fs_mmap (struct cache_op_fs *op, int grow) {
+    off_t old_size = op->file_size;
+
+    if (op->file_size == 0 || grow) {
+        // update size and ftruncate
+        op->file_size = op->file_size ? op->file_size * 2 : FS_INITIAL_SIZE;
+
+        if (ftruncate(ctx->fd, ctx->file_size))
+            PERROR("ftruncate");
+    }
+    
+    if (op->mmap) {
+        // XXX: test
+        if ((ctx->mmap = mremap(ctx->mmap, old_size, ctx->file_size, MREMAP_MAYMOVE)) == MAP_FAILED)
+            PERROR("mremap");
+
+    } else {
+        if ((ctx->mmap = mmap(NULL, ctx->file_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ctx->fd, 0)) == MAP_FAILED)
+            PERROR("mmap");
+
+    }
+
+    return 0;
+
+error:
+    return -1;
+}
+
+static int _fs_do_init (struct cache_engine *engine, struct cache **cache_ptr) {
+    struct cache_engine_fs *ctx = (struct cache_engine_fs *) engine;
+    struct cache_fs *cache = NULL;
+
+
+    if ((cache = calloc(1, sizeof(*cache))) == NULL)
+        ERROR("calloc");
+
+    if (cache_init(&cache->base, &ctx->base))
+        goto error;
+
+    *cache_ptr = &cache->base;
+    
+    return 0;
+
+error:
+    free(cache);
+
+    return -1;
+}
+    
+static int _fs_do_op_start (struct cache *cache, struct cache_op **op_ptr, struct cache_key *key) {
+    struct cache_engine_fs *ctx = (struct cache_engine_fs *) engine;
+    struct cache_op_fs *op = NULL;
+    char path[PATH_MAX];
+    struct stat stat;
+    int found = 0;
+    
+    // allocate it
+    if ((op = calloc(1, sizeof(*op))) == NULL)
+        ERROR("calloc");
+    
+    // init it
+    if (cache_op_init(&op->base, cache, key))
+        goto error;
+    
+    // mark it as being in the lookup state... shouldn't even be needed, as this is sync
+    op->base.state = OP_STATE_LOOKUP;
+    
+    // construct the path to the appropriate file
+    if (snprintf(path, PATH_MAX, "%s/%s", ctx->cache_dir, key->key) >= PATH_MAX)
+        ERROR("path too long: %s/%s", ctx->cache_dir, key->key);
+    
+    // open the appropriate file
+    if ((op->fd = open(path, O_CREAT | O_RDWR, 0644)) == -1)
+        PERROR("open: %s", path);
+    
+    // stat for filesize
+    if (fstat(op->fd, &stat))
+        PERROR("fstat");
+    
+    op->file_size = stat.st_size;
+    
+    // size == 0 -> new file -> not found
+    found = (size > 0);
+
+    // grow if needed, and then mmap
+    if (_fs_mmap(op, 0))
+        goto error;
+    
+    // indicate that the key was found/not found
+    if (cache_op_lookup_done(&op->base, found))
+        goto error;
+    
+    // done!
+    return 0;
+
+error:
+    free(cache);
+
+    return -1;
+   
+}
+
+struct cache_engine_fs *cache_engine_fs (const char *cache_dir) {
+    struct cache_engine_fs *ctx = NULL;
+
+
+    if ((ctx = calloc(1, sizeof(*ctx))) == NULL)
+        ERROR("calloc");
+
+    ctx->cache_dir = cache_dir;
+
+    // set up the fn table
+    ctx->base.fn_init = &_fs_do_init;
+    ctx->base.fn_op_start = &_fs_do_op_start;
+
+    return 0;
+
+error:
+    free(ctx);
+
+    return -1;
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/engines/fs.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/engines/fs.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,14 @@
+#ifndef CACHE_ENGINE_FS_H
+#define CACHE_ENGINE_FS_H
+
+#include "../engine.h"
+
+struct cache_engine_fs {
+    struct cache_engine base;
+    
+    // custom stuff
+    const char *cache_dir;
+};
+
+#endif /* CACHE_ENGINE_FS_H */
+
diff -r 605f4459a294 -r 33e464fd6773 cache/op.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/op.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,55 @@
+
+#include <sys/queue.h>
+#include <string.h>
+
+#include "op.h"
+#include "../common.h"
+
+int cache_op_init(struct cache_op *op, struct cache *cache, struct cache_key *key) {
+    op->cache = cache;
+    op->key = key;
+    op->state = OP_STATE_INVALID;
+
+    LIST_INIT(&op->req_list);
+}
+
+struct cache_op *cache_op_find (struct cache *cache, struct cache_key *key) {
+    struct cache_op *op;
+
+    for (op = cache->op_list.lh_first, op != NULL; op = op->node.le_next) {
+        if (op->key->length == key->length && memcmp(op->key->buf, key->buf, key->length))
+            break;
+    }
+
+    if (op)
+        cache_op_incref(op);
+
+    return op;
+}
+
+int cache_op_register (struct cache_op *op, struct cache_req *req) {
+    LIST_INSERT_HEAD(&op->req_list, req, node);
+}
+
+static int _cache_op_notify (struct cache_op *op) {
+    struct cache_req *req;
+
+    for (req = op->req_list.lh_first, req != NULL; req = req->node.le_next) {
+        if (cache_req_notify(req))
+            goto error;
+    }
+
+    return 0;
+
+error:
+    return -1;
+}
+
+int cache_op_lookup_done (struct cache_op *op, int found) {
+    // modify state
+    op->state = found ? OP_STATE_HIT : OP_STATE_MISS;
+
+    // notify waiting reqs
+    return _cache_op_notify(op);
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/op.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/op.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,45 @@
+#ifndef CACHE_OP_H
+#define CACHE_OP_H
+
+enum cache_op_state {
+    OP_STATE_INVALID,
+    OP_STATE_LOOKUP,
+
+    OP_STATE_MISS,
+    OP_STATE_HIT,
+};
+
+struct cache_op {
+    struct cache *cache;
+
+    LIST_ENTRY(cache_op) node;
+
+    struct cache_key *key;
+
+    LIST_HEAD(cache_op_req_list_head, cache_req) req_list;
+
+    enum cache_op_state state;
+};
+
+/*
+ * Initialize the basic cache op state. Refcount is set to one.
+ */
+int cache_op_init(struct cache_op *op, struct cache *cache, struct cache_key *key);
+
+/*
+ * Look for an existing cache_op in the given cache. Return NULL if not found, return pointer and increment
+ * refcount if found.
+ */
+struct cache_op *cache_op_find (struct cache *cache, struct cache_key *key);
+
+/*
+ * Add the given cache_req to the list of reqs using this op
+ */
+int cache_op_register (struct cache_op *op, struct cache_req *req);
+
+/*
+ * Used by the engines to notify that the key lookup completed
+ */
+int cache_op_lookup_done (struct cache_op *op, int found);
+
+#endif /* CACHE_OP_H */
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,74 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/param.h>
+#include <string.h>
+
+
+#include "cache.h"
+#include "common.h"
+
+static int _cache_open_file (int *fd, const char *cache_path, const char *prefix, unsigned int id) {
+    char path[PATH_MAX];
+
+    if (snprintf(path, PATH_MAX, "%s/%s_%u", cache_path, prefix, id) >= PATH_MAX) 
+        ERROR("path too long: %s/%s_%u", cache_path, prefix, id);
+    
+    if ((*fd = open(path, O_CREAT | O_RDWR, 0644)) == -1)
+        PERROR("open: %s", path);
+    
+    return 0;
+
+error:
+    *fd = -1;
+
+    return -1;
+}
+
+void cache_close (struct cache *ctx) {
+    if (ctx->index.data)
+        munmap(ctx->index.data, ctx->index.size);
+
+    if (ctx->index.fd != -1) 
+        close(ctx->index.fd);
+
+    for (int id = 0; id < CACHE_NEST_COUNT; id++) 
+        if (ctx->nests[id].fd != -1)
+            close(ctx->nests[id].fd);
+}
+
+int cache_open (struct cache *ctx, const char *cache_path) {
+    // zero all state
+    memset(ctx, 0, sizeof(ctx));
+    ctx->index.fd = -1;
+    for (int id = 0; id < CACHE_NEST_COUNT; id++) ctx->nests[id].fd = -1;
+
+    // get system page size
+    if ((sys_pagesize = sysconf(_SC_PAGESIZE)) == -1)
+        PERROR("sysconf(_SC_PAGESIZE)");
+
+    // open the index
+    if (_cache_open_file(&ctx->index.fd, cache_path, "idx", 0))
+        ERROR("open index");
+
+    // open the nests
+    for (int id = 0; id < CACHE_NEST_COUNT; id++) {
+        if (_cache_open_file(&ctx->nests[id].fd, cache_path, "nest", id))
+            ERROR("open nest %d", id);
+    }
+    
+    // init the index
+    if (_cache_index_init(&ctx->index))
+        goto error;
+
+    return 0;
+
+error:
+    cache_close(ctx);
+
+    return -1;
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,46 @@
+#ifndef CACHE_H
+#define CACHE_H
+
+#include <sys/types.h>
+
+/*
+ * Configuration
+ */
+
+#define CACHE_NEST_COUNT 16
+#define CACHE_KEY_SIZE 8
+
+struct cache {
+    struct cache_index {
+        int     fd;
+        struct cache_index_file *data;
+        off_t   size;
+
+    } index;
+
+    struct cache_nest {
+        int     fd;
+        struct cache_nest_header *nest;
+        off_t   size;
+
+        struct cache_op *w_op;
+
+    } nests[CACHE_NEST_COUNT];
+    
+    struct cache_op *wop_queue;
+    struct cache_op *rop_list;
+};
+
+// system page size, initialized by cache_open
+static long sys_pagesize;
+
+// cache.c
+void cache_close (struct cache *ctx);
+int cache_open (struct cache *ctx, const char *cache_path);
+
+// cache_index.c
+int _cache_index_init (struct cache_index *ctx);
+int _cache_index_find (struct cache_index *ctx, const u_int8_t key[CACHE_KEY_SIZE], u_int8_t *nest, u_int32_t *offset, u_int16_t *length);
+int _cache_index_insert (struct cache_index *ctx, const u_int8_t key[CACHE_KEY_SIZE], const u_int8_t nest, const u_int32_t offset, const u_int16_t length);
+
+#endif /* CACHE_H */
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache_index.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache_index.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,134 @@
+#define _GNU_SOURCE
+#include <sys/mman.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "cache.h"
+#include "common.h"
+
+
+/*
+ * On-disk structs
+ */
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct cache_index_data_node {
+    u_int8_t        key[CACHE_KEY_SIZE];
+    u_int8_t        nest;
+    u_int32_t       offset;
+    u_int16_t       length;
+};
+
+struct cache_index_file {
+    u_int16_t        entry_count;
+
+    struct cache_index_data_node entries[];
+};
+
+#pragma pack(pop)
+
+static int _cache_index_grow (struct cache_index *ctx) {
+    off_t old_size;
+    
+    // remember the old size
+    old_size = ctx->size;
+
+    // calc the new size
+    ctx->size += sys_pagesize;
+
+    // grow the underlying file
+    if (ftruncate(ctx->fd, ctx->size))
+        PERROR("ftruncate");
+
+    // mmap/mremap
+    if ((ctx->data = mremap(ctx->data, old_size, ctx->size, MREMAP_MAYMOVE)) == MAP_FAILED)
+        PERROR("mremap"); 
+
+    return 0;
+
+error:
+    return -1;
+}
+
+int _cache_index_init (struct cache_index *ctx) {
+    struct stat stat_info;
+
+    // figure out initial index size
+    if (fstat(ctx->fd, &stat_info))
+        PERROR("fstat");
+    
+    // file size
+    ctx->size = stat_info.st_size;
+    
+    if (ctx->size == 0) {
+        // init it to one page
+        ctx->size = sys_pagesize;
+
+        // grow the underlying file
+        if (ftruncate(ctx->fd, ctx->size))
+            PERROR("ftruncate");
+
+        // the file now contains all zeros...
+    }
+    
+    // initial mmap
+    if ((ctx->data = mmap(NULL, ctx->size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ctx->fd, 0)) == MAP_FAILED)
+        PERROR("mmap");
+    
+    return 0;
+
+error:
+    return -1;
+}
+
+int _cache_index_find (struct cache_index *ctx, const u_int8_t key[CACHE_KEY_SIZE], struct cache_nest **nest, u_int32_t *offset, u_int16_t *length) {
+    int idx = -1;
+
+    for (int i = 0; i < ctx->data->entry_count; i++) {
+        if (memcmp(ctx->data->entries[i].key, key, CACHE_KEY_SIZE) == 0) {
+            idx = i;
+            break;
+        }
+    }
+
+    if (idx == -1) {
+        return -1;
+
+    } else {
+        *nest = &ctx->entries[ctx->data->entries[idx].nest];
+        *offset = ctx->data->entries[idx].offset;
+        *length = ctx->data->entries[idx].length;
+
+        return 0;
+    }
+}
+
+int _cache_index_insert (struct cache_index *ctx, const u_int8_t key[CACHE_KEY_SIZE], const u_int8_t nest, const u_int32_t offset, const u_int16_t length) {
+    // check that this entry will fit into the cache
+    if ((ctx->data->entry_count + 1) * sizeof(struct cache_index_data_node) + sizeof(struct cache_index_file) < ctx->size) {
+        // grow the index table
+        if (_cache_index_grow(ctx))
+            goto error;
+
+    }
+    
+    // the struct that we're using
+    struct cache_index_data_node *info = &ctx->data->entries[ctx->data->entry_count++];
+    
+    // copy the info into the file
+    memcpy(info->key, key, CACHE_KEY_SIZE);
+    info->nest = nest;
+    info->offset = offset;
+    info->length = length;
+
+    return 0;
+
+error:
+    return -1;
+
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache_nest.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache_nest.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,26 @@
+/*
+ * On-disk structs
+ */
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct cache_nest_header {
+    u_int32_t        next_offset;
+};
+
+int _cache_nest_assign_op (struct cache *ctx, struct cache_op *op_info, struct cache_nest **nest_info) {
+    for (int i = 0; i < CACHE_NEST_COUNT; i++) {
+        if (ctx->nests[i].op == NULL) {
+            // found an unused nest
+            ctx->nests[i].op = op_info;
+            *nest_info = ctx->nests[i].op;
+
+            return 0;
+        }
+    }
+    
+    // no unused nests
+    return -1;
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache_op.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache_op.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,73 @@
+
+#include "cache.h"
+#include "common.h"
+
+struct cache_op {
+    u_int8_t key[CACHE_KEY];
+
+    struct cache_nest *nest;
+
+    u_int8_t obj_nest;
+    u_int32_t obj_offset;
+    u_int16_t obj_length:
+
+    void *mmap_addr;
+    
+    // for linked lists
+    struct cache_op *next;
+};
+
+int _cache_op_get (struct cache_op **op_info, struct cache *cache_info, u_int8_t key[CACHE_KEY_SIZE]) {
+    struct cache_op *op;
+
+    // first, search the read queue
+    for (op = cache_info->rop_list; op; op = op->next) {
+        if (memcmp(op->key, key, CACHE_KEY_SIZE))
+            break;
+    }
+
+    if (!op) {
+        // look for in-progress write ops
+        for (int i = 0; i < CACHE_NEST_COUNT; i++) {
+            op = ctx->nests[i].op;
+
+            if (op && memcmp(op->key, key, CACHE_KEY_SIZE))
+                break;
+    }
+
+    if (!op) {
+        // look for queued write ops
+        for (op = cache_info->wop_list; op; op = op->next) {
+            if (memcmp(op->key, key, CACHE_KEY_SIZE))
+                break;
+        }
+    }
+
+    if (!op) {
+        // alloc a new op struct
+        op = calloc(1, sizeof(*op));
+        
+        // look it up in the index
+        if (_cache_index_find(cache_info->index, key, &op->nest, &op->obj_offset, &op->obj_length) == 0) {
+            // found it in some nest
+            
+            // XXX: enter read list and start read
+
+        } else {
+            // not in the cache, try and find an unused nest to write it
+            if (_cache_nest_assign_op(cache_info, op, &op->nest) == 0) {
+                // excellent, we can start writing directly
+                
+                // XXX: start writing
+
+            } else {
+                // enter the wait queue...
+                
+                // XXX: enter write queue
+            }
+        }
+    } else {
+        // XXX: start reading
+    }
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/proto1/cache_req.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/proto1/cache_req.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,35 @@
+
+#include "cache.h"
+#include "common.h"
+
+enum cache_req_mode {
+    REQ_GET,
+    REQ_PUT,
+};
+
+struct cache_req {
+    struct cache_op *op;
+    
+    off_t offset;
+
+};
+
+int cache_req_start (struct cache_req *ctx, struct cache *cache_info, u_int8_t key[CACHE_KEY_SIZE]) {
+    // clear state
+    memset(ctx, 0, sizeof(*ctx));
+    
+    // get the op
+    if (_cache_op_get(&ctx->op, cache_info, key))
+        goto error;
+
+    
+}
+
+int cache_req_read_fd (struct cache_req *ctx, int fd) {
+    return cache_op_read_fd(ctx->op, fd, ctx->offset);
+}
+
+int cache_req_write_fd (struct cache_req *ctx, int fd) {
+    return cache_op_write_fd(ctx->op, fd, ctx->offset);
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/req.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/req.c	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,105 @@
+
+#include "../cache.h"
+#include "req.h"
+#include "op.h"
+#include "common.h"
+
+struct cache_req *cache_request (struct cache *cache, struct cache_key *key, cache_callback cb_func, void *cb_data) {
+    struct cache_req *req = NULL;
+
+    // calloc the req info
+    if ((req = calloc(1, sizeof(*req))) == NULL)
+        ERROR("calloc");
+    
+    // set up basic state
+    req->key = key;
+    req->cb_func = cb_func;
+    req->cb_data = cb_data;
+    req->is_write = 0;
+
+    // look for an existing cache_op for this key
+    if ((req->op = cache_op_find(cache, key)) == NULL) {
+
+        // none available, start a new cache op
+        if (cache->engine->fn_op_start(cache, &req->op, key))
+            goto error;
+
+        // since we were the one that created it, we take care of writing it
+        req->is_write = 1;
+        
+    } else {
+        // we found an existing cache_op, we can just use that
+        
+    }
+    
+    // register
+    if (cache_op_reigster(req->op, req))
+        goto error;
+
+    // hurray, we now have an active cache_op \o/
+    return 0;
+
+error:
+    free(req);
+
+    return -1;
+}
+
+int _cache_req_notify (struct cache_req *req, enum cache_req_event event) {
+    if (req->cb_func(req, event, req->cb_arg)) {
+        // XXX: handle errors
+        assert(0);
+    }
+    
+    return 0;
+}
+
+int cache_req_notify (struct cache_req *req) {
+    switch (req->op->state) {
+        case OP_STATE_INVALID:
+        case OP_STATE_LOOKUP:
+            assert(0);
+
+            break;
+        
+        case OP_STATE_HIT:
+            if (_cache_req_notify(req, CACHE_EVENT_HIT))
+                goto error;
+
+            break;
+
+        case OP_STATE_MISS:
+            if (_cache_req_notify(req, CACHE_EVENT_MISS))
+                goto error;
+            
+            break;
+        
+        default:
+            assert(0);
+
+            break;
+    }
+
+    return 0;
+
+error:
+    return -1;
+}
+
+enum cache_req_state cache_request_state (struct cache_req *req) {
+     switch (req->op->state) {
+        case OP_STATE_INVALID:
+            return CACHE_STATE_INVALID;
+
+        case OP_STATE_LOOKUP:
+            return CACHE_STATE_LOOKUP;
+        
+        case OP_STATE_HIT:
+        case OP_STATE_MISS:
+            return CACHE_STATE_OPEN;
+        
+        default:
+            assert(0);
+    }
+}
+
diff -r 605f4459a294 -r 33e464fd6773 cache/req.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache/req.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,25 @@
+#ifndef CACHE_REQ_H
+#define CACHE_REQ_H
+
+#include "../cache.h"
+#include "op.h"
+
+struct cache_req {
+    struct cache_key *key;
+    
+    LIST_ENTRY(cache_req) node;
+    
+    cache_callback cb_func;
+    void *cb_data;
+    
+    struct cache_op *op;
+
+    int is_write;
+};
+
+/*
+ * Called by cache_op to notify for changes in state
+ */
+int cache_req_notify (struct cache_req *req);
+
+#endif /* CACHE_REQ_H */
diff -r 605f4459a294 -r 33e464fd6773 cache_engines.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/cache_engines.h	Thu Aug 07 20:28:06 2008 +0300
@@ -0,0 +1,9 @@
+#ifndef CACHE_ENGINES_H
+#define CACHE_ENGINES_H
+
+struct cache_engine_fs;
+
+struct cache_engine_fs *cache_engine_fs (const char *cache_dir);
+
+
+#endif /* CACHE_ENGINES_H */