fix doc tpyos, rename some enums, fix printf format len for non-zero terminated strings (hg status), pass args to memcache_cmd_format_header via memcache_req_*, handle zero-length STORE requests, memcache_req is_buf_ours + free, other function name typos (keymemcache_req_key), fix req state behaviour re *_DATA_* for STORE requests and FETCH/END, better memcache_server connpool events/management, modular memcache_test with a working benchmark. This is a long commit message.
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "connection.h"
#include "command.h"
#include "request.h"
#include "../socket.h"
#include "../common.h"
void memcache_conn_send_req_data (struct memcache_conn *conn);
void memcache_conn_finish_req_data (struct memcache_conn *conn);
void memcache_conn_handle_reply (struct memcache_conn *conn);
void memcache_conn_handle_reply_data (struct memcache_conn *conn, struct evbuffer *buf);
static void _memcache_conn_ev_connect (evutil_socket_t fd, short what, void *arg);
static void _memcache_conn_bev_write (struct bufferevent *bev, void *arg);
static void _memcache_conn_bev_read (struct bufferevent *bev, void *arg);
static void _memcache_conn_bev_error (struct bufferevent *bev, short what, void *arg);
static void _memcache_conn_ev_write (evutil_socket_t fd, short event, void *arg);
static void _memcache_conn_ev_read (evutil_socket_t fd, short event, void *arg);
static void memcache_conn_error (struct memcache_conn *conn);
static void memcache_conn_req_done (struct memcache_conn *conn);
void memcache_conn_close (struct memcache_conn *conn);
struct memcache_conn *memcache_conn_open (struct memcache_server *server) {
struct memcache_conn *conn = NULL;
if ((conn = calloc(1, sizeof(*conn))) == NULL)
ERROR("calloc");
// remember the server
conn->server = server;
// attempt connect
if (memcache_conn_connect(conn))
ERROR("failed to connect to server");
// success
return conn;
error:
free(conn);
return NULL;
}
int memcache_conn_connect (struct memcache_conn *conn) {
assert(conn->fd <= 0 && !conn->is_connected);
// begin connect
if ((conn->fd = socket_connect_async(conn->server->endpoint, SOCK_STREAM)) == -1)
goto error;
// fd 0 should be stdin...
assert(conn->fd > 0);
// set up the connect event
event_set(&conn->ev_connect, conn->fd, EV_WRITE, &_memcache_conn_ev_connect, conn);
// add it
if (event_add(&conn->ev_connect, NULL))
PERROR("event_add");
// success
return 0;
error:
if (conn->fd > 0) {
if (close(conn->fd))
PWARNING("close %d", conn->fd);
conn->fd = -1;
}
return -1;
}
int memcache_conn_is_available (struct memcache_conn *conn) {
return (conn->fd > 0 && conn->is_connected && conn->req == NULL);
}
void memcache_conn_do_req (struct memcache_conn *conn, struct memcache_req *req) {
assert(conn->fd > 0 && conn->is_connected);
assert(conn->req == NULL);
// write the request header into our bufferevent's output buffer
if (memcache_cmd_format_header(bufferevent_get_output(conn->bev),
memcache_req_cmd(req),
memcache_req_key(req),
memcache_req_obj(req)
)) {
ERROR("failed to init the cmd");
}
// store the req
conn->req = req;
// tell our bufferevent to send it
if (bufferevent_enable(conn->bev, EV_WRITE))
PERROR("bufferevent_enable");
// tell the req that it is underway
memcache_req_send(req);
// success
return;
error:
if (conn->req)
memcache_conn_error(conn);
else
memcache_req_error(req);
}
/*
* Start writing out the request data
*/
void memcache_conn_send_req_data (struct memcache_conn *conn) {
if (conn->req->obj.bytes > 0) {
// set up the ev_write
event_set(&conn->ev_write, conn->fd, EV_WRITE, &_memcache_conn_ev_write, conn);
// just fake a call to the event handler
_memcache_conn_ev_write(conn->fd, EV_WRITE, conn);
} else {
// just send the \r\n
memcache_conn_finish_req_data(conn);
}
}
/*
* Write out the final \r\n to terminate the request data
*/
void memcache_conn_finish_req_data (struct memcache_conn *conn) {
if (bufferevent_write(conn->bev, "\r\n", 2))
PERROR("bufferevent_write");
// ok
return;
error:
memcache_conn_error(conn);
}
/*
* Start reading a reply from the connection
*/
void memcache_conn_handle_reply (struct memcache_conn *conn) {
// ensure that we either didn't have a command, or it has been sent
assert(conn->req->buf.data == NULL || conn->req->buf.offset == conn->req->buf.len);
// start/continue reading on the bufferevent
if (bufferevent_enable(conn->bev, EV_READ))
PERROR("bufferevent_enable");
// Note: we don't need to recurse into the callback ourselves in case there is data in it, since the read callback
// will consume all available data iteratively.
// ok, wait for the reply
return;
error:
memcache_conn_error(conn);
}
/*
* Start reading reply data from the connection
*/
void memcache_conn_handle_reply_data (struct memcache_conn *conn, struct evbuffer *buf) {
int ret;
// check that the buf doesn't contain any data
assert(conn->req->buf.data == NULL);
// bytes *may* be zero if we have an empty cache entry
if (conn->req->obj.bytes > 0) {
// XXX: memcache_req_make_buffer?
// allocate a buffer for the reply data
if ((conn->req->buf.data = malloc(conn->req->obj.bytes)) == NULL)
ERROR("malloc");
// update the length
conn->req->buf.len = conn->req->obj.bytes;
// set offset to zero
conn->req->buf.offset = 0;
// and note that it is present, and is ours
conn->req->have_buf = 1;
conn->req->is_buf_ours = 1;
// do we have any data in the buf that we need to copy?
if (evbuffer_get_length(buf) > 0) {
// read the data into the memcache_buf
ret = evbuffer_remove(buf, conn->req->buf.data, conn->req->buf.len);
// sanity check...
assert(ret > 0 && ret <= conn->req->buf.len);
// update offset
conn->req->buf.offset += ret;
}
// still need to receive more data?
if (conn->req->buf.offset < conn->req->buf.len) {
// disable the bufferevent while we read the data
if (bufferevent_disable(conn->bev, EV_READ))
PERROR("bufferevent_disable");
// set up the ev_read
event_set(&conn->ev_read, conn->fd, EV_READ, &_memcache_conn_ev_read, conn);
// then receive what data is left to receive
_memcache_conn_ev_read(conn->fd, EV_READ, conn);
// wait for the data to arrive
return;
} else {
// the buffer already contained the cache data, no need to read any more
}
} else {
// there is no data to receive for this item, so we can ignore this
}
// we kind of "recurse" to handle the MEMCACHE_RPL_END reply, that is, we activate the bufferevent for EV_READ
// again, use memcache_cmd_parse_header to parse the data (it will skip the "empty line" after the data and then
// return the MEMCACHE_RPL_END line). This will then have has_data=0, which will cause req_done to be called.
// Elegant!
memcache_conn_handle_reply(conn);
// ok
return;
error:
memcache_conn_error(conn);
}
/*
* The connect() has finished
*/
static void _memcache_conn_ev_connect (evutil_socket_t fd, short what, void *arg) {
struct memcache_conn *conn = arg;
int error;
if (socket_check_error(fd, &error))
goto error;
if (error)
ERROR("connect failed: %s", strerror(error));
// set up the bufferevent
if ((conn->bev = bufferevent_new(fd,
&_memcache_conn_bev_read,
&_memcache_conn_bev_write,
&_memcache_conn_bev_error,
conn
)) == NULL)
ERROR("bufferevent_new");
// mark us as succesfully connected
conn->is_connected = 1;
// notify the server
memcache_server_conn_ready(conn->server, conn);
// good
return;
error:
memcache_conn_error(conn);
}
/*
* The write buffer is empty, which means that we have written out a command header
*/
static void _memcache_conn_bev_write (struct bufferevent *bev, void *arg) {
struct memcache_conn *conn = arg;
// the command header has been sent
assert(evbuffer_get_length(bufferevent_get_output(bev)) == 0);
// does this request have some data to be included in the request?
// if the data has already been sent (we handle the final \r\n as well), then skip this.
if (conn->req->have_buf && conn->req->buf.offset == 0) {
// we need to send the request data next
memcache_conn_send_req_data(conn);
} else {
// wait for a reply
memcache_conn_handle_reply(conn);
}
}
/*
* We have received some reply data, which should include the complete reply line at some point
*/
static void _memcache_conn_bev_read (struct bufferevent *bev, void *arg) {
struct memcache_conn *conn = arg;
struct evbuffer *in_buf = bufferevent_get_input(bev);
struct memcache_key key;
char *header_data;
enum memcache_reply reply_type;
int has_data;
// ensure that we do indeed have some data
assert(evbuffer_get_length(in_buf) > 0);
// consume as much data as possible
do {
// attempt to parse the response header
if (memcache_cmd_parse_header(in_buf, &header_data, &reply_type, &key, &conn->req->obj, &has_data))
ERROR("memcache_cmd_parse_header");
if (!header_data) {
// no complete header received yet
return;
}
// if the reply contains data, check that they key is the same
if (has_data && (key.len != conn->req->key.len || memcmp(key.buf, conn->req->key.buf, key.len) != 0))
ERROR("got reply with wrong key !?!");
// notify the request (no reply data is ready for reading yet, though)
memcache_req_recv(conn->req, reply_type);
// does the reply include data?
if (has_data) {
// start reading the data (including whatever might be left over in the bufferevent buffer...)
memcache_conn_handle_reply_data(conn, in_buf);
} else {
// the request is done with
memcache_conn_req_done(conn);
}
// free the header data
free(header_data);
} while (evbuffer_get_length(in_buf) > 0);
// done
return;
error:
// free the header data read from the buf
free(header_data);
memcache_conn_error(conn);
}
static void _memcache_conn_bev_error (struct bufferevent *bev, short what, void *arg) {
struct memcache_conn *conn = arg;
// fail the entire connection
memcache_conn_error(conn);
}
static void _memcache_conn_ev_write (evutil_socket_t fd, short event, void *arg) {
struct memcache_conn *conn = arg;
struct memcache_buf *buf = &conn->req->buf;
int ret;
// correct event
assert(event == EV_WRITE);
// we do indeed have data to send
assert(buf->len > 0 && buf->data != NULL && buf->offset < buf->len);
// do the actual write()
if ((ret = write(fd, buf->data + buf->offset, buf->len - buf->offset)) == -1 && errno != EAGAIN)
PERROR("write");
// should never be the case... ?
if (ret == 0)
ERROR("write returned EOF !?!");
// did we manage to write some data?
if (ret > 0) {
// update offset
buf->offset += ret;
}
// data left to write?
if (buf->offset < buf->len) {
// reschedule
if (event_add(&conn->ev_write, NULL))
PERROR("event_add");
} else {
// done! Send the terminating \r\n next
memcache_conn_finish_req_data(conn);
}
// success
return;
error:
// fail the entire connection
memcache_conn_error(conn);
}
static void _memcache_conn_ev_read (evutil_socket_t fd, short event, void *arg) {
struct memcache_conn *conn = arg;
struct memcache_buf *buf = &conn->req->buf;
int ret;
// correct event
assert(event == EV_READ);
// we do indeed expect to receive data
assert(buf->len > 0 && buf->data != NULL && buf->offset < buf->len);
// do the actual read()
if ((ret = read(fd, buf->data + buf->offset, buf->len - buf->offset)) == -1 && errno != EAGAIN)
PERROR("read");
// should never be the case... ?
if (ret == 0)
ERROR("read returned EOF !?!");
// did we manage to read some data?
if (ret > 0) {
// update offset
buf->offset += ret;
}
// only notify the req if new data was received, and we won't be calling req_done next.
if (ret > 0 && buf->offset < buf->len) {
// notify the req
memcache_req_data(conn->req);
}
// data left to read?
if (buf->offset < buf->len) {
// reschedule
if (event_add(&conn->ev_read, NULL))
PERROR("event_add");
} else {
// done! We can let the bufferenvet handle the rest of the reply now
memcache_conn_handle_reply(conn);
}
// success
return;
error:
// fail the entire connection
memcache_conn_error(conn);
}
// XXX: need to flush/disable buffers/events on errors
/*
* The entire connection failed
*/
static void memcache_conn_error (struct memcache_conn *conn) {
// fail the request, if we have one
if (conn->req) {
// error out the req
memcache_req_error(conn->req);
// we are now available again
conn->req = NULL;
}
// close the connection
memcache_conn_close(conn);
// tell the server we failed
memcache_server_conn_dead(conn->server, conn);
}
/*
* Detach the request
*/
static void memcache_conn_req_done (struct memcache_conn *conn) {
// ensure that we do currently have a req
assert(conn->req);
// have the req detach
memcache_req_done(conn->req);
// we are now available again
conn->req = NULL;
memcache_server_conn_ready(conn->server, conn);
}
void memcache_conn_close (struct memcache_conn *conn) {
// close the fd if needed
if (conn->fd > 0) {
if (close(conn->fd))
PWARNING("close");
conn->fd = 0;
}
// ensure that the events are not pending anymore
assert(event_pending(&conn->ev_connect, EV_WRITE|EV_TIMEOUT, NULL) == 0);
assert(event_pending(&conn->ev_read, EV_READ|EV_TIMEOUT, NULL) == 0);
assert(event_pending(&conn->ev_write, EV_WRITE|EV_TIMEOUT, NULL) == 0);
// free the bufferevent
if (conn->bev) {
bufferevent_free(conn->bev);
conn->bev = NULL;
}
// not connected anymore
conn->is_connected = 0;
}
void memcache_conn_free (struct memcache_conn *conn) {
// ensure we don't have a req bound to us
assert(conn->req == NULL);
// ensure that the connection is not considered to be connected anymore
assert(!conn->is_connected);
// free it
free(conn);
}