# HG changeset patch # User Tero Marttila # Date 1241740280 -10800 # Node ID 42aedce3e2eb93ad92aa8637d05d41a74746fa72 # Parent 808b1b0476207b9dda0ed0bc1f6cdaece884113c rework test to implement flags, test_results, test_stats, TEST_WILL_FAIL diff -r 808b1b047620 -r 42aedce3e2eb src/error.h --- a/src/error.h Fri May 08 01:43:02 2009 +0300 +++ b/src/error.h Fri May 08 02:51:20 2009 +0300 @@ -252,6 +252,6 @@ * Macro used to mark code segments that should never be executed (e.g. switch-default), kind of like assert */ #include -#define NOT_REACHED() abort() +#define NOT_REACHED(val) abort() #endif diff -r 808b1b047620 -r 42aedce3e2eb src/log.c --- a/src/log.c Fri May 08 01:43:02 2009 +0300 +++ b/src/log.c Fri May 08 02:51:20 2009 +0300 @@ -100,7 +100,7 @@ _log_output_ctx.func(buf, _log_output_ctx.arg); } -void log_msg (enum log_level level, const char *func, const char *format, ...) +void _log_msg (enum log_level level, const char *func, const char *format, ...) { va_list vargs; @@ -140,3 +140,15 @@ va_end(vargs); } +void _log_exit (enum log_level level, int exit_code, const char *func, const char *format, ...) +{ + va_list vargs; + + // formatted output without any suffix + va_start(vargs, format); + log_output_tag(level, "EXIT", func, format, vargs, NULL); + va_end(vargs); + + // exit + exit(exit_code); +} diff -r 808b1b047620 -r 42aedce3e2eb src/log.h --- a/src/log.h Fri May 08 01:43:02 2009 +0300 +++ b/src/log.h Fri May 08 02:51:20 2009 +0300 @@ -65,47 +65,50 @@ /** * Log a message with the given level */ -void log_msg (enum log_level level, const char *func, const char *format, ...) +#define log_msg(level, ...) _log_msg(level, __func__, __VA_ARGS__) +void _log_msg (enum log_level level, const char *func, const char *format, ...) __attribute__ ((format (printf, 3, 4))); /** * Shorthand for log_msg */ -#define log_debug(...) log_msg(LOG_DEBUG, __func__, __VA_ARGS__) -#define log_info(...) log_msg(LOG_INFO, __func__, __VA_ARGS__) -#define log_warn(...) log_msg(LOG_WARN, __func__, __VA_ARGS__) +#define log_debug(...) log_msg(LOG_DEBUG, __VA_ARGS__) +#define log_info(...) log_msg(LOG_INFO, __VA_ARGS__) +#define log_warn(...) log_msg(LOG_WARN, __VA_ARGS__) /* #define log_error(...) log_msg(LOG_ERROR, __func__, __VA_ARGS__) */ -#define log_fatal(...) log_msg(LOG_FATAL, __func__, __VA_ARGS__) +#define log_fatal(...) log_msg(LOG_FATAL, __VA_ARGS__) /** * Log a message with the given level, appending the formatted error code name */ +#define log_err(err_code, ...) _log_err(LOG_ERROR, err_code, __func__, __VA_ARGS__) +#define log_warn_err(err_code, ...) _log_err(LOG_WARN, err_code, __func__, __VA_ARGS__) void _log_err (enum log_level level, err_t err, const char *func, const char *format, ...) __attribute__ ((format (printf, 4, 5))); /** * Log a message with the given level, appending the formatted error message */ +#define log_error(err_info, ...) _log_error(LOG_ERROR, err_info, __func__, __VA_ARGS__) +#define log_warn_error(err_info, ...) _log_error(LOG_WARN, err_info, __func__, __VA_ARGS__) void _log_error (enum log_level level, const error_t *err, const char *func, const char *format, ...) __attribute__ ((format (printf, 4, 5))); /** * Log using errno. */ +#define log_perr(...) _log_perr(LOG_ERROR, __func__, __VA_ARGS__) void _log_perr (enum log_level level, const char *func, const char *format, ...) __attribute__ ((format (printf, 3, 4))); /** - * Shorthand for _log_* - * - * XXX: kill log_err_info, add log_err_code + * Log with an [EXIT] tag at given level, and then exit with given exit code */ -#define log_err(err_code, ...) _log_err(LOG_ERROR, err_code, __func__, __VA_ARGS__) -#define log_error(err_info, ...) _log_error(LOG_ERROR, err_info, __func__, __VA_ARGS__) -#define log_perr(...) _log_perr(LOG_ERROR, __func__, __VA_ARGS__) +#define log_exit(level, exit_code, ...) _log_exit(level, exit_code, __func__, __VA_ARGS__) +void _log_exit (enum log_level level, int exit_code, const char *func, const char *format, ...) + __attribute__ ((format (printf, 4, 5))) + __attribute__ ((noreturn)); -#define log_warn_err(err_code, ...) _log_err(LOG_WARN, err_code, __func__, __VA_ARGS__) -#define log_warn_error(err_info, ...) _log_error(LOG_WARN, err_info, __func__, __VA_ARGS__) /** * log_fatal + exit failure @@ -127,4 +130,12 @@ */ #define FATAL_PERROR(...) do { _log_perr(LOG_FATAL, __func__, __VA_ARGS__); abort(); } while (0) +/** + * Exit with given code, also logging a message at LOG_INFO with an [EXIT] tag + */ +#define EXIT_INFO(exit_code, ...) log_exit(LOG_INFO, exit_code, __VA_ARGS__) +#define EXIT_WARN(exit_code, ...) log_exit(LOG_WARN, exit_code, __VA_ARGS__) +#define EXIT_ERROR(exit_code, ...) log_exit(LOG_ERROR, exit_code, __VA_ARGS__) +#define EXIT_FATAL(exit_code, ...) log_exit(LOG_FATAL, exit_code, __VA_ARGS__) + #endif /* LOG_H */ diff -r 808b1b047620 -r 42aedce3e2eb src/lua_objs.c --- a/src/lua_objs.c Fri May 08 01:43:02 2009 +0300 +++ b/src/lua_objs.c Fri May 08 02:51:20 2009 +0300 @@ -382,7 +382,7 @@ const char *msg = luaL_checkstring(L, 2); // log it - log_msg(level, "lua", "%s", msg); + _log_msg(level, "lua", "%s", msg); // ok return 0; diff -r 808b1b047620 -r 42aedce3e2eb src/test/fail.c --- a/src/test/fail.c Fri May 08 01:43:02 2009 +0300 +++ b/src/test/fail.c Fri May 08 02:51:20 2009 +0300 @@ -1,4 +1,5 @@ #include "fail.h" +#include "test.h" #include "../str.h" #include "../log.h" #include "backtrace.h" @@ -12,9 +13,9 @@ // print out a stack dump, not including this function or the backtrace func test_backtrace(skip + 2); - - // then exit with a failure code - abort(); + + // then jump out + longjmp(_test_ctx.jmp_fail, TEST_FAIL); } void test_fail (const char *func, const char *file, int line, int skip, const char *fmt, ...) @@ -28,3 +29,7 @@ va_end(vargs); } +void test_test_fail (void) +{ + fail_if(1, "testing failure: %s", "xxx"); +} diff -r 808b1b047620 -r 42aedce3e2eb src/test/test.c --- a/src/test/test.c Fri May 08 01:43:02 2009 +0300 +++ b/src/test/test.c Fri May 08 02:51:20 2009 +0300 @@ -31,6 +31,7 @@ OPT_DEBUG = 'd', OPT_QUIET = 'q', OPT_LIST = 'l', + OPT_FORCE = 'f', /** Options without short names */ _OPT_EXT_BEGIN = 0x00ff, @@ -44,6 +45,7 @@ {"debug", 0, NULL, OPT_DEBUG }, {"quiet", 0, NULL, OPT_QUIET }, {"list", 0, NULL, OPT_LIST }, + {"force", 0, NULL, OPT_FORCE }, {0, 0, 0, 0 }, }; @@ -58,6 +60,7 @@ printf(" --debug / -d display DEBUG log messages\n"); printf(" --quiet / -q supress INFO log messages\n"); printf(" --list / -l list all tests\n"); + printf(" --force / -f force tests to continue running after one fails\n"); } /** @@ -75,49 +78,139 @@ } /** - * Run the given test, updating the given counter + * Map the actual test_result for a test into the used test_result, handling things like TEST_WILL_FAIL */ -static void run_test (const struct test *test, size_t *test_count) +static enum test_result map_test_result (const struct test *test, enum test_result result) { + if (result == TEST_ERROR) + return TEST_ERROR; + + else if (test->flags & TEST_WILL_FAIL) + return (result == TEST_FAIL) ? TEST_PASS : TEST_FAIL; + + else + return result; +} + +/** + * Run the given test. + * + * Returns the result of running the test (PASS, FAIL, ERROR). + */ +static enum test_result run_test (struct test_stats *stats, const struct test *test) +{ + enum test_result result; + + // log it log_info("%s", test->name); - // count and run - (*test_count)++; - test->func(); + // set the failure jump point + if (setjmp(_test_ctx.jmp_fail)) { + // failed + result = TEST_FAIL; + + } else { + // count it + stats->run++; + + // run + test->func(); + + // ok + result = TEST_PASS; + } + + // map it + switch ((result = map_test_result(test, result))) { + case TEST_PASS: + log_info("%s: passed", test->name); + + stats->pass++; + + break; + + case TEST_FAIL: + log_warn("%s: failed", test->name); + + stats->fail++; + + break; + + case TEST_ERROR: + log_msg(LOG_ERROR, "%s: error: xxx", test->name); + + stats->error++; + + break; + + default: + NOT_REACHED(result); + } + + return result; } /** * Run the given NULL-terminated list of tests, optionally filtering against the given filter. * - * Returns the number of tests run, which may be zero. + * Updates the given \a stats. + * + * Returns a test_result representing the final state. */ -static size_t run_tests (const struct test tests[], const char *filter) +static enum test_result run_tests (struct test_stats *stats, const struct test tests[], const char *filter) { - size_t test_count = 0; const struct test *test; + enum test_result res; + bool skip = false; // run each test in turn for (test = tests; test->name; test++) { // filter out if given - if ((filter && strcmp(test->name, filter)) || (!filter && test->optional)) + if ((filter && strcmp(test->name, filter)) || (!filter && (test->flags & TEST_OPTIONAL))) continue; + // count it + stats->tests++; + + // skip it? + if (skip) + continue; + // run it - run_test(test, &test_count); + if ((res = run_test(stats, test)) != TEST_PASS && !_test_ctx.options.force) + // mark the rest of the tests to be skipped + skip = true; } - - return test_count; + + // compute return status + if (!stats->run) + return TEST_EMPTY; + + else if (stats->error) + return TEST_ERROR; + + else if (stats->fail) + return TEST_FAIL; + + else + return TEST_PASS; } int main (int argc, char **argv) { int opt, option_index; const char *filter = NULL; - - size_t test_count; + + struct test_stats stats; + enum test_result result; + enum log_level level; + + // initialize + memset(&_test_ctx, 0, sizeof(_test_ctx)); + memset(&stats, 0, sizeof(stats)); // parse options - while ((opt = getopt_long(argc, argv, "hdql", options, &option_index)) != -1) { + while ((opt = getopt_long(argc, argv, "hdqlf", options, &option_index)) != -1) { switch (opt) { case OPT_HELP: usage(argv[0]); @@ -134,6 +227,10 @@ case OPT_LIST: list_tests(_tests); exit(EXIT_SUCCESS); + + case OPT_FORCE: + _test_ctx.options.force = true; + break; case '?': usage(argv[0]); @@ -156,14 +253,36 @@ // setup the sockets stuff _test_ctx.ev_base = setup_sock(); - // run tests - if ((test_count = run_tests(_tests, filter)) == 0) - FATAL("no tests run"); - - // log - log_info("done, ran %zu tests", test_count); - - // ok - return EXIT_SUCCESS; + // run all tests + switch ((result = run_tests(&stats, _tests, filter))) { + case TEST_EMPTY: + EXIT_WARN(result, "no tests run"); + + // map test result to log level + case TEST_ERROR: + level = LOG_ERROR; + + break; + + case TEST_FAIL: + level = LOG_WARN; + + break; + + case TEST_PASS: + level = LOG_INFO; + + break; + + default : + level = LOG_FATAL; + + break; + } + + // exit with stats + log_exit(level, result, "ran %zu/%zu tests, %zu passes, %zu errors, %zu failures", + stats.run, stats.tests, stats.pass, stats.error, stats.fail + ); } diff -r 808b1b047620 -r 42aedce3e2eb src/test/test.h --- a/src/test/test.h Fri May 08 01:43:02 2009 +0300 +++ b/src/test/test.h Fri May 08 02:51:20 2009 +0300 @@ -14,6 +14,7 @@ #include #include #include +#include /** * Global test-running state @@ -22,8 +23,66 @@ /** The event_base that we have setup */ struct event_base *ev_base; + /** Failure jump point */ + jmp_buf jmp_fail; + + /** + * Command-line options for running the tests + */ + struct test_options { + /** Don't stop running tests if one fails */ + bool force; + } options; + } _test_ctx; +/** + * Flags for tests + */ +enum test_flag { + /** Do not run this test by default */ + TEST_OPTIONAL = 0x01, + + /** This test is expected to fail */ + TEST_WILL_FAIL = 0x02, +}; + +/** + * Test results. These all function as process exit codes. + */ +enum test_result { + /** The test ran sucesfully */ + TEST_PASS = 0, + + /** The test failed */ + TEST_FAIL = 1, + + /** The test error'd out */ + TEST_ERROR = 2, + + /** No tests run */ + TEST_EMPTY = 3, +}; + +/** + * Test stats + */ +struct test_stats { + /** How many tests were defined */ + size_t tests; + + /** How many tests were actually run */ + size_t run; + + /** How many of those passed */ + size_t pass; + + /** How many of those failed */ + size_t fail; + + /** How many of those error'd */ + size_t error; +}; /** * Global list of test definitions @@ -35,10 +94,10 @@ /** Test func */ void (*func) (void); - /** Do not run by default */ - bool optional; + /** Flags from enum test_flag */ + int flags; + } _tests[]; - #endif diff -r 808b1b047620 -r 42aedce3e2eb src/test/test_list.c --- a/src/test/test_list.c Fri May 08 01:43:02 2009 +0300 +++ b/src/test/test_list.c Fri May 08 02:51:20 2009 +0300 @@ -3,7 +3,7 @@ /** * Test function prototypes */ -#define TEST(name) extern void test_ ##name (void); +#define TEST(name, flags) extern void test_ ##name (void); #include "test_list.inc" @@ -11,9 +11,8 @@ /** * The array of test structs */ -#define TEST(name) { #name, test_ ## name, false }, -#define TEST_OPTIONAL(name) { #name, test_ ## name, true }, -#define TEST_END { NULL, NULL, false } +#define TEST(name, flags) { #name, test_ ## name, flags }, +#define TEST_END { NULL, NULL, 0 } const struct test _tests[] = { #include "test_list.inc" diff -r 808b1b047620 -r 42aedce3e2eb src/test/test_list.inc --- a/src/test/test_list.inc Fri May 08 01:43:02 2009 +0300 +++ b/src/test/test_list.inc Fri May 08 02:51:20 2009 +0300 @@ -4,8 +4,7 @@ * All tests must be added to this list. * * TEST() macro signature: - * #define TEST(name) - * #define TEST_OPTIONAL(name) + * #define TEST(name, flags) * #define TEST_END */ @@ -14,46 +13,35 @@ #error TEST macro not defined #endif -#ifndef TEST_OPTIONAL - /* Default to the same value as TEST() */ - #define TEST_OPTIONAL(name) TEST(name) -#endif - - /* Tests*/ -TEST ( str_quote ) -TEST ( str_format ) -TEST ( dump_str ) - -TEST ( resolve_addr ) -TEST ( resolve_result ) -TEST ( resolve ) - -TEST ( transport_test ) - -TEST ( fifo ) -TEST ( tcp ) - -TEST ( line_proto ) - -TEST ( irc_cmd ) -TEST ( irc_line_parse ) -TEST ( irc_line_build ) - -TEST ( irc_queue ) -TEST ( irc_conn ) -TEST ( irc_conn_self_nick ) -TEST ( irc_net ) -TEST ( irc_chan_add_offline ) -TEST ( irc_chan_namreply ) -TEST ( irc_chan_user_join ) -TEST ( irc_chan_user_part ) -TEST ( irc_chan_user_kick ) -TEST ( irc_chan_self_kick ) -TEST ( irc_chan_user_nick ) -TEST ( irc_chan_user_quit ) -TEST ( irc_chan_CTCP_ACTION ) -TEST ( irc_chan_privmsg ) +TEST ( test_fail, TEST_WILL_FAIL ) +TEST ( str_quote, 0 ) +TEST ( str_format, 0 ) +TEST ( dump_str, 0 ) +TEST ( resolve_addr, 0 ) +TEST ( resolve_result, 0 ) +TEST ( resolve, 0 ) +TEST ( transport_test, 0 ) +TEST ( fifo, TEST_OPTIONAL ) +TEST ( tcp, 0 ) +TEST ( line_proto, 0 ) +TEST ( irc_cmd, 0 ) +TEST ( irc_line_parse, 0 ) +TEST ( irc_line_build, 0 ) +TEST ( irc_queue, 0 ) +TEST ( irc_conn, 0 ) +TEST ( irc_conn_self_nick, 0 ) +TEST ( irc_net, 0 ) +TEST ( irc_chan_add_offline, 0 ) +TEST ( irc_chan_namreply, 0 ) +TEST ( irc_chan_user_join, 0 ) +TEST ( irc_chan_user_part, 0 ) +TEST ( irc_chan_user_kick, 0 ) +TEST ( irc_chan_self_kick, 0 ) +TEST ( irc_chan_user_nick, 0 ) +TEST ( irc_chan_user_quit, 0 ) +TEST ( irc_chan_CTCP_ACTION, 0 ) +TEST ( irc_chan_privmsg, 0 ) /* * End of list