UnitTestFramework.cpp

Go to the documentation of this file.

// ============================================================================
 
#include "UnitTestFramework.h"
#include "PsyTime.h"
#include "jsmn.h"
 
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
 
#ifdef WINDOWS
    #include <io.h>
    #define ISATTY(fd) _isatty(fd)
    #define FILENO(f)  _fileno(f)
#else
    #include <unistd.h>
    #include <dirent.h>
    #include <sys/wait.h>
    #include <signal.h>
    #include <execinfo.h>
    #include <fcntl.h>
    #define ISATTY(fd) isatty(fd)
    #define FILENO(f)  fileno(f)
    #define UT_HAVE_FORK 1
#endif
 
// Hard per-test wall-clock cap when running each test in its own process. A
// test that exceeds this is killed and reported as a timeout (rather than
// hanging the whole suite). Scaled by UT_TIME_SCALE (UnitTestFramework.h) so
// instrumented runs get slack while bare runs keep the tight 30 s ceiling:
// builder_golden runs three node start/stop cycles, and under ASan each startup
// costs ~11.8 s versus well under a second bare, so ~35 s of legitimate work was
// hitting a 30 s cap on Linux.
#define UT_TEST_TIMEOUT_US (30ULL * 1000000ULL * UT_TIME_SCALE)
 
namespace cmlabs {
 
// Best-effort removal of stale CMSDK shared-memory segments left behind by a
// previous test process that was killed (kill -9) mid-operation. Such a process
// can leave a PROCESS_SHARED pthread mutex locked inside a persisted segment,
// which would deadlock the next run that re-opens it. Tests run under system id
// 0, so clearing these test artifacts before a run makes the suite self-healing
// and safely repeatable (e.g. for an agent looping to chase a perf target).
// This assumes no live Psyclone node is using system id 0 on this machine while
// tests run (which is already required by the "another node running" guard).
static void ut_clearSegmentsIn(const char* dir, const char* prefix) {
#ifndef WINDOWS
    DIR* d = opendir(dir);
    if (!d) return;
    size_t plen = strlen(prefix);
    struct dirent* e;
    while ((e = readdir(d)) != NULL) {
        if (strncmp(e->d_name, prefix, plen) == 0) {
            std::string path = std::string(dir) + "/" + e->d_name;
            unlink(path.c_str());
        }
    }
    closedir(d);
#endif
}
 
static void ClearStaleTestSegments() {
#ifndef WINDOWS
    // ⚠️ A test that DELIBERATELY attaches to another process's segment must not run this.
    // The cross-process ring fuzz (PSY_CONC_CHILD) re-execs this binary as a reader: with
    // the cleanup enabled the child unlinked the PARENT's live segment before it could
    // attach, connect() failed in 0.03 ms, and the tier reported "child produced NO read
    // records". That looked like a cross-process defect and was purely this cleanup.
    // Skipping it in the child is safe: the PARENT still clears stale segments on entry,
    // and the child creates nothing of its own.
    const char* attaching = getenv("PSY_CONC_CHILD");
    if (attaching && *attaching)
        return;
    // macOS file-backed segments: /tmp/Psyclone_Shmem_<name>
    ut_clearSegmentsIn("/tmp", "Psyclone_Shmem_");
    // Linux / Cygwin POSIX shm: shm_open("/Shmem_<name>") -> /dev/shm/Shmem_<name>
    ut_clearSegmentsIn("/dev/shm", "Shmem_");
#endif
}
 
#ifdef UT_HAVE_FORK
// ---- Timeout diagnostics for forked tests --------------------------------
// When a test hangs, the parent's only signal is the wall-clock timeout, and
// the child's buffered output is lost when it is killed. To make a hang
// diagnosable we (a) have the child record its current phase to a file on every
// progress() call (read back by the parent on timeout) and (b) on timeout ask
// the child, via SIGUSR1, to dump a native stack trace of the stuck thread to a
// file before we kill it. backtrace_symbols_fd() is async-signal-safe, so the
// handler is safe to run from a signal.
static int ut_traceFd = -1;
static void ut_timeoutTraceHandler(int) {
    if (ut_traceFd >= 0) {
        void* frames[128];
        int n = backtrace(frames, 128);
        backtrace_symbols_fd(frames, n, ut_traceFd);
    }
    _exit(3);
}
static void ut_installTimeoutTraceHandler(const std::string& tracePath) {
    ut_traceFd = open(tracePath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
    signal(SIGUSR1, ut_timeoutTraceHandler);
}
 
// Read back the child's last-recorded phase ("percent\taction") as a short
// human-readable note for the status line.
static std::string ut_readProgressNote(const std::string& path) {
    FILE* f = fopen(path.c_str(), "r");
    if (!f) return "";
    char buf[512];
    size_t r = fread(buf, 1, sizeof(buf) - 1, f);
    fclose(f);
    buf[r] = 0;
    std::string s(buf);
    std::string::size_type tab = s.find('\t');
    if (tab == std::string::npos) return s;
    return s.substr(0, tab) + "% \"" + s.substr(tab + 1) + "\"";
}
 
// Print the captured stack trace (if any) as indented diagnostic lines.
static void ut_printTraceFile(const std::string& path) {
    FILE* f = fopen(path.c_str(), "r");
    if (!f) return;
    char line[1024];
    bool header = false;
    while (fgets(line, sizeof(line), f)) {
        if (!header) { printf("           stack trace where the test hung:\n"); header = true; }
        printf("             %s", line);
    }
    if (header) { printf("\n"); fflush(stdout); }
    fclose(f);
}
#endif
 
// ----------------------------------------------------------------------------
//  Layout constants for consistent output
// ----------------------------------------------------------------------------
#define UT_NAME_WIDTH    28   // width of the test-name column on the status line
#define UT_INDENT        "           "  // 11 spaces, lines up after "[ PASS ] "
#define UT_METRIC_WIDTH  26   // width of the metric-name column on a subline
 
// ----------------------------------------------------------------------------
//  Singleton
// ----------------------------------------------------------------------------
UnitTestRunner& UnitTestRunner::instance() {
    static UnitTestRunner theInstance;
    return theInstance;
}
 
UnitTestRunner::UnitTestRunner()
    : current(NULL), verboseFlag(false), writeJSONFlag(true),
      forkPerTest(true), isTTY(false), lastProgressLen(0), outputDir("."),
      compareLoaded(false) {
    isTTY = ISATTY(FILENO(stdout)) ? true : false;
}
 
// ----------------------------------------------------------------------------
//  Registration & configuration
// ----------------------------------------------------------------------------
void UnitTestRunner::registerTest(const char* name, UnitTestFunc func,
                                  const char* description, const char* category,
                                  bool inDefaultRun) {
    if (!name || !func) return;
    // Replace if a test with this name was already registered.
    for (size_t i = 0; i < tests.size(); i++) {
        if (tests[i].name == name) {
            tests[i].func = func;
            tests[i].description = description ? description : "";
            tests[i].category = category ? category : "";
            tests[i].inDefaultRun = inDefaultRun;
            return;
        }
    }
    UnitTestRecord rec;
    rec.name = name;
    rec.func = func;
    rec.description = description ? description : "";
    rec.category = category ? category : "";
    rec.inDefaultRun = inDefaultRun;
    tests.push_back(rec);
}
 
void UnitTestRunner::setVerbose(bool v)             { verboseFlag = v; }
void UnitTestRunner::setCompareFile(const char* p)  { if (p) compareFile = p; }
void UnitTestRunner::setOutputFile(const char* p)   { if (p) outputFile = p; }
void UnitTestRunner::setOutputDir(const char* d)    { if (d) outputDir = d; }
void UnitTestRunner::setWriteJSON(bool v)           { writeJSONFlag = v; }
void UnitTestRunner::setForkPerTest(bool v)         { forkPerTest = v; }
 
bool UnitTestRunner::hasTest(const char* name) const {
    if (!name) return false;
    for (size_t i = 0; i < tests.size(); i++)
        if (stricmp(tests[i].name.c_str(), name) == 0) return true;
    return false;
}
 
std::vector<std::string> UnitTestRunner::testNames() const {
    std::vector<std::string> names;
    for (size_t i = 0; i < tests.size(); i++) names.push_back(tests[i].name);
    return names;
}
 
void UnitTestRunner::listTests() const {
    printf("\nAvailable CMSDK unit tests (%u):\n\n", (uint32)tests.size());
    std::string lastCat = "\x01";
    for (size_t i = 0; i < tests.size(); i++) {
        if (tests[i].category != lastCat) {
            lastCat = tests[i].category;
            printf("  [%s]\n", lastCat.empty() ? "general" : lastCat.c_str());
        }
        printf("    %-22s %s\n", tests[i].name.c_str(), tests[i].description.c_str());
    }
    printf("\nRun all with: test=cmsdk    Run one with: test=<name>\n\n");
}
 
// ----------------------------------------------------------------------------
//  Hooks used by the unittest:: API for the currently running test
// ----------------------------------------------------------------------------
bool UnitTestRunner::hookVerbose() const { return verboseFlag; }
 
void UnitTestRunner::hookProgress(int percent, const char* action) {
    if (!current) return;
    if (percent < 0)   percent = 0;
    if (percent > 100) percent = 100;
 
    // Record the current phase to a file (forked child only) so that, if the test
    // hangs and is killed by the timeout, the parent can still report which phase
    // it was in. progress() is called only a handful of times per test, so this is
    // cheap, and it happens regardless of whether stdout is a terminal.
    if (!progressPath.empty()) {
        FILE* pf = fopen(progressPath.c_str(), "w");
        if (pf) { fprintf(pf, "%d\t%s", percent, action ? action : ""); fclose(pf); }
    }
 
    if (!isTTY) return;            // keep piped/redirected logs clean
 
    char buf[512];
    int n = snprintf(buf, sizeof(buf), "  [%3d%%] %-*.*s %s",
                     percent, UT_NAME_WIDTH, UT_NAME_WIDTH,
                     current->name.c_str(), action ? action : "");
    if (n < 0) return;
 
    // Pad to clear any longer text drawn previously.
    int pad = lastProgressLen - n;
    fputc('\r', stdout);
    fputs(buf, stdout);
    for (int i = 0; i < pad; i++) fputc(' ', stdout);
    fputc('\r', stdout);
    fflush(stdout);
    lastProgressLen = (n > lastProgressLen) ? n : lastProgressLen;
    // (we keep the max so the next pad clears the longest line)
    lastProgressLen = n;
}
 
void UnitTestRunner::clearProgressLine() {
    if (!isTTY || lastProgressLen <= 0) return;
    fputc('\r', stdout);
    for (int i = 0; i < lastProgressLen; i++) fputc(' ', stdout);
    fputc('\r', stdout);
    fflush(stdout);
    lastProgressLen = 0;
}
 
void UnitTestRunner::hookMetric(const char* name, double value, const char* unit, bool higherIsBetter) {
    if (!current || !name) return;
    lock.enter();
    UnitTestMetric m;
    m.name = name;
    m.value = value;
    m.unit = unit ? unit : "";
    m.higherIsBetter = higherIsBetter;
    current->metrics.push_back(m);
    lock.leave();
}
 
void UnitTestRunner::hookDetail(const char* text) {
    if (!current) return;
    // ALWAYS record, even when not verbose: if this test goes on to fail we print the
    // buffer with the reason (see hookFail). Previously detail() was discarded unless
    // verbose was set, so a rare intermittent failure gave a one-line reason and no
    // diagnostics at all - and you cannot re-run a 2%-rate flake on demand to get
    // them. Bounded ring so a chatty passing test cannot grow without limit.
    static const size_t UT_DETAIL_KEEP = 64;
    current->detailLog.push_back(text ? text : "");
    if (current->detailLog.size() > UT_DETAIL_KEEP)
        current->detailLog.erase(current->detailLog.begin());
    // Only ECHO live when verbose, so normal runs stay quiet.
    if (!verboseFlag) return;
    clearProgressLine();
    printf(UT_INDENT "%s\n", text ? text : "");
    fflush(stdout);
}
 
void UnitTestRunner::hookFail(const char* text) {
    if (!current) return;
    std::string reason = text ? text : "";
    // Trim trailing whitespace/newlines so the reason renders on one line.
    while (!reason.empty() &&
           (reason[reason.size() - 1] == '\n' || reason[reason.size() - 1] == '\r' ||
            reason[reason.size() - 1] == ' '  || reason[reason.size() - 1] == '\t'))
        reason.erase(reason.size() - 1);
    current->failReason = reason;
 
    // Print the reason RIGHT NOW (always, regardless of verbose). The status
    // line is printed later by the parent process, and if the test goes on to
    // hang or crash before then, the per-test timeout/crash note would be all
    // that survives - so emit the reason immediately while we still can.
    if (!reason.empty()) {
        clearProgressLine();
        // Diagnostics FIRST, then the reason, so the reason stays the last line and
        // existing log-scrapers that grep for "reason:" are unaffected. Only when not
        // verbose - in verbose mode these were already echoed live and repeating them
        // would just duplicate output.
        if (!verboseFlag && !current->detailLog.empty()) {
            printf(UT_INDENT "--- diagnostics (%u detail line(s) captured before failure) ---\n",
                (uint32)current->detailLog.size());
            for (size_t i = 0; i < current->detailLog.size(); i++)
                printf(UT_INDENT "  %s\n", current->detailLog[i].c_str());
        }
        printf(UT_INDENT "reason: %s\n", reason.c_str());
        fflush(stdout);
    }
}
 
// ----------------------------------------------------------------------------
//  Output of results
// ----------------------------------------------------------------------------
void UnitTestRunner::printStatusLine(const UnitTestRecord* rec) {
    const char* tag = rec->passed ? "PASS" : "FAIL";
    double ms = rec->durationUs / 1000.0;
    // The test's own fail reason was already printed immediately by hookFail();
    // here we only append the framework outcome note (timed out / crashed / no
    // test function) so a hung or crashed test still shows why on the status line.
    if (rec->passed || rec->endNote.empty())
        printf("[ %s ] %-*s %10.2f ms\n", tag, UT_NAME_WIDTH, rec->name.c_str(), ms);
    else
        printf("[ %s ] %-*s %10.2f ms   %s\n", tag, UT_NAME_WIDTH, rec->name.c_str(), ms,
               rec->endNote.c_str());
    fflush(stdout);
}
 
void UnitTestRunner::printMetricLines(const UnitTestRecord* rec) {
    for (size_t i = 0; i < rec->metrics.size(); i++) {
        const UnitTestMetric& m = rec->metrics[i];
        char line[512];
        snprintf(line, sizeof(line), UT_INDENT "%-*s %14.3f %-8s",
                 UT_METRIC_WIDTH, m.name.c_str(), m.value, m.unit.c_str());
        printf("%s", line);
 
        // Comparison against a previous run
        double prev = 0.0;
        std::string key = rec->name + "/" + m.name;
        if (compareLoaded && comparisonValue(key, prev)) {
            if (prev != 0.0) {
                double pct = (m.value - prev) / prev * 100.0;
                const char* dir;
                bool improved = m.higherIsBetter ? (m.value > prev) : (m.value < prev);
                bool worse    = m.higherIsBetter ? (m.value < prev) : (m.value > prev);
                dir = improved ? "better" : (worse ? "worse" : "same");
                printf("   prev %.3f   %+.1f%% (%s)", prev, pct, dir);
            } else {
                printf("   prev %.3f", prev);
            }
        }
        printf("\n");
    }
    fflush(stdout);
}
 
// ----------------------------------------------------------------------------
//  Running
// ----------------------------------------------------------------------------
void UnitTestRunner::beginTest(UnitTestRecord* rec) {
    current = rec;
    lastProgressLen = 0;
    rec->ran = true;
    rec->passed = false;
    rec->metrics.clear();
    rec->failReason.clear();
    rec->endNote.clear();
}
 
void UnitTestRunner::finishTest(UnitTestRecord* rec) {
    clearProgressLine();
    printStatusLine(rec);
    printMetricLines(rec);
    current = NULL;
}
 
// ---- child-result serialisation (used by the forked-execution path) --------
// The child runs one test and records its metrics / fail reason into its own
// copy of the record; it then serialises them to a small file the parent reads
// back (fork copies memory, so the parent cannot see the child's heap directly).
static std::string ut_escape_field(const std::string& s) {
    std::string out;
    for (size_t i = 0; i < s.size(); i++) {
        char c = s[i];
        if (c == '\n' || c == '\r' || c == '\t') out += ' ';
        else out += c;
    }
    return out;
}
 
static void ut_writeChildResult(const std::string& path, const UnitTestRecord* rec) {
    std::string s;
    s += "fail\t" + ut_escape_field(rec->failReason) + "\n";
    for (size_t i = 0; i < rec->metrics.size(); i++) {
        const UnitTestMetric& m = rec->metrics[i];
        char num[64];
        snprintf(num, sizeof(num), "%.10g", m.value);
        s += std::string("metric\t") + ut_escape_field(m.name) + "\t" + num + "\t" +
             ut_escape_field(m.unit) + "\t" + (m.higherIsBetter ? "1" : "0") + "\n";
    }
    utils::WriteAFile(path.c_str(), s.c_str(), (uint32)s.size());
}
 
static void ut_readChildResult(const std::string& path, UnitTestRecord* rec) {
    std::string data = utils::ReadAFileString(path);
    if (data.empty()) return;
    size_t pos = 0;
    while (pos < data.size()) {
        size_t eol = data.find('\n', pos);
        if (eol == std::string::npos) eol = data.size();
        std::string line = data.substr(pos, eol - pos);
        pos = eol + 1;
        if (line.empty()) continue;
        // split on tabs
        std::vector<std::string> f;
        size_t p = 0;
        while (true) {
            size_t t = line.find('\t', p);
            if (t == std::string::npos) { f.push_back(line.substr(p)); break; }
            f.push_back(line.substr(p, t - p));
            p = t + 1;
        }
        if (f[0] == "fail" && f.size() >= 2) {
            rec->failReason = f[1];
        } else if (f[0] == "metric" && f.size() >= 5) {
            UnitTestMetric m;
            m.name = f[1];
            m.value = utils::Ascii2Float64(f[2].c_str());
            m.unit = f[3];
            m.higherIsBetter = (f[4] == "1");
            rec->metrics.push_back(m);
        }
    }
}
 
// Run a single test in its own process so a crash or hang in the test cannot
// take down the whole suite, and so global/shared state (thread manager,
// shared memory, singletons) never leaks between tests. Returns true on pass.
// On non-fork platforms falls back to running inline.
bool UnitTestRunner::executeRecord(UnitTestRecord* rec) {
    // Clean any stale shared-memory segments left by a previously crashed/killed
    // test before starting this one, so it always begins from a clean slate.
    ClearStaleTestSegments();
 
#ifdef UT_HAVE_FORK
    if (forkPerTest) {
        std::string dir = outputDir.size() ? outputDir : ".";
        std::string resultPath = dir + "/.cmsdk_result_" + rec->name;
        std::string progPath   = resultPath + ".progress";
        std::string tracePath  = resultPath + ".trace";
        unlink(resultPath.c_str());
        unlink(progPath.c_str());
        unlink(tracePath.c_str());
 
        fflush(stdout);
        fflush(stderr);
        uint64 t0 = GetTimeNow();
        pid_t pid = fork();
        if (pid < 0) {
            // fork failed - run inline as a fallback
            bool ok = rec->func ? rec->func() : false;
            rec->durationUs = GetTimeNow() - t0;
            return ok;
        }
        if (pid == 0) {
            // ---- child ----
            current = rec;
            progressPath = progPath;                      // record current phase for timeout diagnostics
            ut_installTimeoutTraceHandler(tracePath);     // SIGUSR1 -> dump stack trace, then exit
            lastProgressLen = 0;
            if (isTTY) hookProgress(0, "starting...");
            bool ok = false;
            if (rec->func) ok = rec->func();
            else rec->failReason = "no test function registered";
            clearProgressLine();   // erase our progress line so the parent's status line is clean
            ut_writeChildResult(resultPath, rec);
            fflush(stdout);
            _exit(ok ? 0 : 1);
        }
 
        // ---- parent ----
        int status = 0;
        bool timedOut = false;
        while (true) {
            pid_t r = waitpid(pid, &status, WNOHANG);
            if (r == pid) break;
            if (r < 0) break;
            if (GetTimeNow() - t0 > UT_TEST_TIMEOUT_US) {
                // Ask the stuck child to dump a stack trace of where it hung, give
                // it a brief grace period to do so, then force-kill if necessary.
                kill(pid, SIGUSR1);
                bool reaped = false;
                uint64 graceStart = GetTimeNow();
                while (GetTimeNow() - graceStart < 2000000ULL) {   // up to 2s
                    pid_t rr = waitpid(pid, &status, WNOHANG);
                    if (rr == pid || rr < 0) { reaped = true; break; }
                    utils::Sleep(15);
                }
                if (!reaped) {
                    kill(pid, SIGKILL);
                    waitpid(pid, &status, 0);
                }
                timedOut = true;
                break;
            }
            utils::Sleep(15);
        }
        rec->durationUs = GetTimeNow() - t0;
 
        bool ok = false;
        if (timedOut) {
            // Report which phase the test was in when it hung, and print the stack
            // trace the child captured (if any) so the hang is diagnosable.
            std::string note = utils::StringFormat("timed out after %llus", UT_TEST_TIMEOUT_US / 1000000ULL);
            std::string lastPhase = ut_readProgressNote(progPath);
            if (!lastPhase.empty()) note += " (last phase: " + lastPhase + ")";
            rec->endNote = note;
            ut_readChildResult(resultPath, rec);   // recover any reason/metrics gathered pre-hang
            ut_printTraceFile(tracePath);
        } else if (WIFSIGNALED(status)) {
            rec->endNote = utils::StringFormat("crashed (signal %d)", WTERMSIG(status));
            ut_readChildResult(resultPath, rec);  // recover the reason/metrics gathered pre-crash
        } else {
            ut_readChildResult(resultPath, rec);
            ok = (WIFEXITED(status) && WEXITSTATUS(status) == 0);
        }
        unlink(resultPath.c_str());
        unlink(progPath.c_str());
        unlink(tracePath.c_str());
        return ok;
    }
#endif
 
    // Inline fallback (no fork available, or fork disabled).
    current = rec;
    lastProgressLen = 0;
    if (isTTY) hookProgress(0, "starting...");
    uint64 t0 = GetTimeNow();
    bool ok = rec->func ? rec->func() : false;
    rec->durationUs = GetTimeNow() - t0;
    if (!rec->func) rec->endNote = "no test function registered";
    return ok;
}
 
bool UnitTestRunner::runRecords(std::vector<UnitTestRecord*>& records) {
    // Start from a clean shared-memory slate so a previously killed run can't
    // deadlock this one on a stale process-shared mutex.
    ClearStaleTestSegments();
 
    if (compareFile.size() && !compareLoaded) loadComparison();
 
    // In non-verbose mode, silence the logging system's console output so the
    // only thing on stdout is the consistent status/metric lines. Tests still
    // report failures via unittest::fail(). Verbose mode leaves logging on.
    bool savedLogToStdOut = true;
    if (!verboseFlag) {
        LogSystem::SetLogReceiver(NULL);            // ensures the singleton exists
        if (LogSystem::LogSingleton) {
            savedLogToStdOut = LogSystem::LogSingleton->printToStdOut;
            LogSystem::LogSingleton->printToStdOut = false;
        }
    }
 
    uint64 startAll = GetTimeNow();
    int passed = 0, failed = 0;
 
    for (size_t i = 0; i < records.size(); i++) {
        UnitTestRecord* rec = records[i];
        beginTest(rec);
        bool ok = executeRecord(rec);
        rec->passed = ok;
        finishTest(rec);
        if (ok) passed++; else failed++;
    }
 
    uint64 totalUs = GetTimeNow() - startAll;
 
    printf("\n----------------------------------------------------------------\n");
    printf("CMSDK tests: %d passed, %d failed, %d total in %.2f ms\n",
           passed, failed, (int)records.size(), totalUs / 1000.0);
 
    if (writeJSONFlag) writeResults(records, totalUs, passed, failed);
    printf("\n");
 
    if (!verboseFlag && LogSystem::LogSingleton)
        LogSystem::LogSingleton->printToStdOut = savedLogToStdOut;
 
    return failed == 0;
}
 
bool UnitTestRunner::runAll() {
    std::vector<UnitTestRecord*> records;
    int excluded = 0;
    for (size_t i = 0; i < tests.size(); i++) {
        if (tests[i].inDefaultRun) records.push_back(&tests[i]);
        else excluded++;
    }
    if (records.empty()) {
        printf("No CMSDK unit tests are registered.\n");
        return false;
    }
    printf("\nRunning %u CMSDK unit tests", (uint32)records.size());
    if (excluded)
        printf(" (%d excluded from the default run; run them by name)", excluded);
    printf("...\n\n");
    return runRecords(records);
}
 
bool UnitTestRunner::runOne(const char* name) {
    if (!name) return false;
    UnitTestRecord* found = NULL;
    for (size_t i = 0; i < tests.size(); i++) {
        if (stricmp(tests[i].name.c_str(), name) == 0) { found = &tests[i]; break; }
    }
    if (!found) {
        printf("No CMSDK unit test named '%s'.\n", name);
        listTests();
        return false;
    }
    std::vector<UnitTestRecord*> records;
    records.push_back(found);
    printf("\nRunning CMSDK unit test '%s'...\n\n", found->name.c_str());
    return runRecords(records);
}
 
// ----------------------------------------------------------------------------
//  Comparison file loading
// ----------------------------------------------------------------------------
bool UnitTestRunner::comparisonValue(const std::string& key, double& out) const {
    std::map<std::string, double>::const_iterator it = compareMetrics.find(key);
    if (it == compareMetrics.end()) return false;
    out = it->second;
    return true;
}
 
void UnitTestRunner::loadComparison() {
    compareLoaded = false;
    compareMetrics.clear();
 
    std::string json = utils::ReadAFileString(compareFile);
    if (json.empty()) {
        printf("Warning: could not read comparison file '%s'\n", compareFile.c_str());
        return;
    }
 
    // Count tokens first, then parse.
    jsmn_parser parser;
    jsmn_init(&parser);
    int needed = jsmn_parse(&parser, json.c_str(), json.size(), NULL, 0);
    if (needed <= 0) {
        printf("Warning: comparison file '%s' is not valid JSON\n", compareFile.c_str());
        return;
    }
    jsmntok_t* tokens = new jsmntok_t[needed + 1];
    jsmn_init(&parser);
    int count = jsmn_parse(&parser, json.c_str(), json.size(), tokens, needed + 1);
    if (count <= 0) {
        delete[] tokens;
        printf("Warning: failed to parse comparison file '%s'\n", compareFile.c_str());
        return;
    }
 
    // Pull every "test/metric" -> number pair from the flat section. Flat keys
    // are the only keys containing '/', so they are unambiguous.
    for (int n = 0; n < count - 1; n++) {
        if (tokens[n].type != JSMN_STRING) continue;
        int klen = tokens[n].end - tokens[n].start;
        if (klen <= 0) continue;
        const char* kstart = json.c_str() + tokens[n].start;
        bool hasSlash = false;
        for (int c = 0; c < klen; c++) { if (kstart[c] == '/') { hasSlash = true; break; } }
        if (!hasSlash) continue;
        if (tokens[n + 1].type != JSMN_PRIMITIVE) continue;
        std::string key(kstart, klen);
        std::string val(json.c_str() + tokens[n + 1].start, tokens[n + 1].end - tokens[n + 1].start);
        compareMetrics[key] = utils::Ascii2Float64(val.c_str());
    }
 
    delete[] tokens;
    compareLoaded = true;
    printf("Comparing against '%s' (%u recorded metrics)\n\n",
           compareFile.c_str(), (uint32)compareMetrics.size());
}
 
// ----------------------------------------------------------------------------
//  Results JSON writing
// ----------------------------------------------------------------------------
static std::string jsonEscape(const std::string& s) {
    std::string out;
    for (size_t i = 0; i < s.size(); i++) {
        char c = s[i];
        switch (c) {
            case '\"': out += "\\\""; break;
            case '\\': out += "\\\\"; break;
            case '\n': out += "\\n";  break;
            case '\r': out += "\\r";  break;
            case '\t': out += "\\t";  break;
            default:   out += c;      break;
        }
    }
    return out;
}
 
std::string UnitTestRunner::timestampForFilename(uint64 t) {
    struct PsyDateAndTime d = GetDateAndTime(t);
    char buf[40];   // worst-case uint16 fields (see writeResults) => keep truncation-warning-free
    snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u",
             d.year, d.mon, d.day, d.hour, d.min, d.sec);
    return std::string(buf);
}
 
void UnitTestRunner::writeResults(const std::vector<UnitTestRecord*>& records,
                                  uint64 totalUs, int passed, int failed) {
    uint64 now = GetTimeNow();
    struct PsyDateAndTime d = GetDateAndTime(now);
    // Sized for the theoretical worst case of the format below: the calendar
    // fields are uint16, so the compiler must assume up to 5 digits each
    // ("%04u-%02u-%02u %02u:%02u:%02u" => up to 35 chars + NUL). 40 keeps
    // -Wformat-truncation quiet without relying on the values staying small.
    char dt[40];
    snprintf(dt, sizeof(dt), "%04u-%02u-%02u %02u:%02u:%02u",
             d.year, d.mon, d.day, d.hour, d.min, d.sec);
 
    const char* platform =
#if defined(WINDOWS)
        "win"
#elif defined(DARWIN) || defined(OSX) || defined(__APPLE__)
        "macosx"
#elif defined(LINUX) || defined(__linux)
        "linux"
#else
        "unknown"
#endif
#if defined(ARCH_64)
        "64"
#else
        "32"
#endif
        ;
 
    std::string json;
    json += "{\n";
    json += "  \"meta\": {\n";
    json += utils::StringFormat("    \"version\": 1,\n");
    json += utils::StringFormat("    \"timestamp_us\": %llu,\n", (unsigned long long)now);
    json += utils::StringFormat("    \"datetime\": \"%s\",\n", dt);
    json += utils::StringFormat("    \"platform\": \"%s\",\n", platform);
    json += utils::StringFormat("    \"total\": %d,\n", (int)records.size());
    json += utils::StringFormat("    \"passed\": %d,\n", passed);
    json += utils::StringFormat("    \"failed\": %d,\n", failed);
    json += utils::StringFormat("    \"duration_us\": %llu\n", (unsigned long long)totalUs);
    json += "  },\n";
 
    // Human-readable structured section
    json += "  \"tests\": {\n";
    for (size_t i = 0; i < records.size(); i++) {
        const UnitTestRecord* rec = records[i];
        json += utils::StringFormat("    \"%s\": {\n", jsonEscape(rec->name).c_str());
        json += utils::StringFormat("      \"status\": \"%s\",\n", rec->passed ? "pass" : "fail");
        json += utils::StringFormat("      \"duration_us\": %llu,\n", (unsigned long long)rec->durationUs);
        json += "      \"metrics\": {\n";
        for (size_t j = 0; j < rec->metrics.size(); j++) {
            const UnitTestMetric& m = rec->metrics[j];
            json += utils::StringFormat(
                "        \"%s\": { \"value\": %.6f, \"unit\": \"%s\", \"higher_is_better\": %s }%s\n",
                jsonEscape(m.name).c_str(), m.value, jsonEscape(m.unit).c_str(),
                m.higherIsBetter ? "true" : "false",
                (j + 1 < rec->metrics.size()) ? "," : "");
        }
        json += "      }\n";
        json += utils::StringFormat("    }%s\n", (i + 1 < records.size()) ? "," : "");
    }
    json += "  },\n";
 
    // Flat machine-comparison section ("test/metric" -> number)
    json += "  \"flat\": {\n";
    std::vector<std::string> flat;
    for (size_t i = 0; i < records.size(); i++) {
        const UnitTestRecord* rec = records[i];
        flat.push_back(utils::StringFormat("    \"%s/__duration_us\": %llu",
            jsonEscape(rec->name).c_str(), (unsigned long long)rec->durationUs));
        for (size_t j = 0; j < rec->metrics.size(); j++) {
            const UnitTestMetric& m = rec->metrics[j];
            flat.push_back(utils::StringFormat("    \"%s/%s\": %.6f",
                jsonEscape(rec->name).c_str(), jsonEscape(m.name).c_str(), m.value));
        }
    }
    for (size_t i = 0; i < flat.size(); i++)
        json += flat[i] + ((i + 1 < flat.size()) ? ",\n" : "\n");
    json += "  }\n";
    json += "}\n";
 
    // Decide output paths.
    std::vector<std::string> paths;
    if (outputFile.size()) {
        paths.push_back(outputFile);
    } else {
        std::string dir = outputDir.size() ? outputDir : ".";
        paths.push_back(dir + "/cmsdk_perf_" + timestampForFilename(now) + ".json");
        paths.push_back(dir + "/cmsdk_perf_latest.json");
    }
 
    for (size_t i = 0; i < paths.size(); i++) {
        if (utils::WriteAFile(paths[i].c_str(), json.c_str(), (uint32)json.size()))
            printf("Performance data written to %s\n", paths[i].c_str());
        else
            printf("Warning: could not write performance data to %s\n", paths[i].c_str());
    }
}
 
// ----------------------------------------------------------------------------
//  unittest:: free function API
// ----------------------------------------------------------------------------
namespace unittest {
 
// Format a va_list into a std::string using CMSDK's portable formatter
// (handles the snprintf/_vsnprintf_s differences across MSVC/Cygwin/POSIX).
static std::string ut_vformat(const char* fmt, va_list args) {
    uint32 sz = 0;
    char* s = utils::StringFormatVA(sz, fmt, args);
    std::string out = s ? s : "";
    if (s) free(s);
    return out;
}
 
void progress(int percent, const char* action) {
    UnitTestRunner::instance().hookProgress(percent, action);
}
 
void progressf(int percent, const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    std::string s = ut_vformat(fmt, args);
    va_end(args);
    UnitTestRunner::instance().hookProgress(percent, s.c_str());
}
 
void metric(const char* name, double value, const char* unit, bool higherIsBetter) {
    UnitTestRunner::instance().hookMetric(name, value, unit, higherIsBetter);
}
 
void detail(const char* fmt, ...) {
    // NO verbose gate here any more: hookDetail() records unconditionally (so a
    // failing test can print its diagnostics) and echoes live only when verbose.
    va_list args;
    va_start(args, fmt);
    std::string s = ut_vformat(fmt, args);
    va_end(args);
    UnitTestRunner::instance().hookDetail(s.c_str());
}
 
void fail(const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    std::string s = ut_vformat(fmt, args);
    va_end(args);
    UnitTestRunner::instance().hookFail(s.c_str());
}
 
bool verbose() {
    return UnitTestRunner::instance().hookVerbose();
}
 
} // namespace unittest
 
} // namespace cmlabs