UnitTestFramework.h

Go to the documentation of this file.

#if !defined(_UNITTESTFRAMEWORK_H_)
#define _UNITTESTFRAMEWORK_H_
 
#include "Standard.h"
#include "Types.h"
#include "Utils.h"
 
#include <string>
#include <vector>
#include <map>
 
// --- Instrumentation-aware time budgets -------------------------------------
// AddressSanitizer/TSan slow execution by roughly 2-4x, which blows every FIXED
// wall-clock deadline in the suite. Two real cases, same root cause but different
// symptoms on different platforms:
//   Linux  - builder_golden exceeded the framework's 30 s per-test cap. It runs
//            three node start/stop cycles, and under ASan each node startup
//            ("PsySpace 'Root' has shut down" -> "Success synchronising ID
//            Manager") costs ~11.8 s versus well under a second bare, so ~35 s of
//            legitimate work hit a 30 s ceiling.
//   macOS  - the same test failed at ~1.2 s on an in-test elapsed-time assertion
//            instead, because that box is fast enough to stay inside the cap.
// Both are budget problems, not defects: the suite is 55/55 bare on both
// platforms with zero memory reports.
//
// Scale time budgets by UT_TIME_SCALE rather than widening constants outright, so
// bare runs keep their original (tight, meaningful) deadlines and only
// instrumented runs get slack. Deliberately NOT applied to product timeouts -
// only to test/framework budgets.
#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
    #define UT_SANITIZED 1
#elif defined(__has_feature)
    #if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer)
        #define UT_SANITIZED 1
    #endif
#endif
#ifdef UT_SANITIZED
    #define UT_TIME_SCALE 5
#else
    #define UT_TIME_SCALE 1
#endif
 
namespace cmlabs {

typedef bool (*UnitTestFunc)();

namespace unittest {

    void progress(int percent, const char* action);
    void progressf(int percent, const char* fmt, ...);

    void metric(const char* name, double value, const char* unit = "", bool higherIsBetter = true);

    void detail(const char* fmt, ...);

    void fail(const char* fmt, ...);

    bool verbose();
 
} // namespace unittest

struct UnitTestMetric {
    std::string name;           
    double      value;          
    std::string unit;           
    bool        higherIsBetter; 
};

struct UnitTestRecord {
    std::string  name;          
    std::string  description;   
    std::string  category;      
    UnitTestFunc func;          
    bool         inDefaultRun;  
 
    // Filled in at run time
    bool         ran;           
    bool         passed;        
    uint64       durationUs;    
    std::string  failReason;    
    std::string  endNote;       
    std::vector<UnitTestMetric> metrics; 
    std::vector<std::string> detailLog;

    UnitTestRecord() : func(NULL), inDefaultRun(true), ran(false), passed(false), durationUs(0) {}
};

class UnitTestRunner {
public:
    static UnitTestRunner& instance();

    void registerTest(const char* name, UnitTestFunc func,
                       const char* description = "", const char* category = "",
                       bool inDefaultRun = true);

    void setVerbose(bool v);
    void setCompareFile(const char* path);
    void setOutputFile(const char* path);
    void setOutputDir(const char* dir);
    void setWriteJSON(bool v);
    void setForkPerTest(bool v);

    bool hasTest(const char* name) const;
    std::vector<std::string> testNames() const;
    void listTests() const;

    bool runAll();
    bool runOne(const char* name);

    void hookProgress(int percent, const char* action);
    void hookMetric(const char* name, double value, const char* unit, bool higherIsBetter);
    void hookDetail(const char* text);
    void hookFail(const char* text);
    bool hookVerbose() const;
 
private:
    UnitTestRunner();

    bool runRecords(std::vector<UnitTestRecord*>& records);
    bool executeRecord(UnitTestRecord* rec);
    void beginTest(UnitTestRecord* rec);
    void finishTest(UnitTestRecord* rec);
    void clearProgressLine();
    void printStatusLine(const UnitTestRecord* rec);
    void printMetricLines(const UnitTestRecord* rec);
    void writeResults(const std::vector<UnitTestRecord*>& records,
                      uint64 totalUs, int passed, int failed);
    void loadComparison();
    bool comparisonValue(const std::string& key, double& out) const;
    static std::string timestampForFilename(uint64 t);
 
    std::vector<UnitTestRecord> tests;  
    UnitTestRecord* current;            
 
    bool   verboseFlag;         
    bool   writeJSONFlag;       
    bool   forkPerTest;         
    bool   isTTY;               
    int    lastProgressLen;     
    std::string progressPath;   
 
    std::string compareFile;    
    std::string outputFile;     
    std::string outputDir;      
 
    bool compareLoaded;         
    std::map<std::string, double> compareMetrics; 
 
    utils::Mutex lock;          
};
 
} // namespace cmlabs
 
#endif // _UNITTESTFRAMEWORK_H_