#!/usr/bin/env python3 from subprocess import run, CompletedProcess from os import remove, makedirs, path from shutil import rmtree from typing import Callable import argparse RESULT_DIR = "test_results" def run_bench( name: str, cleanup: Callable[[], None], success_cond: Callable[[str], bool] ) -> None: """Runs a particular testbench 'name': used in the command `make {name}` 'cleanup': function to delete testbench output of the previous run 'success_cond': takes in the stdout of running a testbench returns whether the run was successful """ print(f"Running make {name}: ", end="") cleanup() proc = run(["make", name], capture_output=True) stdout = proc.stdout.decode("utf-8") stderr = proc.stderr.decode("utf-8") with open(path.join(RESULT_DIR, f'{name.split("/")[-1]}.out'), "w") as fout: fout.write(stdout) with open(path.join(RESULT_DIR, f'{name.split("/")[-1]}.err'), "w") as ferr: ferr.write(stderr) if not success_cond(stdout): print(f"Failed") print("stdout:") print(stdout) print("stderr:") print(stderr) print(f"See {RESULT_DIR} for logs") exit(1) else: print("Passed") def cleanup_isa_tests() -> None: rmtree("sim/isa", ignore_errors=True) def cleanup_c_tests() -> None: rmtree("sim/c_tests", ignore_errors=True) def get_grep_output(proc: CompletedProcess) -> str: stdout = proc.stdout.decode("utf-8").strip() return stdout.split("\n") def isa_test_success_cond(_: str) -> bool: proc_failed = run( 'grep -r -i "failed" sim/isa/*.log', shell=True, capture_output=True ) proc_timout = run( 'grep -r -i "timeout" sim/isa/*.log', shell=True, capture_output=True ) tests_failed = get_grep_output(proc_failed) + get_grep_output(proc_timout) filtered_tests = list( filter(lambda l: ("fence_i" not in l) and l != "", tests_failed) ) if filtered_tests: return False return True def c_test_success_cond(_: str) -> bool: proc_failed = run( 'grep -r -i "failed" sim/c_tests/*.log', shell=True, capture_output=True ) proc_timout = run( 'grep -r -i "timeout" sim/c_tests/*.log', shell=True, capture_output=True ) tests_failed = get_grep_output(proc_failed) + get_grep_output(proc_timout) filtered_tests = list(filter(lambda l: l != "", tests_failed)) if filtered_tests: return False return True def silent_remove_factory(testbench: str, simulator: str) -> Callable[[], None]: def foo(): try: if simulator == "vcs": remove(f"sim/{testbench}.tb") remove(f"sim/{testbench}.vpd") else: remove(f"sim/{testbench}.tbi") remove(f"sim/{testbench}.fst") remove(f"sim/{testbench}.log") except OSError: pass return foo def main(): makedirs(RESULT_DIR, exist_ok=True) parser = argparse.ArgumentParser(description="Run all RTL simulations and perform checks") parser.add_argument("--simulator", type=str, choices=["vcs", "iverilog"], default="vcs", help="The RTL simulator to use") args = parser.parse_args() waveform_suffix = "vpd" if args.simulator == "vcs" else "fst" run_bench( f"sim/cpu_tb.{waveform_suffix}", silent_remove_factory("cpu_tb", args.simulator), lambda stdout: "All tests passed!" in stdout, ) run_bench( f"sim/asm_tb.{waveform_suffix}", silent_remove_factory("asm_tb", args.simulator), lambda stdout: "ALL ASSEMBLY TESTS PASSED!" in stdout, ) run_bench("isa-tests", cleanup_isa_tests, isa_test_success_cond) run_bench("c-tests", cleanup_c_tests, c_test_success_cond) run_bench( f"sim/echo_tb.{waveform_suffix}", silent_remove_factory("echo_tb", args.simulator), lambda stdout: "Test passed!" in stdout, ) run_bench( f"sim/uart_parse_tb.{waveform_suffix}", silent_remove_factory("uart_parse_tb", args.simulator), lambda stdout: "CSR test PASSED! Strings matched." in stdout, ) run_bench( f"sim/bios_tb.{waveform_suffix}", silent_remove_factory("bios_tb", args.simulator), lambda stdout: "BIOS testbench done! Num failed tests: 0" in stdout, ) print("All tests passed!") if __name__ == "__main__": main()