FPGA-RISC-V-CPU / hardware / run_all_sims
run_all_sims
Raw
#!/usr/bin/env python3

from subprocess import run, CompletedProcess
from os import remove, makedirs, path
from shutil import rmtree
from typing import Callable
import argparse

RESULT_DIR = "test_results"


def run_bench(
    name: str, cleanup: Callable[[], None], success_cond: Callable[[str], bool]
) -> None:
    """Runs a particular testbench

    'name': used in the command `make {name}`
    'cleanup': function to delete testbench output of the previous run
    'success_cond': takes in the stdout of running a testbench
                    returns whether the run was successful
    """
    print(f"Running make {name}: ", end="")
    cleanup()
    proc = run(["make", name], capture_output=True)
    stdout = proc.stdout.decode("utf-8")
    stderr = proc.stderr.decode("utf-8")
    with open(path.join(RESULT_DIR, f'{name.split("/")[-1]}.out'), "w") as fout:
        fout.write(stdout)
    with open(path.join(RESULT_DIR, f'{name.split("/")[-1]}.err'), "w") as ferr:
        ferr.write(stderr)
    if not success_cond(stdout):
        print(f"Failed")
        print("stdout:")
        print(stdout)
        print("stderr:")
        print(stderr)
        print(f"See {RESULT_DIR} for logs")
        exit(1)
    else:
        print("Passed")


def cleanup_isa_tests() -> None:
    rmtree("sim/isa", ignore_errors=True)


def cleanup_c_tests() -> None:
    rmtree("sim/c_tests", ignore_errors=True)


def get_grep_output(proc: CompletedProcess) -> str:
    stdout = proc.stdout.decode("utf-8").strip()
    return stdout.split("\n")


def isa_test_success_cond(_: str) -> bool:
    proc_failed = run(
        'grep -r -i "failed" sim/isa/*.log', shell=True, capture_output=True
    )
    proc_timout = run(
        'grep -r -i "timeout" sim/isa/*.log', shell=True, capture_output=True
    )
    tests_failed = get_grep_output(proc_failed) + get_grep_output(proc_timout)
    filtered_tests = list(
        filter(lambda l: ("fence_i" not in l) and l != "", tests_failed)
    )
    if filtered_tests:
        return False
    return True


def c_test_success_cond(_: str) -> bool:
    proc_failed = run(
        'grep -r -i "failed" sim/c_tests/*.log', shell=True, capture_output=True
    )
    proc_timout = run(
        'grep -r -i "timeout" sim/c_tests/*.log', shell=True, capture_output=True
    )
    tests_failed = get_grep_output(proc_failed) + get_grep_output(proc_timout)
    filtered_tests = list(filter(lambda l: l != "", tests_failed))
    if filtered_tests:
        return False
    return True


def silent_remove_factory(testbench: str, simulator: str) -> Callable[[], None]:
    def foo():
        try:
            if simulator == "vcs":
                remove(f"sim/{testbench}.tb")
                remove(f"sim/{testbench}.vpd")
            else:
                remove(f"sim/{testbench}.tbi")
                remove(f"sim/{testbench}.fst")
            remove(f"sim/{testbench}.log")
        except OSError:
            pass

    return foo


def main():
    makedirs(RESULT_DIR, exist_ok=True)
    parser = argparse.ArgumentParser(description="Run all RTL simulations and perform checks")
    parser.add_argument("--simulator", type=str, choices=["vcs", "iverilog"], default="vcs", help="The RTL simulator to use")
    args = parser.parse_args()
    waveform_suffix = "vpd" if args.simulator == "vcs" else "fst"
    run_bench(
        f"sim/cpu_tb.{waveform_suffix}",
        silent_remove_factory("cpu_tb", args.simulator),
        lambda stdout: "All tests passed!" in stdout,
    )
    run_bench(
        f"sim/asm_tb.{waveform_suffix}",
        silent_remove_factory("asm_tb", args.simulator),
        lambda stdout: "ALL ASSEMBLY TESTS PASSED!" in stdout,
    )
    run_bench("isa-tests", cleanup_isa_tests, isa_test_success_cond)
    run_bench("c-tests", cleanup_c_tests, c_test_success_cond)
    run_bench(
        f"sim/echo_tb.{waveform_suffix}",
        silent_remove_factory("echo_tb", args.simulator),
        lambda stdout: "Test passed!" in stdout,
    )
    run_bench(
        f"sim/uart_parse_tb.{waveform_suffix}",
        silent_remove_factory("uart_parse_tb", args.simulator),
        lambda stdout: "CSR test PASSED! Strings matched." in stdout,
    )
    run_bench(
        f"sim/bios_tb.{waveform_suffix}",
        silent_remove_factory("bios_tb", args.simulator),
        lambda stdout: "BIOS testbench done! Num failed tests:          0" in stdout,
    )
    print("All tests passed!")


if __name__ == "__main__":
    main()