import pandas as pd
model_files = {
"LogisticRegression": "../results/logreg_5fold_threshold0.50_by_tier.csv",
"RandomForest": "../results/rf_5fold_threshold0.50_by_tier.csv",
"LightGBM": "../results/lightgbm_5fold_threshold0.50_by_tier.csv",
"XGBoost": "../results/xgb_5fold_threshold0.50_by_tier.csv",
}
metrics_keep = ["Accuracy", "Sensitivity", "Specificity", "PPV", "NPV", "AUC", "F1", "F2"]
tier_to_set_label = {
"Tier_1_Basic": "Tier 1 (6 features)",
"Tier_2_Clinical": "Tier 2 (10 features)",
"Tier_3_Personalized": "Tier 3 (24 features)",
}
# tier_feature_counts = {"Tier_1_Basic": 6, "Tier_2_Clinical": 10, "Tier_3_Personalized": 24}
tier_feature_counts = None
def load_model_means(csv_path: str, model_name: str) -> pd.DataFrame:
df = pd.read_csv(csv_path)
# keeping only needed metrics
df = df[df["Metric"].isin(metrics_keep)].copy()
# pivot: rows=tier, cols=Metric, values=Mean
wide = df.pivot_table(index="Tier", columns="Metric", values="Mean", aggfunc="first").reset_index()
wide.insert(1, "Model", model_name)
return wide
all_models = []
for model_name, path in model_files.items():
one = load_model_means(path, model_name)
all_models.append(one)
combined = pd.concat(all_models, ignore_index=True)
def format_set(tier_name: str) -> str:
base = tier_to_set_label.get(tier_name, tier_name)
if isinstance(tier_feature_counts, dict) and tier_name in tier_feature_counts:
return f"{base} ({tier_feature_counts[tier_name]} features)"
return base
combined.insert(0, "Set", combined["Tier"].map(format_set))
final_cols = ["Set", "Model"] + metrics_keep
final_table = combined[final_cols].copy()
set_order = [format_set(k) for k in ["Tier_1_Basic", "Tier_2_Clinical", "Tier_3_Personalized"]]
model_order = ["LogisticRegression", "RandomForest", "LightGBM", "XGBoost"]
final_table["Set"] = pd.Categorical(final_table["Set"], categories=set_order, ordered=True)
final_table["Model"] = pd.Categorical(final_table["Model"], categories=model_order, ordered=True)
final_table = final_table.sort_values(["Set", "Model"]).reset_index(drop=True)
final_table[metrics_keep] = final_table[metrics_keep].round(3)
final_table
/home/shezin/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).
from pandas.core.computation.check import NUMEXPR_INSTALLED
| Metric |
Set |
Model |
Accuracy |
Sensitivity |
Specificity |
PPV |
NPV |
AUC |
F1 |
F2 |
| 0 |
Tier 1 (6 features) |
LogisticRegression |
0.537 |
0.768 |
0.507 |
0.167 |
0.944 |
0.696 |
0.275 |
0.447 |
| 1 |
Tier 1 (6 features) |
RandomForest |
0.660 |
0.624 |
0.665 |
0.194 |
0.932 |
0.708 |
0.296 |
0.432 |
| 2 |
Tier 1 (6 features) |
LightGBM |
0.647 |
0.638 |
0.648 |
0.190 |
0.933 |
0.705 |
0.292 |
0.433 |
| 3 |
Tier 1 (6 features) |
XGBoost |
0.577 |
0.728 |
0.558 |
0.176 |
0.941 |
0.707 |
0.283 |
0.446 |
| 4 |
Tier 2 (10 features) |
LogisticRegression |
0.821 |
0.914 |
0.809 |
0.382 |
0.986 |
0.925 |
0.539 |
0.715 |
| 5 |
Tier 2 (10 features) |
RandomForest |
0.831 |
0.901 |
0.823 |
0.396 |
0.985 |
0.927 |
0.550 |
0.718 |
| 6 |
Tier 2 (10 features) |
LightGBM |
0.843 |
0.885 |
0.837 |
0.413 |
0.983 |
0.928 |
0.563 |
0.721 |
| 7 |
Tier 2 (10 features) |
XGBoost |
0.831 |
0.906 |
0.821 |
0.395 |
0.985 |
0.928 |
0.550 |
0.720 |
| 8 |
Tier 3 (24 features) |
LogisticRegression |
0.831 |
0.911 |
0.820 |
0.396 |
0.986 |
0.932 |
0.552 |
0.723 |
| 9 |
Tier 3 (24 features) |
RandomForest |
0.838 |
0.899 |
0.831 |
0.407 |
0.985 |
0.932 |
0.560 |
0.724 |
| 10 |
Tier 3 (24 features) |
LightGBM |
0.857 |
0.876 |
0.855 |
0.438 |
0.982 |
0.934 |
0.584 |
0.730 |
| 11 |
Tier 3 (24 features) |
XGBoost |
0.831 |
0.907 |
0.821 |
0.396 |
0.986 |
0.933 |
0.551 |
0.721 |
final_table.to_csv("../results/model_comparison_table_means.csv", index=False)
print("Saved:")
print("- ../results/model_comparison_table_means.csv")
Saved:
- ../results/model_comparison_table_means.csv