Final Metric Tables
# Combine XGBoost CV and Test results into one comprehensive CSV
import pandas as pd
# Load the two XGBoost result files (go up one directory from notebooks/)
cv_results = pd.read_csv("../results/xgb_5fold_threshold0.50_by_tier.csv")
# CV results already have Mean and Std Dev columns
# Just save it as the comprehensive XGBoost results
cv_results.to_csv("../results/xgb_comprehensive_results.csv", index=False)
print("XGBoost Comprehensive Results (Mean ± Std from 5-Fold CV):")
print(cv_results)
print(f"\nSaved to: ../results/xgb_comprehensive_results.csv")
XGBoost Comprehensive Results (Mean ± Std from 5-Fold CV):
Metric Tier Fold 1 Fold 2 Fold 3 Fold 4 \
0 Accuracy Tier_1_Basic 0.558941 0.532761 0.601717 0.568526
1 Sensitivity Tier_1_Basic 0.724656 0.783750 0.707500 0.736250
2 Specificity Tier_1_Basic 0.537555 0.500323 0.588045 0.546850
3 PPV Tier_1_Basic 0.168216 0.168548 0.181643 0.173542
4 NPV Tier_1_Basic 0.937993 0.947095 0.939597 0.941324
5 AUC Tier_1_Basic 0.693584 0.709003 0.714747 0.701434
6 F1 Tier_1_Basic 0.273049 0.277434 0.289070 0.280877
7 F2 Tier_1_Basic 0.436125 0.453035 0.448068 0.446618
8 Accuracy Tier_2_Clinical 0.831903 0.830758 0.827754 0.826753
9 Sensitivity Tier_2_Clinical 0.903630 0.895000 0.908750 0.903750
10 Specificity Tier_2_Clinical 0.822646 0.822456 0.817286 0.816801
11 PPV Tier_2_Clinical 0.396703 0.394490 0.391281 0.389338
12 NPV Tier_2_Clinical 0.985106 0.983768 0.985776 0.984999
13 AUC Tier_2_Clinical 0.931716 0.919203 0.929653 0.925441
14 F1 Tier_2_Clinical 0.551355 0.547610 0.547028 0.544223
15 F2 Tier_2_Clinical 0.719697 0.713858 0.718664 0.714851
16 Accuracy Tier_3_Personalized 0.836481 0.824320 0.825608 0.828755
17 Sensitivity Tier_3_Personalized 0.909887 0.881250 0.923750 0.901250
18 Specificity Tier_3_Personalized 0.827007 0.816963 0.812924 0.819386
19 PPV Tier_3_Personalized 0.404338 0.383569 0.389562 0.392061
20 NPV Tier_3_Personalized 0.986133 0.981561 0.988023 0.984663
21 AUC Tier_3_Personalized 0.937897 0.925039 0.933029 0.931067
22 F1 Tier_3_Personalized 0.559877 0.534496 0.548016 0.546419
23 F2 Tier_3_Personalized 0.727873 0.699682 0.724936 0.715420
Fold 5 Mean Std Dev
0 0.624696 0.577328 0.032374
1 0.685857 0.727603 0.032811
2 0.616801 0.557915 0.040569
3 0.187671 0.175924 0.007618
4 0.938314 0.940865 0.003328
5 0.716441 0.707042 0.008533
6 0.294703 0.283027 0.007853
7 0.448005 0.446370 0.005568
8 0.836314 0.830696 0.003383
9 0.917397 0.905705 0.007331
10 0.825848 0.821007 0.003457
11 0.404749 0.395312 0.005359
12 0.987254 0.985381 0.001139
13 0.933265 0.927855 0.005061
14 0.561686 0.550380 0.006093
15 0.731975 0.719809 0.006470
16 0.840607 0.831154 0.006339
17 0.918648 0.906957 0.014971
18 0.830533 0.821363 0.006487
19 0.411666 0.396239 0.010258
20 0.987514 0.985579 0.002324
21 0.939726 0.933352 0.005210
22 0.568552 0.551472 0.011730
23 0.737096 0.721002 0.012707
Saved to: ../results/xgb_comprehensive_results.csv
# Combine all model results into a well-organized comparison table
import pandas as pd
# Load CV results from all models
logreg_cv = pd.read_csv("../results/logreg_5fold_threshold0.50_by_tier.csv")
rf_cv = pd.read_csv("../results/rf_5fold_threshold0.50_by_tier.csv")
xgb_cv = pd.read_csv("../results/xgb_5fold_threshold0.50_by_tier.csv")
lgbm_cv = pd.read_csv("../results/lightgbm_5fold_threshold0.50_by_tier.csv")
# Standardize logreg format to match others
# Rename 'Tier' -> 'tier' if present
if 'Tier' in logreg_cv.columns:
logreg_cv = logreg_cv.rename(columns={'Tier': 'tier'})
# Capitalize metric names if lowercase
logreg_cv['Metric'] = logreg_cv['Metric'].str.capitalize()
logreg_cv['Metric'] = logreg_cv['Metric'].replace({
'Accuracy': 'Accuracy', 'Sensitivity': 'Sensitivity', 'Specificity': 'Specificity',
'Ppv': 'PPV', 'Npv': 'NPV', 'Auc': 'AUC', 'F1': 'F1', 'F2': 'F2'
})
# Drop existing Model column if present (we'll add our own)
if 'Model' in logreg_cv.columns:
logreg_cv = logreg_cv.drop(columns=['Model'])
# Add model column to each
logreg_cv.insert(0, 'Model', 'Logistic Regression')
rf_cv.insert(0, 'Model', 'Random Forest')
xgb_cv.insert(0, 'Model', 'XGBoost')
lgbm_cv.insert(0, 'Model', 'LightGBM')
# Combine all models
all_models_cv = pd.concat([logreg_cv, rf_cv, xgb_cv, lgbm_cv], ignore_index=True)
# Keep only summary columns
summary_df = all_models_cv[['Model', 'tier', 'Metric', 'Mean', 'Std Dev']].copy()
# Create formatted "Mean ± Std" column
summary_df['Value'] = summary_df.apply(
lambda row: f"{row['Mean']:.3f} ± {row['Std Dev']:.3f}", axis=1
)
# Pivot to wide format: rows = (Tier, Metric), columns = Model
pivot_df = summary_df.pivot_table(
index=['tier', 'Metric'],
columns='Model',
values='Value',
aggfunc='first'
)
# Reorder columns (models)
model_order = ['Logistic Regression', 'Random Forest', 'XGBoost', 'LightGBM']
pivot_df = pivot_df[model_order]
# Reorder index (tiers and metrics)
tier_order = ['Tier_1_Basic', 'Tier_2_Clinical', 'Tier_3_Personalized']
metric_order = ['AUC', 'Sensitivity', 'Specificity', 'PPV', 'NPV', 'Accuracy', 'F1', 'F2']
pivot_df = pivot_df.reindex(
pd.MultiIndex.from_product([tier_order, metric_order], names=['Tier', 'Metric'])
)
# Reset index for cleaner display
final_df = pivot_df.reset_index()
# Save to CSV
final_df.to_csv("../results/all_models_comparison_organized.csv", index=False)
# Display
pd.set_option('display.max_rows', 100)
pd.set_option('display.max_colwidth', 20)
print("=" * 100)
print("MODEL PERFORMANCE COMPARISON (5-Fold CV: Mean ± Std)")
print("=" * 100)
for tier in tier_order:
print(f"\n{'─' * 100}")
print(f" {tier.replace('_', ' ').upper()}")
print(f"{'─' * 100}")
tier_data = final_df[final_df['Tier'] == tier][['Metric'] + model_order]
print(tier_data.to_string(index=False))
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In[4], line 51
49 # Reorder columns (models)
50 model_order = ['Logistic Regression', 'Random Forest', 'XGBoost', 'LightGBM']
---> 51 pivot_df = pivot_df[model_order]
53 # Reorder index (tiers and metrics)
54 tier_order = ['Tier_1_Basic', 'Tier_2_Clinical', 'Tier_3_Personalized']
File c:\Users\agila\Documents\STM\Predicting-Past-Year-Major-Depressive-Episode\venv\Lib\site-packages\pandas\core\frame.py:4119, in DataFrame.__getitem__(self, key)
4117 if is_iterator(key):
4118 key = list(key)
-> 4119 indexer = self.columns._get_indexer_strict(key, "columns")[1]
4121 # take() does not accept boolean indexers
4122 if getattr(indexer, "dtype", None) == bool:
File c:\Users\agila\Documents\STM\Predicting-Past-Year-Major-Depressive-Episode\venv\Lib\site-packages\pandas\core\indexes\base.py:6212, in Index._get_indexer_strict(self, key, axis_name)
6209 else:
6210 keyarr, indexer, new_indexer = self._reindex_non_unique(keyarr)
-> 6212 self._raise_if_missing(keyarr, indexer, axis_name)
6214 keyarr = self.take(indexer)
6215 if isinstance(key, Index):
6216 # GH 42790 - Preserve name from an Index
File c:\Users\agila\Documents\STM\Predicting-Past-Year-Major-Depressive-Episode\venv\Lib\site-packages\pandas\core\indexes\base.py:6264, in Index._raise_if_missing(self, key, indexer, axis_name)
6261 raise KeyError(f"None of [{key}] are in the [{axis_name}]")
6263 not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique())
-> 6264 raise KeyError(f"{not_found} not in index")
KeyError: "['Random Forest', 'XGBoost', 'LightGBM'] not in index"
# Create table with individual folds as columns for each model
import pandas as pd
# Load CV results from all models
logreg_cv = pd.read_csv("../results/logreg_5fold_threshold0.50_by_tier.csv")
rf_cv = pd.read_csv("../results/rf_5fold_threshold0.50_by_tier.csv")
xgb_cv = pd.read_csv("../results/xgb_5fold_threshold0.50_by_tier.csv")
lgbm_cv = pd.read_csv("../results/lightgbm_5fold_threshold0.50_by_tier.csv")
# Standardize logreg format
if 'Tier' in logreg_cv.columns:
logreg_cv = logreg_cv.rename(columns={'Tier': 'tier'})
logreg_cv['Metric'] = logreg_cv['Metric'].str.capitalize()
logreg_cv['Metric'] = logreg_cv['Metric'].replace({'Ppv': 'PPV', 'Npv': 'NPV', 'Auc': 'AUC'})
if 'Model' in logreg_cv.columns:
logreg_cv = logreg_cv.drop(columns=['Model'])
# Add model names
logreg_cv['Model'] = 'LogReg'
rf_cv['Model'] = 'RF'
xgb_cv['Model'] = 'XGB'
lgbm_cv['Model'] = 'LGBM'
# Combine all
all_cv = pd.concat([logreg_cv, rf_cv, xgb_cv, lgbm_cv], ignore_index=True)
# Fold columns
fold_cols = ['Fold 1', 'Fold 2', 'Fold 3', 'Fold 4', 'Fold 5']
# Create wide format: columns = Model_Fold1, Model_Fold2, etc.
rows = []
for _, row in all_cv.iterrows():
for fold in fold_cols:
rows.append({
'Tier': row['tier'],
'Metric': row['Metric'],
'Model': row['Model'],
'Fold': fold,
'Value': row[fold]
})
fold_df = pd.DataFrame(rows)
fold_df['Model_Fold'] = fold_df['Model'] + '_' + fold_df['Fold'].str.replace('Fold ', 'F')
# Pivot: rows = (Tier, Metric), columns = Model_Fold
pivot_folds = fold_df.pivot_table(
index=['Tier', 'Metric'],
columns='Model_Fold',
values='Value',
aggfunc='first'
)
# Reorder columns: group by model
col_order = [f'{m}_F{i}' for m in ['LogReg', 'RF', 'XGB', 'LGBM'] for i in range(1, 6)]
pivot_folds = pivot_folds[col_order]
# Reorder rows
tier_order = ['Tier_1_Basic', 'Tier_2_Clinical', 'Tier_3_Personalized']
metric_order = ['AUC', 'Sensitivity', 'Specificity', 'PPV', 'NPV', 'Accuracy', 'F1', 'F2']
pivot_folds = pivot_folds.reindex(
pd.MultiIndex.from_product([tier_order, metric_order], names=['Tier', 'Metric'])
)
# Reset and save
folds_final = pivot_folds.reset_index()
folds_final.to_csv("../results/all_models_by_fold.csv", index=False)
print("=" * 120)
print("ALL MODELS - INDIVIDUAL FOLD VALUES")
print("=" * 120)
print(f"\nSaved to: ../results/all_models_by_fold.csv")
print(f"\nShape: {folds_final.shape}")
print(f"Columns: {list(folds_final.columns)}")