{ "cells": [ { "cell_type": "markdown", "id": "a7719433", "metadata": {}, "source": [ "# Final Metric Tables" ] }, { "cell_type": "code", "execution_count": 3, "id": "6b262702", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "XGBoost Comprehensive Results (Mean ± Std from 5-Fold CV):\n", " Metric Tier Fold 1 Fold 2 Fold 3 Fold 4 \\\n", "0 Accuracy Tier_1_Basic 0.558941 0.532761 0.601717 0.568526 \n", "1 Sensitivity Tier_1_Basic 0.724656 0.783750 0.707500 0.736250 \n", "2 Specificity Tier_1_Basic 0.537555 0.500323 0.588045 0.546850 \n", "3 PPV Tier_1_Basic 0.168216 0.168548 0.181643 0.173542 \n", "4 NPV Tier_1_Basic 0.937993 0.947095 0.939597 0.941324 \n", "5 AUC Tier_1_Basic 0.693584 0.709003 0.714747 0.701434 \n", "6 F1 Tier_1_Basic 0.273049 0.277434 0.289070 0.280877 \n", "7 F2 Tier_1_Basic 0.436125 0.453035 0.448068 0.446618 \n", "8 Accuracy Tier_2_Clinical 0.831903 0.830758 0.827754 0.826753 \n", "9 Sensitivity Tier_2_Clinical 0.903630 0.895000 0.908750 0.903750 \n", "10 Specificity Tier_2_Clinical 0.822646 0.822456 0.817286 0.816801 \n", "11 PPV Tier_2_Clinical 0.396703 0.394490 0.391281 0.389338 \n", "12 NPV Tier_2_Clinical 0.985106 0.983768 0.985776 0.984999 \n", "13 AUC Tier_2_Clinical 0.931716 0.919203 0.929653 0.925441 \n", "14 F1 Tier_2_Clinical 0.551355 0.547610 0.547028 0.544223 \n", "15 F2 Tier_2_Clinical 0.719697 0.713858 0.718664 0.714851 \n", "16 Accuracy Tier_3_Personalized 0.836481 0.824320 0.825608 0.828755 \n", "17 Sensitivity Tier_3_Personalized 0.909887 0.881250 0.923750 0.901250 \n", "18 Specificity Tier_3_Personalized 0.827007 0.816963 0.812924 0.819386 \n", "19 PPV Tier_3_Personalized 0.404338 0.383569 0.389562 0.392061 \n", "20 NPV Tier_3_Personalized 0.986133 0.981561 0.988023 0.984663 \n", "21 AUC Tier_3_Personalized 0.937897 0.925039 0.933029 0.931067 \n", "22 F1 Tier_3_Personalized 0.559877 0.534496 0.548016 0.546419 \n", "23 F2 Tier_3_Personalized 0.727873 0.699682 0.724936 0.715420 \n", "\n", " Fold 5 Mean Std Dev \n", "0 0.624696 0.577328 0.032374 \n", "1 0.685857 0.727603 0.032811 \n", "2 0.616801 0.557915 0.040569 \n", "3 0.187671 0.175924 0.007618 \n", "4 0.938314 0.940865 0.003328 \n", "5 0.716441 0.707042 0.008533 \n", "6 0.294703 0.283027 0.007853 \n", "7 0.448005 0.446370 0.005568 \n", "8 0.836314 0.830696 0.003383 \n", "9 0.917397 0.905705 0.007331 \n", "10 0.825848 0.821007 0.003457 \n", "11 0.404749 0.395312 0.005359 \n", "12 0.987254 0.985381 0.001139 \n", "13 0.933265 0.927855 0.005061 \n", "14 0.561686 0.550380 0.006093 \n", "15 0.731975 0.719809 0.006470 \n", "16 0.840607 0.831154 0.006339 \n", "17 0.918648 0.906957 0.014971 \n", "18 0.830533 0.821363 0.006487 \n", "19 0.411666 0.396239 0.010258 \n", "20 0.987514 0.985579 0.002324 \n", "21 0.939726 0.933352 0.005210 \n", "22 0.568552 0.551472 0.011730 \n", "23 0.737096 0.721002 0.012707 \n", "\n", "Saved to: ../results/xgb_comprehensive_results.csv\n" ] } ], "source": [ "# Combine XGBoost CV and Test results into one comprehensive CSV\n", "import pandas as pd\n", "\n", "# Load the two XGBoost result files (go up one directory from notebooks/)\n", "cv_results = pd.read_csv(\"../results/xgb_5fold_threshold0.50_by_tier.csv\")\n", "\n", "# CV results already have Mean and Std Dev columns\n", "# Just save it as the comprehensive XGBoost results\n", "cv_results.to_csv(\"../results/xgb_comprehensive_results.csv\", index=False)\n", "\n", "print(\"XGBoost Comprehensive Results (Mean ± Std from 5-Fold CV):\")\n", "print(cv_results)\n", "print(f\"\\nSaved to: ../results/xgb_comprehensive_results.csv\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "34f387b0", "metadata": {}, "outputs": [ { "ename": "KeyError", "evalue": "\"['Random Forest', 'XGBoost', 'LightGBM'] not in index\"", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 51\u001b[39m\n\u001b[32m 49\u001b[39m \u001b[38;5;66;03m# Reorder columns (models)\u001b[39;00m\n\u001b[32m 50\u001b[39m model_order = [\u001b[33m'\u001b[39m\u001b[33mLogistic Regression\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mRandom Forest\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mXGBoost\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mLightGBM\u001b[39m\u001b[33m'\u001b[39m]\n\u001b[32m---> \u001b[39m\u001b[32m51\u001b[39m pivot_df = \u001b[43mpivot_df\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmodel_order\u001b[49m\u001b[43m]\u001b[49m\n\u001b[32m 53\u001b[39m \u001b[38;5;66;03m# Reorder index (tiers and metrics)\u001b[39;00m\n\u001b[32m 54\u001b[39m tier_order = [\u001b[33m'\u001b[39m\u001b[33mTier_1_Basic\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mTier_2_Clinical\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m'\u001b[39m\u001b[33mTier_3_Personalized\u001b[39m\u001b[33m'\u001b[39m]\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\agila\\Documents\\STM\\Predicting-Past-Year-Major-Depressive-Episode\\venv\\Lib\\site-packages\\pandas\\core\\frame.py:4119\u001b[39m, in \u001b[36mDataFrame.__getitem__\u001b[39m\u001b[34m(self, key)\u001b[39m\n\u001b[32m 4117\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m is_iterator(key):\n\u001b[32m 4118\u001b[39m key = \u001b[38;5;28mlist\u001b[39m(key)\n\u001b[32m-> \u001b[39m\u001b[32m4119\u001b[39m indexer = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_get_indexer_strict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcolumns\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m[\u001b[32m1\u001b[39m]\n\u001b[32m 4121\u001b[39m \u001b[38;5;66;03m# take() does not accept boolean indexers\u001b[39;00m\n\u001b[32m 4122\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(indexer, \u001b[33m\"\u001b[39m\u001b[33mdtype\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) == \u001b[38;5;28mbool\u001b[39m:\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\agila\\Documents\\STM\\Predicting-Past-Year-Major-Depressive-Episode\\venv\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:6212\u001b[39m, in \u001b[36mIndex._get_indexer_strict\u001b[39m\u001b[34m(self, key, axis_name)\u001b[39m\n\u001b[32m 6209\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 6210\u001b[39m keyarr, indexer, new_indexer = \u001b[38;5;28mself\u001b[39m._reindex_non_unique(keyarr)\n\u001b[32m-> \u001b[39m\u001b[32m6212\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_raise_if_missing\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkeyarr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 6214\u001b[39m keyarr = \u001b[38;5;28mself\u001b[39m.take(indexer)\n\u001b[32m 6215\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, Index):\n\u001b[32m 6216\u001b[39m \u001b[38;5;66;03m# GH 42790 - Preserve name from an Index\u001b[39;00m\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\agila\\Documents\\STM\\Predicting-Past-Year-Major-Depressive-Episode\\venv\\Lib\\site-packages\\pandas\\core\\indexes\\base.py:6264\u001b[39m, in \u001b[36mIndex._raise_if_missing\u001b[39m\u001b[34m(self, key, indexer, axis_name)\u001b[39m\n\u001b[32m 6261\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mNone of [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m] are in the [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00maxis_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m]\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 6263\u001b[39m not_found = \u001b[38;5;28mlist\u001b[39m(ensure_index(key)[missing_mask.nonzero()[\u001b[32m0\u001b[39m]].unique())\n\u001b[32m-> \u001b[39m\u001b[32m6264\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnot_found\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m not in index\u001b[39m\u001b[33m\"\u001b[39m)\n", "\u001b[31mKeyError\u001b[39m: \"['Random Forest', 'XGBoost', 'LightGBM'] not in index\"" ] } ], "source": [ "# Combine all model results into a well-organized comparison table\n", "import pandas as pd\n", "\n", "# Load CV results from all models\n", "logreg_cv = pd.read_csv(\"../results/logreg_5fold_threshold0.50_by_tier.csv\")\n", "rf_cv = pd.read_csv(\"../results/rf_5fold_threshold0.50_by_tier.csv\")\n", "xgb_cv = pd.read_csv(\"../results/xgb_5fold_threshold0.50_by_tier.csv\")\n", "lgbm_cv = pd.read_csv(\"../results/lightgbm_5fold_threshold0.50_by_tier.csv\")\n", "\n", "# Standardize logreg format to match others\n", "# Rename 'Tier' -> 'tier' if present\n", "if 'Tier' in logreg_cv.columns:\n", " logreg_cv = logreg_cv.rename(columns={'Tier': 'tier'})\n", "# Capitalize metric names if lowercase\n", "logreg_cv['Metric'] = logreg_cv['Metric'].str.capitalize()\n", "logreg_cv['Metric'] = logreg_cv['Metric'].replace({\n", " 'Accuracy': 'Accuracy', 'Sensitivity': 'Sensitivity', 'Specificity': 'Specificity',\n", " 'Ppv': 'PPV', 'Npv': 'NPV', 'Auc': 'AUC', 'F1': 'F1', 'F2': 'F2'\n", "})\n", "# Drop existing Model column if present (we'll add our own)\n", "if 'Model' in logreg_cv.columns:\n", " logreg_cv = logreg_cv.drop(columns=['Model'])\n", "\n", "# Add model column to each\n", "logreg_cv.insert(0, 'Model', 'Logistic Regression')\n", "rf_cv.insert(0, 'Model', 'Random Forest')\n", "xgb_cv.insert(0, 'Model', 'XGBoost')\n", "lgbm_cv.insert(0, 'Model', 'LightGBM')\n", "\n", "# Combine all models\n", "all_models_cv = pd.concat([logreg_cv, rf_cv, xgb_cv, lgbm_cv], ignore_index=True)\n", "\n", "# Keep only summary columns\n", "summary_df = all_models_cv[['Model', 'tier', 'Metric', 'Mean', 'Std Dev']].copy()\n", "\n", "# Create formatted \"Mean ± Std\" column\n", "summary_df['Value'] = summary_df.apply(\n", " lambda row: f\"{row['Mean']:.3f} ± {row['Std Dev']:.3f}\", axis=1\n", ")\n", "\n", "# Pivot to wide format: rows = (Tier, Metric), columns = Model\n", "pivot_df = summary_df.pivot_table(\n", " index=['tier', 'Metric'],\n", " columns='Model',\n", " values='Value',\n", " aggfunc='first'\n", ")\n", "\n", "# Reorder columns (models)\n", "model_order = ['Logistic Regression', 'Random Forest', 'XGBoost', 'LightGBM']\n", "pivot_df = pivot_df[model_order]\n", "\n", "# Reorder index (tiers and metrics)\n", "tier_order = ['Tier_1_Basic', 'Tier_2_Clinical', 'Tier_3_Personalized']\n", "metric_order = ['AUC', 'Sensitivity', 'Specificity', 'PPV', 'NPV', 'Accuracy', 'F1', 'F2']\n", "\n", "pivot_df = pivot_df.reindex(\n", " pd.MultiIndex.from_product([tier_order, metric_order], names=['Tier', 'Metric'])\n", ")\n", "\n", "# Reset index for cleaner display\n", "final_df = pivot_df.reset_index()\n", "\n", "# Save to CSV\n", "final_df.to_csv(\"../results/all_models_comparison_organized.csv\", index=False)\n", "\n", "# Display\n", "pd.set_option('display.max_rows', 100)\n", "pd.set_option('display.max_colwidth', 20)\n", "print(\"=\" * 100)\n", "print(\"MODEL PERFORMANCE COMPARISON (5-Fold CV: Mean ± Std)\")\n", "print(\"=\" * 100)\n", "\n", "for tier in tier_order:\n", " print(f\"\\n{'─' * 100}\")\n", " print(f\" {tier.replace('_', ' ').upper()}\")\n", " print(f\"{'─' * 100}\")\n", " tier_data = final_df[final_df['Tier'] == tier][['Metric'] + model_order]\n", " print(tier_data.to_string(index=False))" ] }, { "cell_type": "code", "execution_count": null, "id": "c9573c45", "metadata": {}, "outputs": [], "source": [ "# Create table with individual folds as columns for each model\n", "import pandas as pd\n", "\n", "# Load CV results from all models\n", "logreg_cv = pd.read_csv(\"../results/logreg_5fold_threshold0.50_by_tier.csv\")\n", "rf_cv = pd.read_csv(\"../results/rf_5fold_threshold0.50_by_tier.csv\")\n", "xgb_cv = pd.read_csv(\"../results/xgb_5fold_threshold0.50_by_tier.csv\")\n", "lgbm_cv = pd.read_csv(\"../results/lightgbm_5fold_threshold0.50_by_tier.csv\")\n", "\n", "# Standardize logreg format\n", "if 'Tier' in logreg_cv.columns:\n", " logreg_cv = logreg_cv.rename(columns={'Tier': 'tier'})\n", "logreg_cv['Metric'] = logreg_cv['Metric'].str.capitalize()\n", "logreg_cv['Metric'] = logreg_cv['Metric'].replace({'Ppv': 'PPV', 'Npv': 'NPV', 'Auc': 'AUC'})\n", "if 'Model' in logreg_cv.columns:\n", " logreg_cv = logreg_cv.drop(columns=['Model'])\n", "\n", "# Add model names\n", "logreg_cv['Model'] = 'LogReg'\n", "rf_cv['Model'] = 'RF'\n", "xgb_cv['Model'] = 'XGB'\n", "lgbm_cv['Model'] = 'LGBM'\n", "\n", "# Combine all\n", "all_cv = pd.concat([logreg_cv, rf_cv, xgb_cv, lgbm_cv], ignore_index=True)\n", "\n", "# Fold columns\n", "fold_cols = ['Fold 1', 'Fold 2', 'Fold 3', 'Fold 4', 'Fold 5']\n", "\n", "# Create wide format: columns = Model_Fold1, Model_Fold2, etc.\n", "rows = []\n", "for _, row in all_cv.iterrows():\n", " for fold in fold_cols:\n", " rows.append({\n", " 'Tier': row['tier'],\n", " 'Metric': row['Metric'],\n", " 'Model': row['Model'],\n", " 'Fold': fold,\n", " 'Value': row[fold]\n", " })\n", "\n", "fold_df = pd.DataFrame(rows)\n", "fold_df['Model_Fold'] = fold_df['Model'] + '_' + fold_df['Fold'].str.replace('Fold ', 'F')\n", "\n", "# Pivot: rows = (Tier, Metric), columns = Model_Fold\n", "pivot_folds = fold_df.pivot_table(\n", " index=['Tier', 'Metric'],\n", " columns='Model_Fold',\n", " values='Value',\n", " aggfunc='first'\n", ")\n", "\n", "# Reorder columns: group by model\n", "col_order = [f'{m}_F{i}' for m in ['LogReg', 'RF', 'XGB', 'LGBM'] for i in range(1, 6)]\n", "pivot_folds = pivot_folds[col_order]\n", "\n", "# Reorder rows\n", "tier_order = ['Tier_1_Basic', 'Tier_2_Clinical', 'Tier_3_Personalized']\n", "metric_order = ['AUC', 'Sensitivity', 'Specificity', 'PPV', 'NPV', 'Accuracy', 'F1', 'F2']\n", "pivot_folds = pivot_folds.reindex(\n", " pd.MultiIndex.from_product([tier_order, metric_order], names=['Tier', 'Metric'])\n", ")\n", "\n", "# Reset and save\n", "folds_final = pivot_folds.reset_index()\n", "folds_final.to_csv(\"../results/all_models_by_fold.csv\", index=False)\n", "\n", "print(\"=\" * 120)\n", "print(\"ALL MODELS - INDIVIDUAL FOLD VALUES\")\n", "print(\"=\" * 120)\n", "print(f\"\\nSaved to: ../results/all_models_by_fold.csv\")\n", "print(f\"\\nShape: {folds_final.shape}\")\n", "print(f\"Columns: {list(folds_final.columns)}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5920bee3", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.9" } }, "nbformat": 4, "nbformat_minor": 5 }