{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "dd98f908", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/shezin/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MetricSetModelAccuracySensitivitySpecificityPPVNPVAUCF1F2
0Tier 1 (6 features)LogisticRegression0.5370.7680.5070.1670.9440.6960.2750.447
1Tier 1 (6 features)RandomForest0.6600.6240.6650.1940.9320.7080.2960.432
2Tier 1 (6 features)LightGBM0.6470.6380.6480.1900.9330.7050.2920.433
3Tier 1 (6 features)XGBoost0.5770.7280.5580.1760.9410.7070.2830.446
4Tier 2 (10 features)LogisticRegression0.8210.9140.8090.3820.9860.9250.5390.715
5Tier 2 (10 features)RandomForest0.8310.9010.8230.3960.9850.9270.5500.718
6Tier 2 (10 features)LightGBM0.8430.8850.8370.4130.9830.9280.5630.721
7Tier 2 (10 features)XGBoost0.8310.9060.8210.3950.9850.9280.5500.720
8Tier 3 (24 features)LogisticRegression0.8310.9110.8200.3960.9860.9320.5520.723
9Tier 3 (24 features)RandomForest0.8380.8990.8310.4070.9850.9320.5600.724
10Tier 3 (24 features)LightGBM0.8570.8760.8550.4380.9820.9340.5840.730
11Tier 3 (24 features)XGBoost0.8310.9070.8210.3960.9860.9330.5510.721
\n", "
" ], "text/plain": [ "Metric Set Model Accuracy Sensitivity \\\n", "0 Tier 1 (6 features) LogisticRegression 0.537 0.768 \n", "1 Tier 1 (6 features) RandomForest 0.660 0.624 \n", "2 Tier 1 (6 features) LightGBM 0.647 0.638 \n", "3 Tier 1 (6 features) XGBoost 0.577 0.728 \n", "4 Tier 2 (10 features) LogisticRegression 0.821 0.914 \n", "5 Tier 2 (10 features) RandomForest 0.831 0.901 \n", "6 Tier 2 (10 features) LightGBM 0.843 0.885 \n", "7 Tier 2 (10 features) XGBoost 0.831 0.906 \n", "8 Tier 3 (24 features) LogisticRegression 0.831 0.911 \n", "9 Tier 3 (24 features) RandomForest 0.838 0.899 \n", "10 Tier 3 (24 features) LightGBM 0.857 0.876 \n", "11 Tier 3 (24 features) XGBoost 0.831 0.907 \n", "\n", "Metric Specificity PPV NPV AUC F1 F2 \n", "0 0.507 0.167 0.944 0.696 0.275 0.447 \n", "1 0.665 0.194 0.932 0.708 0.296 0.432 \n", "2 0.648 0.190 0.933 0.705 0.292 0.433 \n", "3 0.558 0.176 0.941 0.707 0.283 0.446 \n", "4 0.809 0.382 0.986 0.925 0.539 0.715 \n", "5 0.823 0.396 0.985 0.927 0.550 0.718 \n", "6 0.837 0.413 0.983 0.928 0.563 0.721 \n", "7 0.821 0.395 0.985 0.928 0.550 0.720 \n", "8 0.820 0.396 0.986 0.932 0.552 0.723 \n", "9 0.831 0.407 0.985 0.932 0.560 0.724 \n", "10 0.855 0.438 0.982 0.934 0.584 0.730 \n", "11 0.821 0.396 0.986 0.933 0.551 0.721 " ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "model_files = {\n", " \"LogisticRegression\": \"../results/logreg_5fold_threshold0.50_by_tier.csv\",\n", " \"RandomForest\": \"../results/rf_5fold_threshold0.50_by_tier.csv\",\n", " \"LightGBM\": \"../results/lightgbm_5fold_threshold0.50_by_tier.csv\",\n", " \"XGBoost\": \"../results/xgb_5fold_threshold0.50_by_tier.csv\",\n", "}\n", "\n", "metrics_keep = [\"Accuracy\", \"Sensitivity\", \"Specificity\", \"PPV\", \"NPV\", \"AUC\", \"F1\", \"F2\"]\n", "\n", "\n", "tier_to_set_label = {\n", " \"Tier_1_Basic\": \"Tier 1 (6 features)\",\n", " \"Tier_2_Clinical\": \"Tier 2 (10 features)\",\n", " \"Tier_3_Personalized\": \"Tier 3 (24 features)\",\n", "}\n", "\n", "# tier_feature_counts = {\"Tier_1_Basic\": 6, \"Tier_2_Clinical\": 10, \"Tier_3_Personalized\": 24}\n", "tier_feature_counts = None \n", "\n", "\n", "def load_model_means(csv_path: str, model_name: str) -> pd.DataFrame:\n", " df = pd.read_csv(csv_path)\n", "\n", " # keeping only needed metrics\n", " df = df[df[\"Metric\"].isin(metrics_keep)].copy()\n", "\n", " # pivot: rows=tier, cols=Metric, values=Mean\n", " wide = df.pivot_table(index=\"Tier\", columns=\"Metric\", values=\"Mean\", aggfunc=\"first\").reset_index()\n", "\n", " wide.insert(1, \"Model\", model_name)\n", " return wide\n", "\n", "\n", "all_models = []\n", "for model_name, path in model_files.items():\n", " one = load_model_means(path, model_name)\n", " all_models.append(one)\n", "\n", "combined = pd.concat(all_models, ignore_index=True)\n", "\n", "\n", "def format_set(tier_name: str) -> str:\n", " base = tier_to_set_label.get(tier_name, tier_name)\n", " if isinstance(tier_feature_counts, dict) and tier_name in tier_feature_counts:\n", " return f\"{base} ({tier_feature_counts[tier_name]} features)\"\n", " return base\n", "\n", "combined.insert(0, \"Set\", combined[\"Tier\"].map(format_set))\n", "\n", "\n", "final_cols = [\"Set\", \"Model\"] + metrics_keep\n", "final_table = combined[final_cols].copy()\n", "\n", "set_order = [format_set(k) for k in [\"Tier_1_Basic\", \"Tier_2_Clinical\", \"Tier_3_Personalized\"]]\n", "model_order = [\"LogisticRegression\", \"RandomForest\", \"LightGBM\", \"XGBoost\"]\n", "\n", "final_table[\"Set\"] = pd.Categorical(final_table[\"Set\"], categories=set_order, ordered=True)\n", "final_table[\"Model\"] = pd.Categorical(final_table[\"Model\"], categories=model_order, ordered=True)\n", "final_table = final_table.sort_values([\"Set\", \"Model\"]).reset_index(drop=True)\n", "\n", "final_table[metrics_keep] = final_table[metrics_keep].round(3)\n", "\n", "final_table\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "cec44d06", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Saved:\n", "- ../results/model_comparison_table_means.csv\n" ] } ], "source": [ "final_table.to_csv(\"../results/model_comparison_table_means.csv\", index=False)\n", "\n", "print(\"Saved:\")\n", "print(\"- ../results/model_comparison_table_means.csv\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }