Imports

import pandas as pd
import numpy as np
import time

from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score, recall_score

Load Cleaned Dataset

df = pd.read_pickle("../data/cleaned/NSDUH_2023_clean.pkl")

TARGET = "IRAMDEYR"
WEIGHT = "ANALWT2_C"   # remove for feature selection

y = df[TARGET]
X = df.drop(columns=[TARGET, WEIGHT])  # DROP WEIGHT COLUMN

print("X shape:", X.shape)
print("y mean:", y.mean())

X shape: (45133, 2315)
y mean: 0.11488268007887799

Train/Test split

# numeric coercion
X = X.apply(pd.to_numeric, errors="coerce").fillna(0)

# Split (same rule as all other feature-selection notebooks)
X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.2,
    stratify=y,
    random_state=42
)

print("Train:", X_train.shape, " Test:", X_test.shape)

Train: (36106, 2315)  Test: (9027, 2315)

Standardize

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

feature_names = X.columns

Define a LASSO Helper Function

def run_lasso(C_value, label):
    print(f"\nRunning LASSO ({label}) with C={C_value}")
    start = time.time()

    model = LogisticRegression(
        penalty="l1",
        solver="saga",
        C=C_value,
        max_iter=3000,
        n_jobs=-1
    )

    model.fit(X_train_scaled, y_train)

    # Time
    minutes = (time.time() - start) / 60
    print(f"Completed in {minutes:.2f} minutes")

    # evaluate on test
    if label.lower().startswith("auc"):
        probs = model.predict_proba(X_test_scaled)[:, 1]
        score = roc_auc_score(y_test, probs)
        print("AUC:", score)

    else:
        preds = model.predict(X_test_scaled)
        score = recall_score(y_test, preds)
        print("Recall:", score)

    # coefficient importance
    coef = np.abs(model.coef_[0])
    coef_series = pd.Series(coef, index=feature_names).sort_values(ascending=False)

    top100 = coef_series.head(100).index.tolist()

    return top100, coef_series

Run LASSO (AUC version)

lasso_auc_top100, coef_auc = run_lasso(
    C_value=1.0,    # stronger regularization → AUC-focused
    label="AUC-Optimized"
)

pd.Series(lasso_auc_top100).to_csv(
    "../results/lasso_auc_top100.csv",
    index=False
)

print("\nSaved lasso_auc_top100.csv")

Running LASSO (AUC-Optimized) with C=1.0


c:\Users\agila\Documents\STM\Predicting-Past-Year-Major-Depressive-Episode\venv\Lib\site-packages\sklearn\linear_model\_sag.py:348: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge
  warnings.warn(


Completed in 84.03 minutes
AUC: 0.9988266432365432

Saved lasso_auc_top100.csv

Run LASSO (Recall version)

lasso_recall_top100, coef_recall = run_lasso(
    C_value=0.5,     # weaker regularization → increases recall bias
    label="Recall-Optimized"
)

pd.Series(lasso_recall_top100).to_csv(
    "../results/lasso_recall_top100.csv",
    index=False
)

print("\nSaved lasso_recall_top100.csv")

Running LASSO (Recall-Optimized) with C=0.5
Completed in 77.27 minutes
Recall: 0.9816779170684667

Saved lasso_recall_top100.csv


c:\Users\agila\Documents\STM\Predicting-Past-Year-Major-Depressive-Episode\venv\Lib\site-packages\sklearn\linear_model\_sag.py:348: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge
  warnings.warn(