import pandas as pd
import numpy as np
df = pd.read_pickle("../data/cleaned/NSDUH_2023_clean.pkl")
/home/shezin/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).
from pandas.core.computation.check import NUMEXPR_INSTALLED
# Removing these features (person must already have a depression to asnwer these questions)
FEATURES_TO_REMOVE = [
"ADRX12MO",
"IRMHTRXMED",
"ADWRENRG",
"ADWRSLNO",
"ADPSHMGT",
]
df = df.drop(columns=[c for c in FEATURES_TO_REMOVE if c in df.columns])
# Checking to make sure all removed
for col in FEATURES_TO_REMOVE:
print(f"{col}: {'Still exists' if col in df.columns else 'Dropped'}")
ADRX12MO: Dropped
IRMHTRXMED: Dropped
ADWRENRG: Dropped
ADWRSLNO: Dropped
ADPSHMGT: Dropped
RAW_TO_RECODED = {
"AGE3": "CATAG3",
"CAMHRCVR": "RCVYMHPRB",
"MJSKNYR": "MJCMOTHYR",
"DIFGETCRK": "DIFOBTCRK",
"MHTUNENFCV": "MHNTENFCV",
"MHTUNINSCV": "MHNTINSCV",
}
for raw, recoded in RAW_TO_RECODED.items():
if raw in df.columns:
df = df.drop(columns=[raw])
# Removing these original non recoded features
FEATURES_TO_REMOVE_NOT_RECODED = [
"AGE3",
"CAMHRCVR",
"MJSKNYR",
"DIFGETCRK",
"MHTUNENFCV",
"MHTUNINSCV",
]
df = df.drop(columns=[c for c in FEATURES_TO_REMOVE_NOT_RECODED if c in df.columns])
# Checking to make sure all removed
for col in FEATURES_TO_REMOVE_NOT_RECODED:
print(f"{col}: {'Still exists' if col in df.columns else 'Dropped'}")
AGE3: Dropped
CAMHRCVR: Dropped
MJSKNYR: Dropped
DIFGETCRK: Dropped
MHTUNENFCV: Dropped
MHTUNINSCV: Dropped
IRMARIT
"""IRMARIT Len : 2 IMPUTATION REVISED MARITAL STATUS
Freq Pct
1 = Married ......................................................................................................................... 18316 32.30
2 = Widowed....................................................................................................................... 1383 2.44
3 = Divorced or Separated .................................................................................................. 4670 8.24
4 = Never Been Married...................................................................................................... 26755 47.18
99 = LEGITIMATE SKIP Respondent is <= 14 years old.................................................. 5581 9.84"""
'IRMARIT Len : 2 IMPUTATION REVISED MARITAL STATUS\nFreq Pct\n1 = Married ......................................................................................................................... 18316 32.30\n2 = Widowed....................................................................................................................... 1383 2.44\n3 = Divorced or Separated .................................................................................................. 4670 8.24\n4 = Never Been Married...................................................................................................... 26755 47.18\n99 = LEGITIMATE SKIP Respondent is <= 14 years old.................................................. 5581 9.84'
# IRMARIT Abstracted
IRMARIT_MAP = {
1: "Married",
2: "Widowed-Divorced-or-Separated",
3: "Widowed-Divorced-or-Separated",
4: "Never Married",
}
IRMARIT_ORDER = [
"Married",
"Widowed-Divorced-or-Separated",
"Never Married",
]
df["IRMARIT_ABS"] = df["IRMARIT"].map(IRMARIT_MAP)
df["IRMARIT_ABS"] = pd.Categorical(
df["IRMARIT_ABS"],
categories=IRMARIT_ORDER,
ordered=True
)
# Sanity checks
print("IRMARIT_ABS missing:", df["IRMARIT_ABS"].isna().sum())
print(df["IRMARIT_ABS"].value_counts().sort_index())
IRMARIT_ABS missing: 0
IRMARIT_ABS
Married 18309
Widowed-Divorced-or-Separated 6040
Never Married 20784
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["IRMARIT"])
"IRMARIT" in df.columns
False
IRPINC3
"""IRPINC3 Len : 2 RECODE -RESP TOT INCOME - IMPUTATION REVISED
Freq Pct
1 = Less than $10,000 (Including Loss).............................................................................. 21742 38.34
2 = $10,000 - $19,999......................................................................................................... 7740 13.65
3 = $20,000 - $29,999......................................................................................................... 5110 9.01
4 = $30,000 - $39,999......................................................................................................... 4689 8.27
5 = $40,000 - $49,999......................................................................................................... 3845 6.78
6 = $50,000 - $74,999......................................................................................................... 5592 9.86
7 = $75,000 or more............................................................................................................ 7987 14.09"""
'IRPINC3 Len : 2 RECODE -RESP TOT INCOME - IMPUTATION REVISED\nFreq Pct\n1 = Less than $10,000 (Including Loss).............................................................................. 21742 38.34\n2 = $10,000 - $19,999......................................................................................................... 7740 13.65\n3 = $20,000 - $29,999......................................................................................................... 5110 9.01\n4 = $30,000 - $39,999......................................................................................................... 4689 8.27\n5 = $40,000 - $49,999......................................................................................................... 3845 6.78\n6 = $50,000 - $74,999......................................................................................................... 5592 9.86\n7 = $75,000 or more............................................................................................................ 7987 14.09'
# IRPINC3 Abstracted
IRPINC3_MAP = {
1: "a_<10k",
2: "b_10k-to-40k",
3: "b_10k-to-40k",
4: "b_10k-to-40k",
5: "c_50k-to-75k",
6: "c_50k-to-75k",
7: "d_75k+",
}
IRPINC3_ORDER = ["a_<10k", "b_10k-to-40k", "c_50k-to-75k", "d_75k+"]
df["IRPINC3_ABS"] = df["IRPINC3"].map(IRPINC3_MAP)
df["IRPINC3_ABS"] = pd.Categorical(
df["IRPINC3_ABS"],
categories=IRPINC3_ORDER,
ordered=True
)
# Sanity checks
print("IRPINC3_ABS missing:", df["IRPINC3_ABS"].isna().sum())
print(df["IRPINC3_ABS"].value_counts().sort_index())
IRPINC3_ABS missing: 0
IRPINC3_ABS
a_<10k 10720
b_10k-to-40k 17116
c_50k-to-75k 9384
d_75k+ 7913
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["IRPINC3"])
"IRPINC3" in df.columns
False
WRKDRGHLP
"""WRKDRGHLP Len : 2 ANY ASSISTANCE PROGRAM OFFERED THROUGH WRK
Freq Pct
1 = Yes................................................................................................................................ 14585 25.72
2 = No................................................................................................................................. 13733 24.22
85 = BAD DATA Logically assigned ................................................................................. 8 0.01
94 = DON'T KNOW........................................................................................................... 1569 2.77
97 = REFUSED .................................................................................................................. 481 0.85
98 = BLANK (NO ANSWER) ........................................................................................... 1435 2.53
99 = LEGITIMATE SKIP................................................................................................... 24894 43.90"""
"WRKDRGHLP Len : 2 ANY ASSISTANCE PROGRAM OFFERED THROUGH WRK\nFreq Pct\n1 = Yes................................................................................................................................ 14585 25.72\n2 = No................................................................................................................................. 13733 24.22\n85 = BAD DATA Logically assigned ................................................................................. 8 0.01\n94 = DON'T KNOW........................................................................................................... 1569 2.77\n97 = REFUSED .................................................................................................................. 481 0.85\n98 = BLANK (NO ANSWER) ........................................................................................... 1435 2.53\n99 = LEGITIMATE SKIP................................................................................................... 24894 43.90"
WRKDRGHLP_MAP = {
1: "Having access to assistance program at work",
2: "Not having access to assistance program at work",
}
WRKDRGHLP_ORDER = [
"Having access to assistance program at work",
"Not having access to assistance program at work",
"No need or do not know about assistance program at work",
]
df["WRKDRGHLP_ABS"] = df["WRKDRGHLP"].map(WRKDRGHLP_MAP)
# IMPORTANT: collapse skip logic / NA into third class
df["WRKDRGHLP_ABS"] = df["WRKDRGHLP_ABS"].fillna(
"No need or do not know about assistance program at work"
)
# Set categorical
df["WRKDRGHLP_ABS"] = pd.Categorical(
df["WRKDRGHLP_ABS"],
categories=WRKDRGHLP_ORDER,
ordered=True
)
# Sanity checks
print("WRKDRGHLP_ABS missing:", df["WRKDRGHLP_ABS"].isna().sum())
print(df["WRKDRGHLP_ABS"].value_counts().sort_index())
WRKDRGHLP_ABS missing: 0
WRKDRGHLP_ABS
Having access to assistance program at work 14216
Not having access to assistance program at work 12347
No need or do not know about assistance program at work 18570
Name: count, dtype: int64
print("WRKDRGHLP missing:", df["WRKDRGHLP"].isna().sum())
print(df["WRKDRGHLP"].value_counts().sort_index())
WRKDRGHLP missing: 18570
WRKDRGHLP
1.0 14216
2.0 12347
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["WRKDRGHLP"])
"WRKDRGHLP" in df.columns
False
KSSLR6MAX
"""KSSLR6MAX Len : 2 RC-WORST K6 TOTAL SCORE IN PAST YEAR
Freq Pct
RANGE = 0 - 24 ................................................................................................................. 45133 79.59
. = Aged 12-17 .................................................................................................................... 11572 20.41"""
'KSSLR6MAX Len : 2 RC-WORST K6 TOTAL SCORE IN PAST YEAR\nFreq Pct\nRANGE = 0 - 24 ................................................................................................................. 45133 79.59\n. = Aged 12-17 .................................................................................................................... 11572 20.41'
# KSSLR6MAX Abstracted
KSSLR6_BINS = [-1, 6, 12, 18, 24]
KSSLR6_LABELS = ["0-6", "07-12", "13-18", "19-24"]
df["KSSLR6MAX_ABS"] = pd.cut(
df["KSSLR6MAX"],
bins=KSSLR6_BINS,
labels=KSSLR6_LABELS
)
# Making it ordered categorical
df["KSSLR6MAX_ABS"] = pd.Categorical(
df["KSSLR6MAX_ABS"],
categories=KSSLR6_LABELS,
ordered=True
)
df["KSSLR6MAX_ABS_ENC"] = df["KSSLR6MAX_ABS"].cat.codes
print("Missing KSSLR6MAX_ABS:", df["KSSLR6MAX_ABS"].isna().sum())
print(
df["KSSLR6MAX_ABS"]
.value_counts()
.sort_index()
)
Missing KSSLR6MAX_ABS: 0
KSSLR6MAX_ABS
0-6 27458
07-12 8847
13-18 5643
19-24 3185
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["KSSLR6MAX"])
"KSSLR6MAX" in df.columns
False
RCVYMHPRB
"""RCVYMHPRB Len : 1 RC-PERCEIVED RECOVERY FROM MENTAL HEALTH ISSUE
Freq Pct
. = Aged 12-17/Unknown (Otherwise)................................................................................ 13018 22.96
0 = No/No Issue (CAMHRCVR=2 OR CAMHPROB2=0)................................................ 34775 61.33
1 = Yes (CAMHRCVR=1) ................................................................................................. 8912 15.72"""
'RCVYMHPRB Len : 1 RC-PERCEIVED RECOVERY FROM MENTAL HEALTH ISSUE\nFreq Pct\n. = Aged 12-17/Unknown (Otherwise)................................................................................ 13018 22.96\n0 = No/No Issue (CAMHRCVR=2 OR CAMHPROB2=0)................................................ 34775 61.33\n1 = Yes (CAMHRCVR=1) ................................................................................................. 8912 15.72'
df["RCVYMHPRB"].value_counts(dropna=False)
RCVYMHPRB
0.0 34775
1.0 8912
NaN 1446
Name: count, dtype: int64
# Drop rows with missing RCVYMHPRB
df = df.dropna(subset=["RCVYMHPRB"])
df["RCVYMHPRB"].value_counts(dropna=False)
RCVYMHPRB
0.0 34775
1.0 8912
Name: count, dtype: int64
IRIMPGOUT
"""IRIMPGOUT Len : 2 DIFFICULTY GOING OUT ONE MO IN PST 12 MOS - IMP REV
Freq Pct
1 = No difficulty ................................................................................................................. 20700 36.50
2 = Mild difficulty............................................................................................................... 7015 12.37
3 = Moderate difficulty ....................................................................................................... 4505 7.94
4 = Severe difficulty............................................................................................................ 2059 3.63
5 = You didn't leave the house on your own ....................................................................... 1188 2.10
99 = LEGITIMATE SKIP................................................................................................... 21238 37.45"""
"IRIMPGOUT Len : 2 DIFFICULTY GOING OUT ONE MO IN PST 12 MOS - IMP REV\nFreq Pct\n1 = No difficulty ................................................................................................................. 20700 36.50\n2 = Mild difficulty............................................................................................................... 7015 12.37\n3 = Moderate difficulty ....................................................................................................... 4505 7.94\n4 = Severe difficulty............................................................................................................ 2059 3.63\n5 = You didn't leave the house on your own ....................................................................... 1188 2.10\n99 = LEGITIMATE SKIP................................................................................................... 21238 37.45"
# IRIMPGOUT Abstracted
IRIMPGOUT_MAP = {
1: "b_No difficulty",
2: "c_Mild difficulty",
3: "d_Moderate difficulty",
4: "e_Severe difficulty",
5: "a_Not-Going-Out-Alone-Or-No-Difficulty",
99: "a_Not-Going-Out-Alone-Or-No-Difficulty",
}
IRIMPGOUT_ORDER = [
"a_Not-Going-Out-Alone-Or-No-Difficulty",
"b_No difficulty",
"c_Mild difficulty",
"d_Moderate difficulty",
"e_Severe difficulty",
]
df["IRIMPGOUT_ABS"] = df["IRIMPGOUT"].map(IRIMPGOUT_MAP)
# Collapse skip logic / NA into the combined class
df["IRIMPGOUT_ABS"] = df["IRIMPGOUT_ABS"].fillna(
"a_Not-Going-Out-Alone-Or-No-Difficulty"
)
# Set categorical
df["IRIMPGOUT_ABS"] = pd.Categorical(
df["IRIMPGOUT_ABS"],
categories=IRIMPGOUT_ORDER,
ordered=True
)
# Sanity checks
print("IRIMPGOUT_ABS missing:", df["IRIMPGOUT_ABS"].isna().sum())
print(df["IRIMPGOUT_ABS"].value_counts().sort_index())
IRIMPGOUT_ABS missing: 0
IRIMPGOUT_ABS
a_Not-Going-Out-Alone-Or-No-Difficulty 10583
b_No difficulty 20070
c_Mild difficulty 6726
d_Moderate difficulty 4315
e_Severe difficulty 1993
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["IRIMPGOUT"])
"IRIMPGOUT" in df.columns
False
WHODASDASC
"""WHODASDASC Len : 2 RC-ALTERNATIVE WHODAS SCORE
Freq Pct
RANGE = 0 - 8................................................................................................................... 45133 79.59
. = Aged 12-17 .................................................................................................................... 11572 20.41"""
'WHODASDASC Len : 2 RC-ALTERNATIVE WHODAS SCORE\nFreq Pct\nRANGE = 0 - 8................................................................................................................... 45133 79.59\n. = Aged 12-17 .................................................................................................................... 11572 20.41'
# WHODASDASC Abstracted
WHODASDASC_MAP = {
0: "0",
1: "1-2",
2: "1-2",
3: "3-4",
4: "3-4",
5: "5-6",
6: "5-6",
7: "7-8",
8: "7-8",
}
WHODASDASC_ORDER = [
"0",
"1-2",
"3-4",
"5-6",
"7-8",
]
df["WHODASDASC_ABS"] = df["WHODASDASC"].map(WHODASDASC_MAP)
df["WHODASDASC_ABS"] = pd.Categorical(
df["WHODASDASC_ABS"],
categories=WHODASDASC_ORDER,
ordered=True
)
# Sanity checks
print("WHODASDASC_ABS missing:", df["WHODASDASC_ABS"].isna().sum())
print(df["WHODASDASC_ABS"].value_counts().sort_index())
WHODASDASC_ABS missing: 0
WHODASDASC_ABS
0 27951
1-2 5588
3-4 3708
5-6 3039
7-8 3401
Name: count, dtype: int64
# Drop the original variable
df = df.drop(columns=["WHODASDASC"])
print("WHODASDASC" in df.columns)
False
COCLALCUSE
"""COCLALCUSE Len : 1 RC-COLLAPSED HOW COVID-19 AFFECTED AMOUNT ALCOHOL DRANK
Freq Pct
. = Unknown/No PY Alcohol Use (Otherwise)................................................................... 26038 45.92
1 = Drink much less or little less (COALCUSE=1,2) ......................................................... 8670 15.29
2 = Drink about the same (COALCUSE=3)........................................................................ 18142 31.99
3 = Drink a little more or much more (COALCUSE=4,5) .................................................. 3855 6.80"""
'COCLALCUSE Len : 1 RC-COLLAPSED HOW COVID-19 AFFECTED AMOUNT ALCOHOL DRANK\nFreq Pct\n. = Unknown/No PY Alcohol Use (Otherwise)................................................................... 26038 45.92\n1 = Drink much less or little less (COALCUSE=1,2) ......................................................... 8670 15.29\n2 = Drink about the same (COALCUSE=3)........................................................................ 18142 31.99\n3 = Drink a little more or much more (COALCUSE=4,5) .................................................. 3855 6.80'
# Map original values to semantic labels
COCLALCUSE_MAP = {
1: "Drink much less or little less",
2: "Drink about the same",
3: "Drink a little more or much more",
}
df["COCLALCUSE_ABS"] = df["COCLALCUSE"].map(COCLALCUSE_MAP)
# Explicitly assign missing / skip logic
df["COCLALCUSE_ABS"] = df["COCLALCUSE_ABS"].fillna(
"Unknown/No PY Alcohol Use"
)
# COCLALCUSE Abstracted
COCLALCUSE_LABEL_MAP = {
"Unknown/No PY Alcohol Use": "a_Unknown/No PY Alcohol Use",
"Drink much less or little less": "b_Drink much less or little less",
"Drink about the same": "c_Drink about the same",
"Drink a little more or much more": "d_Drink a little more or much more",
}
df["COCLALCUSE_ABS"] = df["COCLALCUSE_ABS"].map(COCLALCUSE_LABEL_MAP)
COCLALCUSE_ORDER = [
"a_Unknown/No PY Alcohol Use",
"b_Drink much less or little less",
"c_Drink about the same",
"d_Drink a little more or much more",
]
df["COCLALCUSE_ABS"] = pd.Categorical(
df["COCLALCUSE_ABS"],
categories=COCLALCUSE_ORDER,
ordered=True
)
print("COCLALCUSE_ABS missing:", df["COCLALCUSE_ABS"].isna().sum())
print(df["COCLALCUSE_ABS"].value_counts().sort_index())
print(df["COCLALCUSE_ABS"].dtype)
COCLALCUSE_ABS missing: 0
COCLALCUSE_ABS
a_Unknown/No PY Alcohol Use 14755
b_Drink much less or little less 8023
c_Drink about the same 17447
d_Drink a little more or much more 3462
Name: count, dtype: int64
category
df = df.drop(columns=["COCLALCUSE"])
OUTPUT_PATH = "../data/prepared/NSDUH_2023_prepared.pkl"
df.to_pickle(OUTPUT_PATH)
print(f"Prepared dataset saved to: {OUTPUT_PATH}")
print(f"Final shape: {df.shape}")
Prepared dataset saved to: ../data/prepared/NSDUH_2023_prepared.pkl
Final shape: (43687, 2306)