Data Loading & Initial Cleaning
# Import Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
/home/shezin/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).
from pandas.core.computation.check import NUMEXPR_INSTALLED
/usr/local/lib/python3.8/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.4
warnings.warn(f"A NumPy version >={np_minversion} and <{np_maxversion}"
# Import NSUDH dataset
df = pd.read_csv("../data/raw/NSDUH_2023_Tab.txt",
sep="\t",
low_memory=False)
# Checking Data
df.head
<bound method NDFrame.head of QUESTID2 FILEDATE ANALWT2_C VESTR_C VEREP PDEN10 COUTYP4 \
0 10000053 03/25/2025 3276.469874 40031 2 2 2
1 10000679 03/25/2025 15630.082955 40021 2 2 3
2 10001208 03/25/2025 4018.172390 40043 1 2 2
3 10001260 03/25/2025 10711.709540 40030 2 2 2
4 10001588 03/25/2025 8195.104779 40023 2 2 2
... ... ... ... ... ... ... ...
56700 50556120 03/25/2025 417.630119 40018 2 2 2
56701 50557151 03/25/2025 7625.717934 40017 1 2 2
56702 50558694 03/25/2025 23556.083908 40042 1 1 1
56703 50558696 03/25/2025 5193.882625 40040 2 1 1
56704 50558785 03/25/2025 2676.234296 40027 1 2 2
MAIIN102 AIIND102 AGE3 ... COSUTELE2 COSUAPTDL2 COSURXDL2 \
0 2 2 10 ... 3.0 3.0 3.0
1 2 2 9 ... 3.0 3.0 3.0
2 2 2 9 ... 3.0 3.0 3.0
3 2 2 1 ... 2.0 2.0 2.0
4 2 2 10 ... 3.0 3.0 3.0
... ... ... ... ... ... ... ...
56700 2 2 9 ... NaN NaN NaN
56701 2 2 11 ... 3.0 3.0 3.0
56702 2 2 10 ... 2.0 2.0 2.0
56703 2 2 11 ... 3.0 3.0 3.0
56704 2 2 3 ... 2.0 2.0 2.0
COSUSVHLT2 COHCTELE2 COHCAPTDL2 COHCRXDL2 COHCSVHLT2 LANGVER \
0 3.0 3.0 3.0 3.0 3.0 1
1 3.0 3.0 3.0 3.0 3.0 1
2 3.0 3.0 3.0 3.0 3.0 1
3 2.0 2.0 2.0 2.0 2.0 1
4 3.0 1.0 3.0 3.0 3.0 1
... ... ... ... ... ... ...
56700 NaN NaN NaN NaN NaN 2
56701 3.0 2.0 2.0 2.0 2.0 1
56702 2.0 2.0 2.0 2.0 2.0 1
56703 3.0 2.0 2.0 2.0 2.0 1
56704 2.0 2.0 2.0 2.0 2.0 1
GQTYPE2
0 NaN
1 NaN
2 NaN
3 NaN
4 NaN
... ...
56700 NaN
56701 NaN
56702 NaN
56703 NaN
56704 NaN
[56705 rows x 2636 columns]>
missing_summary = df.isna().mean() * 100
missing_summary.sort_values(ascending=False)
SRCCLFRSED 99.871264
SRCFRSEDNM 99.871264
GQTYPE2 99.850101
SRCSEDNM2 99.728419
SRCCLFRTRQ 99.370426
...
IRCRKAGE 0.000000
IICRKAGE 0.000000
IRCRKYFU 0.000000
IICRKYFU 0.000000
UDALBLCKCTD 0.000000
Length: 2636, dtype: float64
# Checking for duplicate rows
df.duplicated().sum()
0
# Filtering dataset for relevant age group (18 and above)
'''I used AGE3 instead of CATAG7 because AGE3 is the final edited age variable in NSDUH. It’s the most accurate age measure because it incorporates multiple consistency checks—birthdate, roster age, screener age, and internal corrections. CATAG7 is just a categorical recode derived from AGE3. For filtering out respondents under 18, it’s better to filter at the source (AGE3) and let the model work with the true final age before any recoding.
AGE3 Len : 2 RECODE - FINAL EDITED AGE
Freq Pct
1 = Respondent is 12 or 13 years old
2 = Respondent is 14 or 15 years old
3 = Respondent is 16 or 17 years old
4 = Respondent is between 18 and 20 years old
5 = Respondent is between 21 and 23 years old
6 = Respondent is 24 or 25 years old
7 = Respondent is between 26 and 29 years old
8 = Respondent is between 30 and 34 years old
9 = Respondent is between 35 and 49 years old
10 = Respondent is between 50 and 64 years old
11 = Respondent is 65 years old or older'''
df_no_age = df[df['AGE3'] >= 4]
df_no_age.shape
(45133, 2636)
# Checking how many error codes are present in the dataset
'''Code Meaning Should Convert to NaN?
94 / 994 / 9994 Don't know
97 / 997 / 9997 Refused
98 / 998 / 9998 Blank / Not answered
85 / 985 / 9985 Bad / inconsistent data
99 / 999 / 9999 Legitimate skip
89 / 989 / 9989 Legitimate skip - logically assigned'''
nan_codes = [
94, 97, 98, 85,
994, 997, 998, 985,
9994, 9997, 9998, 9985,
99, 999, 9999,
89, 989, 9989
]
for code in nan_codes:
count = (df_no_age == code).sum().sum()
print(code, count)
94 72183
97 44145
98 1898217
85 9487
994 7015
997 3857
998 33083
985 5817
9994 4675
9997 1899
9998 20242
9985 1774
99 17222835
999 1267661
9999 1453080
89 4185
989 154
9989 1332
# Converting the following error codes to NaN
nan_codes = [
94, 97, 98, 85,
994, 997, 998, 985,
9994, 9997, 9998, 9985,
99, 999, 9999,
89, 989, 9989
]
df_ErrorCode_NaN = df_no_age.replace(nan_codes, np.nan)
nan_codes = [
94, 97, 98, 85,
994, 997, 998, 985,
9994, 9997, 9998, 9985,
99, 999, 9999,
89, 989, 9989
]
for code in nan_codes:
count = (df_ErrorCode_NaN == code).sum().sum()
print(code, count)
94 0
97 0
98 0
85 0
994 0
997 0
998 0
985 0
9994 0
9997 0
9998 0
9985 0
99 0
999 0
9999 0
89 0
989 0
9989 0
# Checking for any NaNs in the Target Variable (IRAMDEYR)
df_ErrorCode_NaN['IRAMDEYR'].isna().sum()
0
# 2. DROP rows where the target is NaN
df_dropped_targetNaN = df_ErrorCode_NaN.dropna(subset=['IRAMDEYR'])
before = df.shape[0]
after = df_dropped_targetNaN.shape[0]
print("Rows before:", before)
print("Rows after :", after)
print("Rows removed:", before - after)
Rows before: 56705
Rows after : 45133
Rows removed: 11572
df_dropped_targetNaN.describe()
|
QUESTID2 |
ANALWT2_C |
VESTR_C |
VEREP |
PDEN10 |
COUTYP4 |
MAIIN102 |
AIIND102 |
AGE3 |
SERVICE |
... |
COMHSVHLT2 |
COSUTELE2 |
COSUAPTDL2 |
COSURXDL2 |
COSUSVHLT2 |
COHCTELE2 |
COHCAPTDL2 |
COHCRXDL2 |
COHCSVHLT2 |
LANGVER |
| count |
4.513300e+04 |
45133.000000 |
45133.000000 |
45133.000000 |
45133.000000 |
45133.000000 |
45133.000000 |
45133.000000 |
45133.000000 |
45108.000000 |
... |
43333.000000 |
43354.000000 |
43364.000000 |
43362.000000 |
43347.000000 |
43378.000000 |
43375.000000 |
43380.000000 |
43347.00000 |
45133.000000 |
| mean |
3.027962e+07 |
5706.297779 |
40025.494849 |
1.429774 |
1.628631 |
1.713602 |
1.984579 |
1.984357 |
7.816697 |
1.948967 |
... |
2.410772 |
2.533884 |
2.537450 |
2.563004 |
2.578518 |
1.968809 |
2.036749 |
2.196358 |
2.26424 |
1.042519 |
| std |
1.172008e+07 |
8511.336263 |
14.499402 |
0.495049 |
0.586271 |
0.730958 |
0.123222 |
0.124090 |
2.194038 |
0.220068 |
... |
0.592895 |
0.618880 |
0.603669 |
0.563047 |
0.542894 |
0.761610 |
0.720678 |
0.611436 |
0.56382 |
0.201772 |
| min |
1.000005e+07 |
1.516955 |
40001.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
4.000000 |
1.000000 |
... |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
1.00000 |
1.000000 |
| 25% |
2.013831e+07 |
973.740425 |
40013.000000 |
1.000000 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
6.000000 |
2.000000 |
... |
2.000000 |
2.000000 |
2.000000 |
2.000000 |
2.000000 |
1.000000 |
2.000000 |
2.000000 |
2.00000 |
1.000000 |
| 50% |
3.027958e+07 |
2697.651023 |
40025.000000 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.000000 |
8.000000 |
2.000000 |
... |
2.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
1.000000 |
| 75% |
4.039094e+07 |
6788.874141 |
40038.000000 |
2.000000 |
2.000000 |
2.000000 |
2.000000 |
2.000000 |
9.000000 |
2.000000 |
... |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.00000 |
1.000000 |
| max |
5.055870e+07 |
118941.430150 |
40050.000000 |
2.000000 |
3.000000 |
3.000000 |
2.000000 |
2.000000 |
11.000000 |
2.000000 |
... |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.000000 |
3.00000 |
2.000000 |
8 rows × 2634 columns
df_dropped_targetNaN.isna().sum().sort_values(ascending=False)
YMDEIMAD5YR 45133
YUCOSUIPLN2 45133
YMSUD5YANY 45133
YMIUD5YANY 45133
YMDEAUD5YR 45133
...
OXYCNANYYR 0
PNRANYYR 0
PNRANYFLAG 0
METHAMMON 0
QUESTID2 0
Length: 2636, dtype: int64
import re
# Patterns for columns to drop (but NOT ANALWT2_C)
drop_patterns = [
r'^CASEID', r'^QUESTID', r'^QUESTID2', r'^PANEL', r'^VERSION',
r'^INTV', r'^FILE', r'^YEAR',
r'^WT', r'WGT', r'WEIGHT', r'ANALWT(?!2_C$)', # drop all WTs except ANALWT2_C
r'^VESTR', r'^VEREP', r'^ESTRAT',
r'_A$', r'_E$', r'_ORIG$', r'_R$', r'_RC$'
]
cols_to_drop = [
c for c in df_dropped_targetNaN.columns
if any(re.search(pat, c) for pat in drop_patterns)
]
# Ensure ANALWT2_C stays
cols_to_drop = [c for c in cols_to_drop if c != "ANALWT2_C"]
# Manual additional drops
manual_drop = [
"WTANSWER", "WTPOUND2", "FILEDATE"
]
cols_to_drop = list(set(cols_to_drop + [col for col in manual_drop
if col in df_dropped_targetNaN.columns]))
# Remove duplicates
cols_to_drop = list(set(cols_to_drop))
print("Dropping", len(cols_to_drop), "columns.")
df_Removed_Columns = df_dropped_targetNaN.drop(columns=cols_to_drop, errors='ignore')
Dropping 6 columns.
# Checking to see which variables have >99% missingess
na_percent = df_Removed_Columns.isna().mean() * 100
# Filter columns with > 99% missing
cols_over_99 = na_percent[na_percent > 99]
# Print count
print("Number of variables with >99% missingness:", len(cols_over_99))
# Print the variable names
print("\nVariables with >99% missingness:")
print(cols_over_99.index.tolist())
# Optional: show top 20 instead of the whole list
print("\nPreview (first 20):")
print(cols_over_99.head(20))
Number of variables with >99% missingness: 282
Variables with >99% missingness:
['EDUSCKEST', 'EDUSKPEST', 'HLCALLFG', 'HLCALL99', 'BKPOSTOB', 'BKOTHOF2', 'ALTOTFG', 'ALFQFLG', 'ALDYSFG', 'MJFQFLG', 'BLRECFL2', 'BLNT30C1', 'BLNT30C2', 'RSNOMRJ', 'RSNMRJMO', 'CCTOTFG', 'CCFQFLG', 'CRTOTFG', 'CRFQFLG', 'HRTOTFG', 'HRFQFLG', 'HALTOTFG', 'HALFQFLG', 'INHTOTFG', 'INHFQFLG', 'METOTFG', 'MEFQFLG', 'PNRNORXFG', 'TRQNORXFG', 'STMNORXFG', 'SEDNORXFG', 'SRCSEDNM2', 'SRCFRPNRNM', 'SRCFRTRQNM', 'SRCFRSEDNM', 'SRCCLFRPNR', 'SRCCLFRTRQ', 'SRCCLFRSED', 'OTCFLAG', 'OTDGNDLA', 'OTDGNDLB', 'OTDGNDLC', 'OTDGNDLD', 'OTDGNDLE', 'SUNTINSCV', 'SUNTENCV', 'SUNTNOOPN', 'YEATNDYR', 'YEHMSLYR', 'YESCHFLT', 'YESCHWRK', 'YESCHIMP', 'YESCHINT', 'YETCGJOB', 'YELSTGRD', 'YECIGFRNDOF2', 'YECIGNEXTYR2', 'YESTSCIG', 'YESTSMJ', 'YESTSALC', 'YESTSDNK', 'YEPCHKHW', 'YEPHLPHW', 'YEPCHORE', 'YEPLMTTV', 'YEPLMTSN', 'YEPGDJOB', 'YEPPROUD', 'YEYARGUP', 'YEYFGTSW', 'YEYFGTGP', 'YEYHGUN', 'YEYSELL', 'YEYSTOLE', 'YEYATTAK', 'YEPPKCIG', 'YEPMJEVR', 'YEPMJMO', 'YEPALDLY', 'YEGPKCIG', 'YEGMJEVR', 'YEGMJMO', 'YEGALDLY', 'YEFPKCIG', 'YEFMJEVR', 'YEFMJMO', 'YEFALDLY', 'YETLKNON', 'YETLKPAR', 'YETLKBGF', 'YETLKOTA', 'YETLKSOP', 'YEPRTDNG', 'YEPRBSLV', 'YEVIOPRV', 'YEDGPRGP', 'YESLFHLP', 'YEPRGSTD', 'YESCHACT', 'YECOMACT', 'YEFAIACT', 'YEOTHACT', 'YEDECLAS', 'YEDERGLR', 'YEDESPCL', 'YEPVNTYR', 'YERLGSVC', 'YERLGIMP', 'YERLDCSN', 'YERLFRND', 'YUSUITHK', 'YUCOSUITHK', 'YUSUIPLN', 'YUCOSUIPLN', 'SCHFELT', 'TCHGJOB', 'AVGGRADE', 'STNDSCIG', 'STNDSMJ', 'STNDALC', 'STNDDNK', 'PARCHKHW', 'PARHLPHW', 'PRCHORE2', 'PRLMTTV2', 'PARLMTSN', 'PRGDJOB2', 'PRPROUD2', 'ARGUPAR', 'YOFIGHT2', 'YOGRPFT2', 'YOHGUN2', 'YOSELL2', 'YOSTOLE2', 'YOATTAK2', 'PRPKCIG2', 'PRMJEVR2', 'PRMJMO', 'PRALDLY2', 'YFLPKCG2', 'YFLTMRJ2', 'YFLMJMO', 'YFLADLY2', 'FRDPCIG2', 'FRDMEVR2', 'FRDMJMON', 'FRDADLY2', 'TALKPROB', 'PRTALK3', 'PRBSOLV2', 'PREVIOL2', 'PRVDRGO2', 'GRPCNSL2', 'PREGPGM2', 'YTHACT2', 'DRPRVME3', 'ANYEDUC3', 'RLGATTD', 'RLGIMPT', 'RLGDCSN', 'RLGFRND', 'YUSUITHKYR', 'YUCOSUITHK2', 'YUSUIPLNYR', 'YUCOSUIPLN2', 'YODPREV', 'YODSCEV', 'YOLOSEV', 'YODPDISC', 'YODPLSIN', 'YODSLSIN', 'YOLSI2WK', 'YODPR2WK', 'YOWRHRS', 'YOWRDST', 'YOWRCHR', 'YOWRIMP', 'YODPPROB', 'YOWRPROB', 'YOWRAGE', 'YOWRDEPR', 'YOWRDISC', 'YOWRLSIN', 'YOWRPLSR', 'YOWRELES', 'YOWREMOR', 'YOWRGAIN', 'YOWRGROW', 'YOWRPREG', 'YOWRGNL2', 'YOWRLOSE', 'YOWRDIET', 'YOWRLSL2', 'YOWRSLEP', 'YOWRSMOR', 'YOWRENRG', 'YOWRSLOW', 'YOWRSLNO', 'YOWRJITT', 'YOWRJINO', 'YOWRTHOT', 'YOWRCONC', 'YOWRDCSN', 'YOWRNOGD', 'YOWRWRTH', 'YO_MDEA1', 'YO_MDEA2', 'YO_MDEA3', 'YO_MDEA4', 'YO_MDEA5', 'YO_MDEA6', 'YO_MDEA7', 'YO_MDEA8', 'YODSMMDE', 'YOPBINTF', 'YOPBDLYA', 'YOPBRMBR', 'YOPBAGE', 'YOPBNUM', 'YOPB2WK', 'YOPSHMGT', 'YOPSWORK', 'YOPSRELS', 'YOPSSOC', 'YOPSDAYS', 'YOSEEDOC', 'YOFAMDOC', 'YOOTHDOC', 'YOPSYCH', 'YOPSYMD', 'YOSOCWRK', 'YOCOUNS', 'YOOTHMHP', 'YONURSE', 'YORELIG', 'YOHERBAL', 'YOOTHHLP', 'YOTMTNOW', 'YORX12MO', 'YORXNOW', 'YORXHLP', 'YOTMTHLP', 'YMDELT', 'YMDEYR', 'YMDEAUD5YR', 'YMIUD5YANY', 'YMSUD5YANY', 'YMDERSUD5ANY', 'YMDESUD5ANYO', 'YTXMDEYR', 'YRXMDEYR', 'YMDETXRX', 'YDOCMDE', 'YOMDMDE', 'YPSY1MDE', 'YPSY2MDE', 'YSOCMDE', 'YCOUNMDE', 'YOMHMDE', 'YNURSMDE', 'YRELMDE', 'YHBCHMDE', 'YHLTMDE', 'YALTMDE', 'YMDEHPRX', 'YMDEHPO', 'YMDERXO2', 'YMDEHARX', 'YSDSHOME', 'YSDSWRK', 'YSDSREL', 'YSDSSOC', 'YSDSOVRL', 'MDEIMPY', 'YMDEIMAD5YR', 'YMIMI5YANY', 'YMIMR5YANY', 'YMIMS5YANY', 'CIRROSAGE', 'HEPBCAGE', 'HIVAIDSAG', 'GQTYPE2']
Preview (first 20):
EDUSCKEST 99.915804
EDUSKPEST 99.960118
HLCALLFG 99.953471
HLCALL99 99.953471
BKPOSTOB 100.000000
BKOTHOF2 99.519199
ALTOTFG 99.339729
ALFQFLG 99.295416
ALDYSFG 99.960118
MJFQFLG 99.550218
BLRECFL2 99.915804
BLNT30C1 99.884785
BLNT30C2 99.802805
RSNOMRJ 99.922451
RSNMRJMO 99.898079
CCTOTFG 99.995569
CCFQFLG 99.953471
CRTOTFG 99.995569
CRFQFLG 99.991137
HRTOTFG 99.986706
dtype: float64
df_99 = df_Removed_Columns.drop(columns=cols_over_99.index)
df_99.shape
(45133, 2348)
# Removing target leakage features that was used to impute / recode the target IRAMDEYR
# IRAMDEYR → AMDEYR → AMDELT → ADSMMDEA → MDE symptom items + fallback items + timing
leakage_vars = [
# Target + imputation
"IRAMDEYR", "IIAMDEYR",
# Pre-imputation past-year MDE
"AMDEYR",
# Lifetime MDE + imputation
"AMDELT", "IRAMDELT", "IIAMDELT",
# MDE with impairment (logically dependent on AMDEYR)
"AMDEIMP", "IRAMDEIMP", "IIAMDEIMP", "AMDETXRX",
# DSM summary
"ADSMMDEA",
# Raw DSM symptom items (if present)
"D_MDEA1", "D_MDEA2", "D_MDEA3", "D_MDEA4", "D_MDEA5",
"D_MDEA6", "D_MDEA7", "D_MDEA8", "D_MDEA9",
# Symptom recodes (one-digit suffixes, if present)
"AD_MDEA1", "AD_MDEA2", "AD_MDEA3", "AD_MDEA4",
"AD_MDEA5", "AD_MDEA6", "AD_MDEA7", "AD_MDEA8",
# Symptom recodes (true DSM flags: two-digit suffix)
"AD_MDEA11", "AD_MDEA21", "AD_MDEA31", "AD_MDEA41",
"AD_MDEA51", "AD_MDEA61", "AD_MDEA71", "AD_MDEA81", "AD_MDEA91",
# Fallback/gating items
"ADPB2WK",
"ADDPREV", "ADDSCEV", "ADLOSEV", "ADLSI2WK", "ADDPR2WK",
"ADWRHRS", "ADWRDST", "ADWRCHR", "ADWRIMP", "ADDPPROB",
# Soft-leakage variables
"ARXMDEYR", # received prescription meds
"ATXMDEYR", # received counseling/therapy
"AHLTMDE", # told by provider they have depression
]
print(leakage_vars)
len(leakage_vars)
['IRAMDEYR', 'IIAMDEYR', 'AMDEYR', 'AMDELT', 'IRAMDELT', 'IIAMDELT', 'AMDEIMP', 'IRAMDEIMP', 'IIAMDEIMP', 'AMDETXRX', 'ADSMMDEA', 'D_MDEA1', 'D_MDEA2', 'D_MDEA3', 'D_MDEA4', 'D_MDEA5', 'D_MDEA6', 'D_MDEA7', 'D_MDEA8', 'D_MDEA9', 'AD_MDEA1', 'AD_MDEA2', 'AD_MDEA3', 'AD_MDEA4', 'AD_MDEA5', 'AD_MDEA6', 'AD_MDEA7', 'AD_MDEA8', 'AD_MDEA11', 'AD_MDEA21', 'AD_MDEA31', 'AD_MDEA41', 'AD_MDEA51', 'AD_MDEA61', 'AD_MDEA71', 'AD_MDEA81', 'AD_MDEA91', 'ADPB2WK', 'ADDPREV', 'ADDSCEV', 'ADLOSEV', 'ADLSI2WK', 'ADDPR2WK', 'ADWRHRS', 'ADWRDST', 'ADWRCHR', 'ADWRIMP', 'ADDPPROB', 'ARXMDEYR', 'ATXMDEYR', 'AHLTMDE']
51
# Drop leakage variables except the target itself
leakage_to_drop = [col for col in leakage_vars if col in df_99.columns and col != "IRAMDEYR"]
df_clean = df_99.drop(columns=leakage_to_drop, errors="ignore")
df_clean .shape
(45133, 2316)
# Summary statistics
print("Final cleaned df shape:", df_clean.shape)
df_clean.info()
df_clean.describe(include='all').transpose().head(20)
Final cleaned df shape: (45133, 2316)
<class 'pandas.core.frame.DataFrame'>
Index: 45133 entries, 0 to 56703
Columns: 2316 entries, ANALWT2_C to LANGVER
dtypes: float64(1528), int64(788)
memory usage: 797.8 MB
|
count |
mean |
std |
min |
25% |
50% |
75% |
max |
| ANALWT2_C |
45133.0 |
5706.297779 |
8511.336263 |
1.516955 |
973.740425 |
2697.651023 |
6788.874141 |
118941.43015 |
| PDEN10 |
45133.0 |
1.628631 |
0.586271 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
3.00000 |
| COUTYP4 |
45133.0 |
1.713602 |
0.730958 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
3.00000 |
| MAIIN102 |
45133.0 |
1.984579 |
0.123222 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| AIIND102 |
45133.0 |
1.984357 |
0.124090 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| AGE3 |
45133.0 |
7.816697 |
2.194038 |
4.000000 |
6.000000 |
8.000000 |
9.000000 |
11.00000 |
| SERVICE |
45108.0 |
1.948967 |
0.220068 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| MILSTAT |
2294.0 |
2.934612 |
0.247263 |
2.000000 |
3.000000 |
3.000000 |
3.000000 |
3.00000 |
| ACTDEVER |
2299.0 |
1.233145 |
0.422926 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
2.00000 |
| ACTD2001 |
1756.0 |
1.561503 |
0.496344 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
2.00000 |
| ACTD9001 |
1756.0 |
1.771071 |
0.420263 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| ACTD7590 |
1756.0 |
1.765376 |
0.423884 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| ACTDVIET |
1755.0 |
1.719658 |
0.449294 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
2.00000 |
| ACTDPRIV |
1754.0 |
1.952109 |
0.213596 |
1.000000 |
2.000000 |
2.000000 |
2.000000 |
2.00000 |
| COMBATPY |
1756.0 |
1.586560 |
0.492591 |
1.000000 |
1.000000 |
2.000000 |
2.000000 |
2.00000 |
| NOMARR2 |
24284.0 |
1.202355 |
0.401764 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
2.00000 |
| HEALTH |
45119.0 |
2.399366 |
0.973482 |
1.000000 |
2.000000 |
2.000000 |
3.000000 |
5.00000 |
| MOVSINPYR2 |
43538.0 |
0.347306 |
0.689924 |
0.000000 |
0.000000 |
0.000000 |
0.000000 |
3.00000 |
| SEXATRACT2 |
43487.0 |
1.485019 |
1.124561 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
6.00000 |
| SEXIDENT22 |
43410.0 |
1.489887 |
1.210123 |
1.000000 |
1.000000 |
1.000000 |
1.000000 |
6.00000 |
# Save cleaned dataset
df_clean.to_pickle("../data/cleaned/NSDUH_2023_clean.pkl")