{ "cells": [ { "cell_type": "markdown", "id": "961e18c7", "metadata": {}, "source": [ "# Data Loading & Initial Cleaning" ] }, { "cell_type": "code", "execution_count": 1, "id": "90b87909", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/shezin/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n", "/usr/local/lib/python3.8/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.4\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n" ] } ], "source": [ "\n", "# Import Libraries\n", "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns" ] }, { "cell_type": "code", "execution_count": 2, "id": "a3786215", "metadata": {}, "outputs": [], "source": [ "# Import NSUDH dataset\n", "df = pd.read_csv(\"../data/raw/NSDUH_2023_Tab.txt\", \n", " sep=\"\\t\", \n", " low_memory=False)" ] }, { "cell_type": "code", "execution_count": 3, "id": "b17fa3f8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Checking Data\n", "df.head" ] }, { "cell_type": "code", "execution_count": 4, "id": "ce756b8c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "SRCCLFRSED 99.871264\n", "SRCFRSEDNM 99.871264\n", "GQTYPE2 99.850101\n", "SRCSEDNM2 99.728419\n", "SRCCLFRTRQ 99.370426\n", " ... \n", "IRCRKAGE 0.000000\n", "IICRKAGE 0.000000\n", "IRCRKYFU 0.000000\n", "IICRKYFU 0.000000\n", "UDALBLCKCTD 0.000000\n", "Length: 2636, dtype: float64" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "missing_summary = df.isna().mean() * 100\n", "missing_summary.sort_values(ascending=False)" ] }, { "cell_type": "code", "execution_count": 5, "id": "6247efb1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Checking for duplicate rows\n", "df.duplicated().sum()" ] }, { "cell_type": "code", "execution_count": 6, "id": "6e5829ae", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(45133, 2636)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Filtering dataset for relevant age group (18 and above)\n", "'''I used AGE3 instead of CATAG7 because AGE3 is the final edited age variable in NSDUH. It’s the most accurate age measure because it incorporates multiple consistency checks—birthdate, roster age, screener age, and internal corrections. CATAG7 is just a categorical recode derived from AGE3. For filtering out respondents under 18, it’s better to filter at the source (AGE3) and let the model work with the true final age before any recoding.\n", "\n", "AGE3 Len : 2 RECODE - FINAL EDITED AGE\n", "Freq Pct\n", "1 = Respondent is 12 or 13 years old\n", "2 = Respondent is 14 or 15 years old\n", "3 = Respondent is 16 or 17 years old\n", "4 = Respondent is between 18 and 20 years old\n", "5 = Respondent is between 21 and 23 years old\n", "6 = Respondent is 24 or 25 years old\n", "7 = Respondent is between 26 and 29 years old\n", "8 = Respondent is between 30 and 34 years old\n", "9 = Respondent is between 35 and 49 years old\n", "10 = Respondent is between 50 and 64 years old\n", "11 = Respondent is 65 years old or older'''\n", "\n", "df_no_age = df[df['AGE3'] >= 4] \n", "df_no_age.shape" ] }, { "cell_type": "code", "execution_count": 7, "id": "59e10e60", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "94 72183\n", "97 44145\n", "98 1898217\n", "85 9487\n", "994 7015\n", "997 3857\n", "998 33083\n", "985 5817\n", "9994 4675\n", "9997 1899\n", "9998 20242\n", "9985 1774\n", "99 17222835\n", "999 1267661\n", "9999 1453080\n", "89 4185\n", "989 154\n", "9989 1332\n" ] } ], "source": [ "# Checking how many error codes are present in the dataset\n", "'''Code\tMeaning\tShould Convert to NaN?\n", "94 / 994 / 9994\tDon't know\n", "97 / 997 / 9997\tRefused\n", "98 / 998 / 9998\tBlank / Not answered\n", "85 / 985 / 9985\tBad / inconsistent data\n", "99 / 999 / 9999\tLegitimate skip\t\n", "89 / 989 / 9989\tLegitimate skip - logically assigned'''\n", "\n", "nan_codes = [\n", " 94, 97, 98, 85,\n", " 994, 997, 998, 985,\n", " 9994, 9997, 9998, 9985,\n", " 99, 999, 9999,\n", " 89, 989, 9989\n", "]\n", "\n", "for code in nan_codes:\n", " count = (df_no_age == code).sum().sum()\n", " print(code, count)" ] }, { "cell_type": "code", "execution_count": 8, "id": "8c22802b", "metadata": {}, "outputs": [], "source": [ "# Converting the following error codes to NaN\n", "\n", "nan_codes = [\n", " 94, 97, 98, 85,\n", " 994, 997, 998, 985,\n", " 9994, 9997, 9998, 9985,\n", " 99, 999, 9999,\n", " 89, 989, 9989\n", "]\n", "\n", "df_ErrorCode_NaN = df_no_age.replace(nan_codes, np.nan)" ] }, { "cell_type": "code", "execution_count": 9, "id": "b7f8f370", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "94 0\n", "97 0\n", "98 0\n", "85 0\n", "994 0\n", "997 0\n", "998 0\n", "985 0\n", "9994 0\n", "9997 0\n", "9998 0\n", "9985 0\n", "99 0\n", "999 0\n", "9999 0\n", "89 0\n", "989 0\n", "9989 0\n" ] } ], "source": [ "nan_codes = [\n", " 94, 97, 98, 85,\n", " 994, 997, 998, 985,\n", " 9994, 9997, 9998, 9985,\n", " 99, 999, 9999,\n", " 89, 989, 9989\n", "]\n", "\n", "for code in nan_codes:\n", " count = (df_ErrorCode_NaN == code).sum().sum()\n", " print(code, count)" ] }, { "cell_type": "code", "execution_count": 10, "id": "9fb5f29d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Checking for any NaNs in the Target Variable (IRAMDEYR)\n", "df_ErrorCode_NaN['IRAMDEYR'].isna().sum()" ] }, { "cell_type": "code", "execution_count": 11, "id": "ee14e6bc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Rows before: 56705\n", "Rows after : 45133\n", "Rows removed: 11572\n" ] } ], "source": [ "# 2. DROP rows where the target is NaN\n", "df_dropped_targetNaN = df_ErrorCode_NaN.dropna(subset=['IRAMDEYR'])\n", "\n", "before = df.shape[0]\n", "after = df_dropped_targetNaN.shape[0]\n", "\n", "print(\"Rows before:\", before)\n", "print(\"Rows after :\", after)\n", "print(\"Rows removed:\", before - after)" ] }, { "cell_type": "code", "execution_count": 12, "id": "9214f5b3", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QUESTID2ANALWT2_CVESTR_CVEREPPDEN10COUTYP4MAIIN102AIIND102AGE3SERVICE...COMHSVHLT2COSUTELE2COSUAPTDL2COSURXDL2COSUSVHLT2COHCTELE2COHCAPTDL2COHCRXDL2COHCSVHLT2LANGVER
count4.513300e+0445133.00000045133.00000045133.00000045133.00000045133.00000045133.00000045133.00000045133.00000045108.000000...43333.00000043354.00000043364.00000043362.00000043347.00000043378.00000043375.00000043380.00000043347.0000045133.000000
mean3.027962e+075706.29777940025.4948491.4297741.6286311.7136021.9845791.9843577.8166971.948967...2.4107722.5338842.5374502.5630042.5785181.9688092.0367492.1963582.264241.042519
std1.172008e+078511.33626314.4994020.4950490.5862710.7309580.1232220.1240902.1940380.220068...0.5928950.6188800.6036690.5630470.5428940.7616100.7206780.6114360.563820.201772
min1.000005e+071.51695540001.0000001.0000001.0000001.0000001.0000001.0000004.0000001.000000...1.0000001.0000001.0000001.0000001.0000001.0000001.0000001.0000001.000001.000000
25%2.013831e+07973.74042540013.0000001.0000001.0000001.0000002.0000002.0000006.0000002.000000...2.0000002.0000002.0000002.0000002.0000001.0000002.0000002.0000002.000001.000000
50%3.027958e+072697.65102340025.0000001.0000002.0000002.0000002.0000002.0000008.0000002.000000...2.0000003.0000003.0000003.0000003.0000002.0000002.0000002.0000002.000001.000000
75%4.039094e+076788.87414140038.0000002.0000002.0000002.0000002.0000002.0000009.0000002.000000...3.0000003.0000003.0000003.0000003.0000003.0000003.0000003.0000003.000001.000000
max5.055870e+07118941.43015040050.0000002.0000003.0000003.0000002.0000002.00000011.0000002.000000...3.0000003.0000003.0000003.0000003.0000003.0000003.0000003.0000003.000002.000000
\n", "

8 rows × 2634 columns

\n", "
" ], "text/plain": [ " QUESTID2 ANALWT2_C VESTR_C VEREP PDEN10 \\\n", "count 4.513300e+04 45133.000000 45133.000000 45133.000000 45133.000000 \n", "mean 3.027962e+07 5706.297779 40025.494849 1.429774 1.628631 \n", "std 1.172008e+07 8511.336263 14.499402 0.495049 0.586271 \n", "min 1.000005e+07 1.516955 40001.000000 1.000000 1.000000 \n", "25% 2.013831e+07 973.740425 40013.000000 1.000000 1.000000 \n", "50% 3.027958e+07 2697.651023 40025.000000 1.000000 2.000000 \n", "75% 4.039094e+07 6788.874141 40038.000000 2.000000 2.000000 \n", "max 5.055870e+07 118941.430150 40050.000000 2.000000 3.000000 \n", "\n", " COUTYP4 MAIIN102 AIIND102 AGE3 SERVICE \\\n", "count 45133.000000 45133.000000 45133.000000 45133.000000 45108.000000 \n", "mean 1.713602 1.984579 1.984357 7.816697 1.948967 \n", "std 0.730958 0.123222 0.124090 2.194038 0.220068 \n", "min 1.000000 1.000000 1.000000 4.000000 1.000000 \n", "25% 1.000000 2.000000 2.000000 6.000000 2.000000 \n", "50% 2.000000 2.000000 2.000000 8.000000 2.000000 \n", "75% 2.000000 2.000000 2.000000 9.000000 2.000000 \n", "max 3.000000 2.000000 2.000000 11.000000 2.000000 \n", "\n", " ... COMHSVHLT2 COSUTELE2 COSUAPTDL2 COSURXDL2 \\\n", "count ... 43333.000000 43354.000000 43364.000000 43362.000000 \n", "mean ... 2.410772 2.533884 2.537450 2.563004 \n", "std ... 0.592895 0.618880 0.603669 0.563047 \n", "min ... 1.000000 1.000000 1.000000 1.000000 \n", "25% ... 2.000000 2.000000 2.000000 2.000000 \n", "50% ... 2.000000 3.000000 3.000000 3.000000 \n", "75% ... 3.000000 3.000000 3.000000 3.000000 \n", "max ... 3.000000 3.000000 3.000000 3.000000 \n", "\n", " COSUSVHLT2 COHCTELE2 COHCAPTDL2 COHCRXDL2 COHCSVHLT2 \\\n", "count 43347.000000 43378.000000 43375.000000 43380.000000 43347.00000 \n", "mean 2.578518 1.968809 2.036749 2.196358 2.26424 \n", "std 0.542894 0.761610 0.720678 0.611436 0.56382 \n", "min 1.000000 1.000000 1.000000 1.000000 1.00000 \n", "25% 2.000000 1.000000 2.000000 2.000000 2.00000 \n", "50% 3.000000 2.000000 2.000000 2.000000 2.00000 \n", "75% 3.000000 3.000000 3.000000 3.000000 3.00000 \n", "max 3.000000 3.000000 3.000000 3.000000 3.00000 \n", "\n", " LANGVER \n", "count 45133.000000 \n", "mean 1.042519 \n", "std 0.201772 \n", "min 1.000000 \n", "25% 1.000000 \n", "50% 1.000000 \n", "75% 1.000000 \n", "max 2.000000 \n", "\n", "[8 rows x 2634 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_dropped_targetNaN.describe()" ] }, { "cell_type": "code", "execution_count": 13, "id": "709bd39d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "YMDEIMAD5YR 45133\n", "YUCOSUIPLN2 45133\n", "YMSUD5YANY 45133\n", "YMIUD5YANY 45133\n", "YMDEAUD5YR 45133\n", " ... \n", "OXYCNANYYR 0\n", "PNRANYYR 0\n", "PNRANYFLAG 0\n", "METHAMMON 0\n", "QUESTID2 0\n", "Length: 2636, dtype: int64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_dropped_targetNaN.isna().sum().sort_values(ascending=False)" ] }, { "cell_type": "code", "execution_count": 14, "id": "610bb25e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dropping 6 columns.\n" ] } ], "source": [ "import re\n", "\n", "# Patterns for columns to drop (but NOT ANALWT2_C)\n", "drop_patterns = [\n", " r'^CASEID', r'^QUESTID', r'^QUESTID2', r'^PANEL', r'^VERSION',\n", " r'^INTV', r'^FILE', r'^YEAR',\n", " r'^WT', r'WGT', r'WEIGHT', r'ANALWT(?!2_C$)', # drop all WTs except ANALWT2_C\n", " r'^VESTR', r'^VEREP', r'^ESTRAT',\n", " r'_A$', r'_E$', r'_ORIG$', r'_R$', r'_RC$'\n", "]\n", "\n", "cols_to_drop = [\n", " c for c in df_dropped_targetNaN.columns\n", " if any(re.search(pat, c) for pat in drop_patterns)\n", "]\n", "\n", "# Ensure ANALWT2_C stays\n", "cols_to_drop = [c for c in cols_to_drop if c != \"ANALWT2_C\"]\n", "\n", "# Manual additional drops\n", "manual_drop = [\n", " \"WTANSWER\", \"WTPOUND2\", \"FILEDATE\"\n", "]\n", "\n", "cols_to_drop = list(set(cols_to_drop + [col for col in manual_drop\n", " if col in df_dropped_targetNaN.columns]))\n", "\n", "# Remove duplicates\n", "cols_to_drop = list(set(cols_to_drop))\n", "\n", "print(\"Dropping\", len(cols_to_drop), \"columns.\")\n", "\n", "df_Removed_Columns = df_dropped_targetNaN.drop(columns=cols_to_drop, errors='ignore')" ] }, { "cell_type": "code", "execution_count": 15, "id": "56aa1ad8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of variables with >99% missingness: 282\n", "\n", "Variables with >99% missingness:\n", "['EDUSCKEST', 'EDUSKPEST', 'HLCALLFG', 'HLCALL99', 'BKPOSTOB', 'BKOTHOF2', 'ALTOTFG', 'ALFQFLG', 'ALDYSFG', 'MJFQFLG', 'BLRECFL2', 'BLNT30C1', 'BLNT30C2', 'RSNOMRJ', 'RSNMRJMO', 'CCTOTFG', 'CCFQFLG', 'CRTOTFG', 'CRFQFLG', 'HRTOTFG', 'HRFQFLG', 'HALTOTFG', 'HALFQFLG', 'INHTOTFG', 'INHFQFLG', 'METOTFG', 'MEFQFLG', 'PNRNORXFG', 'TRQNORXFG', 'STMNORXFG', 'SEDNORXFG', 'SRCSEDNM2', 'SRCFRPNRNM', 'SRCFRTRQNM', 'SRCFRSEDNM', 'SRCCLFRPNR', 'SRCCLFRTRQ', 'SRCCLFRSED', 'OTCFLAG', 'OTDGNDLA', 'OTDGNDLB', 'OTDGNDLC', 'OTDGNDLD', 'OTDGNDLE', 'SUNTINSCV', 'SUNTENCV', 'SUNTNOOPN', 'YEATNDYR', 'YEHMSLYR', 'YESCHFLT', 'YESCHWRK', 'YESCHIMP', 'YESCHINT', 'YETCGJOB', 'YELSTGRD', 'YECIGFRNDOF2', 'YECIGNEXTYR2', 'YESTSCIG', 'YESTSMJ', 'YESTSALC', 'YESTSDNK', 'YEPCHKHW', 'YEPHLPHW', 'YEPCHORE', 'YEPLMTTV', 'YEPLMTSN', 'YEPGDJOB', 'YEPPROUD', 'YEYARGUP', 'YEYFGTSW', 'YEYFGTGP', 'YEYHGUN', 'YEYSELL', 'YEYSTOLE', 'YEYATTAK', 'YEPPKCIG', 'YEPMJEVR', 'YEPMJMO', 'YEPALDLY', 'YEGPKCIG', 'YEGMJEVR', 'YEGMJMO', 'YEGALDLY', 'YEFPKCIG', 'YEFMJEVR', 'YEFMJMO', 'YEFALDLY', 'YETLKNON', 'YETLKPAR', 'YETLKBGF', 'YETLKOTA', 'YETLKSOP', 'YEPRTDNG', 'YEPRBSLV', 'YEVIOPRV', 'YEDGPRGP', 'YESLFHLP', 'YEPRGSTD', 'YESCHACT', 'YECOMACT', 'YEFAIACT', 'YEOTHACT', 'YEDECLAS', 'YEDERGLR', 'YEDESPCL', 'YEPVNTYR', 'YERLGSVC', 'YERLGIMP', 'YERLDCSN', 'YERLFRND', 'YUSUITHK', 'YUCOSUITHK', 'YUSUIPLN', 'YUCOSUIPLN', 'SCHFELT', 'TCHGJOB', 'AVGGRADE', 'STNDSCIG', 'STNDSMJ', 'STNDALC', 'STNDDNK', 'PARCHKHW', 'PARHLPHW', 'PRCHORE2', 'PRLMTTV2', 'PARLMTSN', 'PRGDJOB2', 'PRPROUD2', 'ARGUPAR', 'YOFIGHT2', 'YOGRPFT2', 'YOHGUN2', 'YOSELL2', 'YOSTOLE2', 'YOATTAK2', 'PRPKCIG2', 'PRMJEVR2', 'PRMJMO', 'PRALDLY2', 'YFLPKCG2', 'YFLTMRJ2', 'YFLMJMO', 'YFLADLY2', 'FRDPCIG2', 'FRDMEVR2', 'FRDMJMON', 'FRDADLY2', 'TALKPROB', 'PRTALK3', 'PRBSOLV2', 'PREVIOL2', 'PRVDRGO2', 'GRPCNSL2', 'PREGPGM2', 'YTHACT2', 'DRPRVME3', 'ANYEDUC3', 'RLGATTD', 'RLGIMPT', 'RLGDCSN', 'RLGFRND', 'YUSUITHKYR', 'YUCOSUITHK2', 'YUSUIPLNYR', 'YUCOSUIPLN2', 'YODPREV', 'YODSCEV', 'YOLOSEV', 'YODPDISC', 'YODPLSIN', 'YODSLSIN', 'YOLSI2WK', 'YODPR2WK', 'YOWRHRS', 'YOWRDST', 'YOWRCHR', 'YOWRIMP', 'YODPPROB', 'YOWRPROB', 'YOWRAGE', 'YOWRDEPR', 'YOWRDISC', 'YOWRLSIN', 'YOWRPLSR', 'YOWRELES', 'YOWREMOR', 'YOWRGAIN', 'YOWRGROW', 'YOWRPREG', 'YOWRGNL2', 'YOWRLOSE', 'YOWRDIET', 'YOWRLSL2', 'YOWRSLEP', 'YOWRSMOR', 'YOWRENRG', 'YOWRSLOW', 'YOWRSLNO', 'YOWRJITT', 'YOWRJINO', 'YOWRTHOT', 'YOWRCONC', 'YOWRDCSN', 'YOWRNOGD', 'YOWRWRTH', 'YO_MDEA1', 'YO_MDEA2', 'YO_MDEA3', 'YO_MDEA4', 'YO_MDEA5', 'YO_MDEA6', 'YO_MDEA7', 'YO_MDEA8', 'YODSMMDE', 'YOPBINTF', 'YOPBDLYA', 'YOPBRMBR', 'YOPBAGE', 'YOPBNUM', 'YOPB2WK', 'YOPSHMGT', 'YOPSWORK', 'YOPSRELS', 'YOPSSOC', 'YOPSDAYS', 'YOSEEDOC', 'YOFAMDOC', 'YOOTHDOC', 'YOPSYCH', 'YOPSYMD', 'YOSOCWRK', 'YOCOUNS', 'YOOTHMHP', 'YONURSE', 'YORELIG', 'YOHERBAL', 'YOOTHHLP', 'YOTMTNOW', 'YORX12MO', 'YORXNOW', 'YORXHLP', 'YOTMTHLP', 'YMDELT', 'YMDEYR', 'YMDEAUD5YR', 'YMIUD5YANY', 'YMSUD5YANY', 'YMDERSUD5ANY', 'YMDESUD5ANYO', 'YTXMDEYR', 'YRXMDEYR', 'YMDETXRX', 'YDOCMDE', 'YOMDMDE', 'YPSY1MDE', 'YPSY2MDE', 'YSOCMDE', 'YCOUNMDE', 'YOMHMDE', 'YNURSMDE', 'YRELMDE', 'YHBCHMDE', 'YHLTMDE', 'YALTMDE', 'YMDEHPRX', 'YMDEHPO', 'YMDERXO2', 'YMDEHARX', 'YSDSHOME', 'YSDSWRK', 'YSDSREL', 'YSDSSOC', 'YSDSOVRL', 'MDEIMPY', 'YMDEIMAD5YR', 'YMIMI5YANY', 'YMIMR5YANY', 'YMIMS5YANY', 'CIRROSAGE', 'HEPBCAGE', 'HIVAIDSAG', 'GQTYPE2']\n", "\n", "Preview (first 20):\n", "EDUSCKEST 99.915804\n", "EDUSKPEST 99.960118\n", "HLCALLFG 99.953471\n", "HLCALL99 99.953471\n", "BKPOSTOB 100.000000\n", "BKOTHOF2 99.519199\n", "ALTOTFG 99.339729\n", "ALFQFLG 99.295416\n", "ALDYSFG 99.960118\n", "MJFQFLG 99.550218\n", "BLRECFL2 99.915804\n", "BLNT30C1 99.884785\n", "BLNT30C2 99.802805\n", "RSNOMRJ 99.922451\n", "RSNMRJMO 99.898079\n", "CCTOTFG 99.995569\n", "CCFQFLG 99.953471\n", "CRTOTFG 99.995569\n", "CRFQFLG 99.991137\n", "HRTOTFG 99.986706\n", "dtype: float64\n" ] } ], "source": [ "# Checking to see which variables have >99% missingess\n", "na_percent = df_Removed_Columns.isna().mean() * 100\n", "\n", "# Filter columns with > 99% missing\n", "cols_over_99 = na_percent[na_percent > 99]\n", "\n", "# Print count\n", "print(\"Number of variables with >99% missingness:\", len(cols_over_99))\n", "\n", "# Print the variable names\n", "print(\"\\nVariables with >99% missingness:\")\n", "print(cols_over_99.index.tolist())\n", "\n", "# Optional: show top 20 instead of the whole list\n", "print(\"\\nPreview (first 20):\")\n", "print(cols_over_99.head(20))\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "1da00727", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(45133, 2348)" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_99 = df_Removed_Columns.drop(columns=cols_over_99.index)\n", "df_99.shape" ] }, { "cell_type": "code", "execution_count": 17, "id": "f2213f6a", "metadata": {}, "outputs": [], "source": [ "# Removing target leakage features that was used to impute / recode the target IRAMDEYR\n", "# IRAMDEYR → AMDEYR → AMDELT → ADSMMDEA → MDE symptom items + fallback items + timing\n", "\n", "leakage_vars = [\n", "\n", " # Target + imputation\n", " \"IRAMDEYR\", \"IIAMDEYR\",\n", "\n", " # Pre-imputation past-year MDE\n", " \"AMDEYR\",\n", "\n", " # Lifetime MDE + imputation\n", " \"AMDELT\", \"IRAMDELT\", \"IIAMDELT\",\n", "\n", " # MDE with impairment (logically dependent on AMDEYR)\n", " \"AMDEIMP\", \"IRAMDEIMP\", \"IIAMDEIMP\", \"AMDETXRX\",\n", "\n", " # DSM summary\n", " \"ADSMMDEA\",\n", "\n", " # Raw DSM symptom items (if present)\n", " \"D_MDEA1\", \"D_MDEA2\", \"D_MDEA3\", \"D_MDEA4\", \"D_MDEA5\",\n", " \"D_MDEA6\", \"D_MDEA7\", \"D_MDEA8\", \"D_MDEA9\",\n", "\n", " # Symptom recodes (one-digit suffixes, if present)\n", " \"AD_MDEA1\", \"AD_MDEA2\", \"AD_MDEA3\", \"AD_MDEA4\",\n", " \"AD_MDEA5\", \"AD_MDEA6\", \"AD_MDEA7\", \"AD_MDEA8\",\n", "\n", " # Symptom recodes (true DSM flags: two-digit suffix)\n", " \"AD_MDEA11\", \"AD_MDEA21\", \"AD_MDEA31\", \"AD_MDEA41\",\n", " \"AD_MDEA51\", \"AD_MDEA61\", \"AD_MDEA71\", \"AD_MDEA81\", \"AD_MDEA91\",\n", "\n", " # Fallback/gating items\n", " \"ADPB2WK\",\n", " \"ADDPREV\", \"ADDSCEV\", \"ADLOSEV\", \"ADLSI2WK\", \"ADDPR2WK\",\n", " \"ADWRHRS\", \"ADWRDST\", \"ADWRCHR\", \"ADWRIMP\", \"ADDPPROB\",\n", "\n", " # Soft-leakage variables\n", " \"ARXMDEYR\", # received prescription meds\n", " \"ATXMDEYR\", # received counseling/therapy\n", " \"AHLTMDE\", # told by provider they have depression\n", "]" ] }, { "cell_type": "code", "execution_count": 18, "id": "dd235744", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['IRAMDEYR', 'IIAMDEYR', 'AMDEYR', 'AMDELT', 'IRAMDELT', 'IIAMDELT', 'AMDEIMP', 'IRAMDEIMP', 'IIAMDEIMP', 'AMDETXRX', 'ADSMMDEA', 'D_MDEA1', 'D_MDEA2', 'D_MDEA3', 'D_MDEA4', 'D_MDEA5', 'D_MDEA6', 'D_MDEA7', 'D_MDEA8', 'D_MDEA9', 'AD_MDEA1', 'AD_MDEA2', 'AD_MDEA3', 'AD_MDEA4', 'AD_MDEA5', 'AD_MDEA6', 'AD_MDEA7', 'AD_MDEA8', 'AD_MDEA11', 'AD_MDEA21', 'AD_MDEA31', 'AD_MDEA41', 'AD_MDEA51', 'AD_MDEA61', 'AD_MDEA71', 'AD_MDEA81', 'AD_MDEA91', 'ADPB2WK', 'ADDPREV', 'ADDSCEV', 'ADLOSEV', 'ADLSI2WK', 'ADDPR2WK', 'ADWRHRS', 'ADWRDST', 'ADWRCHR', 'ADWRIMP', 'ADDPPROB', 'ARXMDEYR', 'ATXMDEYR', 'AHLTMDE']\n" ] }, { "data": { "text/plain": [ "51" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(leakage_vars)\n", "len(leakage_vars)" ] }, { "cell_type": "code", "execution_count": 19, "id": "c6619dfa", "metadata": {}, "outputs": [], "source": [ "# Drop leakage variables except the target itself\n", "leakage_to_drop = [col for col in leakage_vars if col in df_99.columns and col != \"IRAMDEYR\"]\n", "\n", "df_clean = df_99.drop(columns=leakage_to_drop, errors=\"ignore\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "633dee57", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(45133, 2316)" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_clean .shape" ] }, { "cell_type": "code", "execution_count": 21, "id": "771a9d3c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Final cleaned df shape: (45133, 2316)\n", "\n", "Index: 45133 entries, 0 to 56703\n", "Columns: 2316 entries, ANALWT2_C to LANGVER\n", "dtypes: float64(1528), int64(788)\n", "memory usage: 797.8 MB\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countmeanstdmin25%50%75%max
ANALWT2_C45133.05706.2977798511.3362631.516955973.7404252697.6510236788.874141118941.43015
PDEN1045133.01.6286310.5862711.0000001.0000002.0000002.0000003.00000
COUTYP445133.01.7136020.7309581.0000001.0000002.0000002.0000003.00000
MAIIN10245133.01.9845790.1232221.0000002.0000002.0000002.0000002.00000
AIIND10245133.01.9843570.1240901.0000002.0000002.0000002.0000002.00000
AGE345133.07.8166972.1940384.0000006.0000008.0000009.00000011.00000
SERVICE45108.01.9489670.2200681.0000002.0000002.0000002.0000002.00000
MILSTAT2294.02.9346120.2472632.0000003.0000003.0000003.0000003.00000
ACTDEVER2299.01.2331450.4229261.0000001.0000001.0000001.0000002.00000
ACTD20011756.01.5615030.4963441.0000001.0000002.0000002.0000002.00000
ACTD90011756.01.7710710.4202631.0000002.0000002.0000002.0000002.00000
ACTD75901756.01.7653760.4238841.0000002.0000002.0000002.0000002.00000
ACTDVIET1755.01.7196580.4492941.0000001.0000002.0000002.0000002.00000
ACTDPRIV1754.01.9521090.2135961.0000002.0000002.0000002.0000002.00000
COMBATPY1756.01.5865600.4925911.0000001.0000002.0000002.0000002.00000
NOMARR224284.01.2023550.4017641.0000001.0000001.0000001.0000002.00000
HEALTH45119.02.3993660.9734821.0000002.0000002.0000003.0000005.00000
MOVSINPYR243538.00.3473060.6899240.0000000.0000000.0000000.0000003.00000
SEXATRACT243487.01.4850191.1245611.0000001.0000001.0000001.0000006.00000
SEXIDENT2243410.01.4898871.2101231.0000001.0000001.0000001.0000006.00000
\n", "
" ], "text/plain": [ " count mean std min 25% \\\n", "ANALWT2_C 45133.0 5706.297779 8511.336263 1.516955 973.740425 \n", "PDEN10 45133.0 1.628631 0.586271 1.000000 1.000000 \n", "COUTYP4 45133.0 1.713602 0.730958 1.000000 1.000000 \n", "MAIIN102 45133.0 1.984579 0.123222 1.000000 2.000000 \n", "AIIND102 45133.0 1.984357 0.124090 1.000000 2.000000 \n", "AGE3 45133.0 7.816697 2.194038 4.000000 6.000000 \n", "SERVICE 45108.0 1.948967 0.220068 1.000000 2.000000 \n", "MILSTAT 2294.0 2.934612 0.247263 2.000000 3.000000 \n", "ACTDEVER 2299.0 1.233145 0.422926 1.000000 1.000000 \n", "ACTD2001 1756.0 1.561503 0.496344 1.000000 1.000000 \n", "ACTD9001 1756.0 1.771071 0.420263 1.000000 2.000000 \n", "ACTD7590 1756.0 1.765376 0.423884 1.000000 2.000000 \n", "ACTDVIET 1755.0 1.719658 0.449294 1.000000 1.000000 \n", "ACTDPRIV 1754.0 1.952109 0.213596 1.000000 2.000000 \n", "COMBATPY 1756.0 1.586560 0.492591 1.000000 1.000000 \n", "NOMARR2 24284.0 1.202355 0.401764 1.000000 1.000000 \n", "HEALTH 45119.0 2.399366 0.973482 1.000000 2.000000 \n", "MOVSINPYR2 43538.0 0.347306 0.689924 0.000000 0.000000 \n", "SEXATRACT2 43487.0 1.485019 1.124561 1.000000 1.000000 \n", "SEXIDENT22 43410.0 1.489887 1.210123 1.000000 1.000000 \n", "\n", " 50% 75% max \n", "ANALWT2_C 2697.651023 6788.874141 118941.43015 \n", "PDEN10 2.000000 2.000000 3.00000 \n", "COUTYP4 2.000000 2.000000 3.00000 \n", "MAIIN102 2.000000 2.000000 2.00000 \n", "AIIND102 2.000000 2.000000 2.00000 \n", "AGE3 8.000000 9.000000 11.00000 \n", "SERVICE 2.000000 2.000000 2.00000 \n", "MILSTAT 3.000000 3.000000 3.00000 \n", "ACTDEVER 1.000000 1.000000 2.00000 \n", "ACTD2001 2.000000 2.000000 2.00000 \n", "ACTD9001 2.000000 2.000000 2.00000 \n", "ACTD7590 2.000000 2.000000 2.00000 \n", "ACTDVIET 2.000000 2.000000 2.00000 \n", "ACTDPRIV 2.000000 2.000000 2.00000 \n", "COMBATPY 2.000000 2.000000 2.00000 \n", "NOMARR2 1.000000 1.000000 2.00000 \n", "HEALTH 2.000000 3.000000 5.00000 \n", "MOVSINPYR2 0.000000 0.000000 3.00000 \n", "SEXATRACT2 1.000000 1.000000 6.00000 \n", "SEXIDENT22 1.000000 1.000000 6.00000 " ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Summary statistics\n", "print(\"Final cleaned df shape:\", df_clean.shape)\n", "df_clean.info()\n", "df_clean.describe(include='all').transpose().head(20)" ] }, { "cell_type": "code", "execution_count": 22, "id": "850f1c13", "metadata": {}, "outputs": [], "source": [ "# Save cleaned dataset\n", "df_clean.to_pickle(\"../data/cleaned/NSDUH_2023_clean.pkl\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }