{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "0653c355",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import pickle\n",
"import sys\n",
"sys.path.append('../src')\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
" \n",
"sns.set(font_scale = 3) \n",
"sns.set_theme()\n",
"sns.set_style(\"whitegrid\") "
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "2b8e4dac",
"metadata": {},
"outputs": [],
"source": [
"datasets = [\n",
" 'conference-1',\n",
" 'conference-2',\n",
" 'conference-3',\n",
" 'conference-4',\n",
" 'conference-5',\n",
" 'conference-6',\n",
" 'conference-7',\n",
" 'conference-8',\n",
" 'conference-9',\n",
" 'conference-10',\n",
" 'conference-11',\n",
" 'conference-12',\n",
" 'conference-13',\n",
" 'conference-14',\n",
" 'conference-15',\n",
" 'conference-16',\n",
" 'conference-17',\n",
" 'conference-18',\n",
" 'conference-19',\n",
" 'conference-20',\n",
" 'conference-21',\n",
" 'ai4eu-1',\n",
" 'ai4eu-2',\n",
" 'ai4eu-3',\n",
" 'nasa',\n",
"]\n",
"\n",
"# loading the results from each dataset\n",
"methods = [\n",
" 'WeSAL',\n",
" 'AL-RF',\n",
" 'DualLoop'\n",
"]\n"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "0329ab03",
"metadata": {},
"outputs": [],
"source": [
"recall_range = [70, 80, 90, 98]\n",
"result_folder = '../results/'\n",
"\n",
"all_measurements = []\n",
"for ds in datasets: \n",
" for method in methods:\n",
" measurement = {}\n",
" measurement['dataset'] = ds\n",
" measurement['method'] = method\n",
" \n",
" result = pickle.load( open(result_folder + ds + '/result_' + method + '_' + ds + \".pk\", \"rb\" ) ) \n",
" for iteration in result:\n",
" r = result[iteration]\n",
" recall = r['recall'] * 100.0\n",
" budget = r['human_effort'] \n",
" \n",
" for p in recall_range:\n",
" key = str(p)\n",
" if recall > p and key not in measurement:\n",
" measurement[key] = budget\n",
" \n",
" last_budget = result[iteration]['human_effort'] \n",
"\n",
" for p in recall_range:\n",
" key = str(p) \n",
" if key not in measurement:\n",
" measurement[key] = last_budget\n",
" \n",
" all_measurements.append(measurement)\n"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "394d3657",
"metadata": {},
"outputs": [],
"source": [
"aggregated_measurement = []\n",
"for m in all_measurements:\n",
" measurement = m.copy()\n",
" measurement['dataset'] = measurement['dataset'].split(\"-\", 1)[0]\n",
" aggregated_measurement.append(measurement)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "dc8b5a57",
"metadata": {},
"outputs": [],
"source": [
"budget_df = pd.DataFrame(aggregated_measurement)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "f9fb0e06",
"metadata": {},
"outputs": [],
"source": [
"cost_reduction = budget_df.groupby(['dataset','method']).sum()"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "e1345bef",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th></th>\n",
" <th>70</th>\n",
" <th>80</th>\n",
" <th>90</th>\n",
" <th>98</th>\n",
" </tr>\n",
" <tr>\n",
" <th>dataset</th>\n",
" <th>method</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">ai4eu</th>\n",
" <th>AL-RF</th>\n",
" <td>1652</td>\n",
" <td>1802</td>\n",
" <td>10000</td>\n",
" <td>17100</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DualLoop</th>\n",
" <td>390</td>\n",
" <td>886</td>\n",
" <td>3600</td>\n",
" <td>8350</td>\n",
" </tr>\n",
" <tr>\n",
" <th>WeSAL</th>\n",
" <td>7894</td>\n",
" <td>7894</td>\n",
" <td>11425</td>\n",
" <td>13844</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">conference</th>\n",
" <th>AL-RF</th>\n",
" <td>8852</td>\n",
" <td>11051</td>\n",
" <td>15731</td>\n",
" <td>20191</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DualLoop</th>\n",
" <td>1892</td>\n",
" <td>2870</td>\n",
" <td>6080</td>\n",
" <td>13290</td>\n",
" </tr>\n",
" <tr>\n",
" <th>WeSAL</th>\n",
" <td>22329</td>\n",
" <td>34103</td>\n",
" <td>37919</td>\n",
" <td>39103</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">nasa</th>\n",
" <th>AL-RF</th>\n",
" <td>1500</td>\n",
" <td>1800</td>\n",
" <td>7600</td>\n",
" <td>11500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DualLoop</th>\n",
" <td>800</td>\n",
" <td>1400</td>\n",
" <td>4900</td>\n",
" <td>10000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>WeSAL</th>\n",
" <td>6213</td>\n",
" <td>7013</td>\n",
" <td>8441</td>\n",
" <td>11041</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 70 80 90 98\n",
"dataset method \n",
"ai4eu AL-RF 1652 1802 10000 17100\n",
" DualLoop 390 886 3600 8350\n",
" WeSAL 7894 7894 11425 13844\n",
"conference AL-RF 8852 11051 15731 20191\n",
" DualLoop 1892 2870 6080 13290\n",
" WeSAL 22329 34103 37919 39103\n",
"nasa AL-RF 1500 1800 7600 11500\n",
" DualLoop 800 1400 4900 10000\n",
" WeSAL 6213 7013 8441 11041"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cost_reduction"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "9b193276",
"metadata": {},
"outputs": [],
"source": [
"cost_df = cost_reduction.reset_index()"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "575dd228",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Index(['dataset', 'method', '70', '80', '90', '98'], dtype='object')"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cost_df.columns"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "2604df8d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th>method</th>\n",
" <th>70</th>\n",
" <th>80</th>\n",
" <th>90</th>\n",
" <th>98</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>ai4eu</td>\n",
" <td>AL-RF</td>\n",
" <td>1652</td>\n",
" <td>1802</td>\n",
" <td>10000</td>\n",
" <td>17100</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>ai4eu</td>\n",
" <td>DualLoop</td>\n",
" <td>390</td>\n",
" <td>886</td>\n",
" <td>3600</td>\n",
" <td>8350</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>ai4eu</td>\n",
" <td>WeSAL</td>\n",
" <td>7894</td>\n",
" <td>7894</td>\n",
" <td>11425</td>\n",
" <td>13844</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>conference</td>\n",
" <td>AL-RF</td>\n",
" <td>8852</td>\n",
" <td>11051</td>\n",
" <td>15731</td>\n",
" <td>20191</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>conference</td>\n",
" <td>DualLoop</td>\n",
" <td>1892</td>\n",
" <td>2870</td>\n",
" <td>6080</td>\n",
" <td>13290</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>conference</td>\n",
" <td>WeSAL</td>\n",
" <td>22329</td>\n",
" <td>34103</td>\n",
" <td>37919</td>\n",
" <td>39103</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>nasa</td>\n",
" <td>AL-RF</td>\n",
" <td>1500</td>\n",
" <td>1800</td>\n",
" <td>7600</td>\n",
" <td>11500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>nasa</td>\n",
" <td>DualLoop</td>\n",
" <td>800</td>\n",
" <td>1400</td>\n",
" <td>4900</td>\n",
" <td>10000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>nasa</td>\n",
" <td>WeSAL</td>\n",
" <td>6213</td>\n",
" <td>7013</td>\n",
" <td>8441</td>\n",
" <td>11041</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" dataset method 70 80 90 98\n",
"0 ai4eu AL-RF 1652 1802 10000 17100\n",
"1 ai4eu DualLoop 390 886 3600 8350\n",
"2 ai4eu WeSAL 7894 7894 11425 13844\n",
"3 conference AL-RF 8852 11051 15731 20191\n",
"4 conference DualLoop 1892 2870 6080 13290\n",
"5 conference WeSAL 22329 34103 37919 39103\n",
"6 nasa AL-RF 1500 1800 7600 11500\n",
"7 nasa DualLoop 800 1400 4900 10000\n",
"8 nasa WeSAL 6213 7013 8441 11041"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cost_df"
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "9bc8cf7b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"22329 8852 1892\n",
"7894 1652 390\n",
"6213 1500 800\n",
"34103 11051 2870\n",
"7894 1802 886\n",
"7013 1800 1400\n",
"37919 15731 6080\n",
"11425 10000 3600\n",
"8441 7600 4900\n",
"39103 20191 13290\n",
"13844 17100 8350\n",
"11041 11500 10000\n"
]
}
],
"source": [
"dataset_categories = ['conference', 'ai4eu', 'nasa']\n",
"\n",
"reductions_wesal_dualloop = []\n",
"reductions_alrf_dualloop = []\n",
"\n",
"for r in recall_range:\n",
" for ds in dataset_categories: \n",
" cost_wesal = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='WeSAL')][str(r)].iloc[0]\n",
" cost_alrf = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='AL-RF')][str(r)].iloc[0]\n",
" cost_dualloop = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='DualLoop')][str(r)].iloc[0]\n",
" print(cost_wesal, cost_alrf, cost_dualloop)\n",
" reduction1 = 100.0 - cost_dualloop * 100.0/cost_wesal\n",
" reduction2 = 100.0 - cost_dualloop * 100.0/cost_alrf\n",
" \n",
" reduction = {\n",
" 'recall': r,\n",
" 'reduction': reduction1\n",
" }\n",
" reductions_wesal_dualloop.append(reduction)\n",
" \n",
" reduction = {\n",
" 'recall': r, \n",
" 'reduction': reduction2\n",
" }\n",
" reductions_alrf_dualloop.append(reduction) \n",
" "
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "cc669484",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th colspan=\"3\" halign=\"left\">reduction</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>min</th>\n",
" <th>max</th>\n",
" </tr>\n",
" <tr>\n",
" <th>recall</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>70</th>\n",
" <td>91.236675</td>\n",
" <td>87.123773</td>\n",
" <td>95.059539</td>\n",
" </tr>\n",
" <tr>\n",
" <th>80</th>\n",
" <td>86.799226</td>\n",
" <td>80.037074</td>\n",
" <td>91.584318</td>\n",
" </tr>\n",
" <tr>\n",
" <th>90</th>\n",
" <td>64.801994</td>\n",
" <td>41.950006</td>\n",
" <td>83.965822</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>38.375465</td>\n",
" <td>9.428494</td>\n",
" <td>66.012838</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" reduction \n",
" mean min max\n",
"recall \n",
"70 91.236675 87.123773 95.059539\n",
"80 86.799226 80.037074 91.584318\n",
"90 64.801994 41.950006 83.965822\n",
"98 38.375465 9.428494 66.012838"
]
},
"execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reduction_df_wesal_dualloop = pd.DataFrame(reductions_wesal_dualloop)\n",
"reduction_df_wesal_dualloop.groupby(['recall']).agg({'reduction': ['mean', 'min', 'max']})"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "282fb641",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th colspan=\"3\" halign=\"left\">reduction</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>min</th>\n",
" <th>max</th>\n",
" </tr>\n",
" <tr>\n",
" <th>recall</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>70</th>\n",
" <td>67.228406</td>\n",
" <td>46.666667</td>\n",
" <td>78.626299</td>\n",
" </tr>\n",
" <tr>\n",
" <th>80</th>\n",
" <td>49.028043</td>\n",
" <td>22.222222</td>\n",
" <td>74.029500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>90</th>\n",
" <td>53.625505</td>\n",
" <td>35.526316</td>\n",
" <td>64.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>32.797221</td>\n",
" <td>13.043478</td>\n",
" <td>51.169591</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" reduction \n",
" mean min max\n",
"recall \n",
"70 67.228406 46.666667 78.626299\n",
"80 49.028043 22.222222 74.029500\n",
"90 53.625505 35.526316 64.000000\n",
"98 32.797221 13.043478 51.169591"
]
},
"execution_count": 78,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reduction_df_alrf_dualloop = pd.DataFrame(reductions_alrf_dualloop)\n",
"reduction_df_alrf_dualloop.groupby(['recall']).agg({'reduction': ['mean', 'min', 'max']})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f38db93c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}