{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0653c355",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import sys\n",
    "sys.path.append('../src')\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "   \n",
    "sns.set(font_scale = 3)    \n",
    "sns.set_theme()\n",
    "sns.set_style(\"whitegrid\")     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2b8e4dac",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    'conference-1',\n",
    "    'conference-2',\n",
    "    'conference-3',\n",
    "    'conference-4',\n",
    "    'conference-5',\n",
    "    'conference-6',\n",
    "    'conference-7',\n",
    "    'conference-8',\n",
    "    'conference-9',\n",
    "    'conference-10',\n",
    "    'conference-11',\n",
    "    'conference-12',\n",
    "    'conference-13',\n",
    "    'conference-14',\n",
    "    'conference-15',\n",
    "    'conference-16',\n",
    "    'conference-17',\n",
    "    'conference-18',\n",
    "    'conference-19',\n",
    "    'conference-20',\n",
    "    'conference-21',\n",
    "    'ai4eu-1',\n",
    "    'ai4eu-2',\n",
    "    'ai4eu-3',\n",
    "    'nasa',\n",
    "]\n",
    "\n",
    "# loading the results from each dataset\n",
    "methods = [\n",
    "    'WeSAL',\n",
    "    'AL-RF',\n",
    "    'DualLoop'\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0329ab03",
   "metadata": {},
   "outputs": [],
   "source": [
    "recall_range = [70, 80, 90, 98]\n",
    "result_folder = '../results/'\n",
    "\n",
    "all_measurements = []\n",
    "for ds in datasets:    \n",
    "    for method in methods:\n",
    "        measurement = {}\n",
    "        measurement['dataset'] = ds\n",
    "        measurement['method'] = method\n",
    "        \n",
    "        result = pickle.load( open(result_folder + ds + '/result_' + method + '_' + ds  + \".pk\", \"rb\" ) )        \n",
    "        for iteration in result:\n",
    "            r = result[iteration]\n",
    "            recall = r['recall'] * 100.0\n",
    "            budget = r['human_effort'] \n",
    "            \n",
    "            for p in recall_range:\n",
    "                key = str(p)\n",
    "                if recall > p and key not in measurement:\n",
    "                    measurement[key] = budget\n",
    "            \n",
    "        last_budget = result[iteration]['human_effort'] \n",
    "\n",
    "        for p in recall_range:\n",
    "            key = str(p) \n",
    "            if key not in measurement:\n",
    "                measurement[key] = last_budget\n",
    "                    \n",
    "        all_measurements.append(measurement)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "394d3657",
   "metadata": {},
   "outputs": [],
   "source": [
    "aggregated_measurement = []\n",
    "for m in all_measurements:\n",
    "    measurement = m.copy()\n",
    "    measurement['dataset'] = measurement['dataset'].split(\"-\", 1)[0]\n",
    "    aggregated_measurement.append(measurement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "dc8b5a57",
   "metadata": {},
   "outputs": [],
   "source": [
    "budget_df = pd.DataFrame(aggregated_measurement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f9fb0e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "cost_reduction = budget_df.groupby(['dataset','method']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e1345bef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>70</th>\n",
       "      <th>80</th>\n",
       "      <th>90</th>\n",
       "      <th>98</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ai4eu</th>\n",
       "      <th>AL-RF</th>\n",
       "      <td>1652</td>\n",
       "      <td>1802</td>\n",
       "      <td>10000</td>\n",
       "      <td>17100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DualLoop</th>\n",
       "      <td>390</td>\n",
       "      <td>886</td>\n",
       "      <td>3600</td>\n",
       "      <td>8350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WeSAL</th>\n",
       "      <td>7894</td>\n",
       "      <td>7894</td>\n",
       "      <td>11425</td>\n",
       "      <td>13844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">conference</th>\n",
       "      <th>AL-RF</th>\n",
       "      <td>8852</td>\n",
       "      <td>11051</td>\n",
       "      <td>15731</td>\n",
       "      <td>20191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DualLoop</th>\n",
       "      <td>1892</td>\n",
       "      <td>2870</td>\n",
       "      <td>6080</td>\n",
       "      <td>13290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WeSAL</th>\n",
       "      <td>22329</td>\n",
       "      <td>34103</td>\n",
       "      <td>37919</td>\n",
       "      <td>39103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">nasa</th>\n",
       "      <th>AL-RF</th>\n",
       "      <td>1500</td>\n",
       "      <td>1800</td>\n",
       "      <td>7600</td>\n",
       "      <td>11500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DualLoop</th>\n",
       "      <td>800</td>\n",
       "      <td>1400</td>\n",
       "      <td>4900</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WeSAL</th>\n",
       "      <td>6213</td>\n",
       "      <td>7013</td>\n",
       "      <td>8441</td>\n",
       "      <td>11041</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        70     80     90     98\n",
       "dataset    method                              \n",
       "ai4eu      AL-RF      1652   1802  10000  17100\n",
       "           DualLoop    390    886   3600   8350\n",
       "           WeSAL      7894   7894  11425  13844\n",
       "conference AL-RF      8852  11051  15731  20191\n",
       "           DualLoop   1892   2870   6080  13290\n",
       "           WeSAL     22329  34103  37919  39103\n",
       "nasa       AL-RF      1500   1800   7600  11500\n",
       "           DualLoop    800   1400   4900  10000\n",
       "           WeSAL      6213   7013   8441  11041"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost_reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9b193276",
   "metadata": {},
   "outputs": [],
   "source": [
    "cost_df = cost_reduction.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "575dd228",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['dataset', 'method', '70', '80', '90', '98'], dtype='object')"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2604df8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>method</th>\n",
       "      <th>70</th>\n",
       "      <th>80</th>\n",
       "      <th>90</th>\n",
       "      <th>98</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ai4eu</td>\n",
       "      <td>AL-RF</td>\n",
       "      <td>1652</td>\n",
       "      <td>1802</td>\n",
       "      <td>10000</td>\n",
       "      <td>17100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ai4eu</td>\n",
       "      <td>DualLoop</td>\n",
       "      <td>390</td>\n",
       "      <td>886</td>\n",
       "      <td>3600</td>\n",
       "      <td>8350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ai4eu</td>\n",
       "      <td>WeSAL</td>\n",
       "      <td>7894</td>\n",
       "      <td>7894</td>\n",
       "      <td>11425</td>\n",
       "      <td>13844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>conference</td>\n",
       "      <td>AL-RF</td>\n",
       "      <td>8852</td>\n",
       "      <td>11051</td>\n",
       "      <td>15731</td>\n",
       "      <td>20191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>conference</td>\n",
       "      <td>DualLoop</td>\n",
       "      <td>1892</td>\n",
       "      <td>2870</td>\n",
       "      <td>6080</td>\n",
       "      <td>13290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>conference</td>\n",
       "      <td>WeSAL</td>\n",
       "      <td>22329</td>\n",
       "      <td>34103</td>\n",
       "      <td>37919</td>\n",
       "      <td>39103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>nasa</td>\n",
       "      <td>AL-RF</td>\n",
       "      <td>1500</td>\n",
       "      <td>1800</td>\n",
       "      <td>7600</td>\n",
       "      <td>11500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>nasa</td>\n",
       "      <td>DualLoop</td>\n",
       "      <td>800</td>\n",
       "      <td>1400</td>\n",
       "      <td>4900</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>nasa</td>\n",
       "      <td>WeSAL</td>\n",
       "      <td>6213</td>\n",
       "      <td>7013</td>\n",
       "      <td>8441</td>\n",
       "      <td>11041</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      dataset    method     70     80     90     98\n",
       "0       ai4eu     AL-RF   1652   1802  10000  17100\n",
       "1       ai4eu  DualLoop    390    886   3600   8350\n",
       "2       ai4eu     WeSAL   7894   7894  11425  13844\n",
       "3  conference     AL-RF   8852  11051  15731  20191\n",
       "4  conference  DualLoop   1892   2870   6080  13290\n",
       "5  conference     WeSAL  22329  34103  37919  39103\n",
       "6        nasa     AL-RF   1500   1800   7600  11500\n",
       "7        nasa  DualLoop    800   1400   4900  10000\n",
       "8        nasa     WeSAL   6213   7013   8441  11041"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "9bc8cf7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22329 8852 1892\n",
      "7894 1652 390\n",
      "6213 1500 800\n",
      "34103 11051 2870\n",
      "7894 1802 886\n",
      "7013 1800 1400\n",
      "37919 15731 6080\n",
      "11425 10000 3600\n",
      "8441 7600 4900\n",
      "39103 20191 13290\n",
      "13844 17100 8350\n",
      "11041 11500 10000\n"
     ]
    }
   ],
   "source": [
    "dataset_categories = ['conference', 'ai4eu', 'nasa']\n",
    "\n",
    "reductions_wesal_dualloop = []\n",
    "reductions_alrf_dualloop = []\n",
    "\n",
    "for r in recall_range:\n",
    "    for ds in dataset_categories:   \n",
    "        cost_wesal = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='WeSAL')][str(r)].iloc[0]\n",
    "        cost_alrf = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='AL-RF')][str(r)].iloc[0]\n",
    "        cost_dualloop = cost_df[(cost_df['dataset']==ds) & (cost_df['method']=='DualLoop')][str(r)].iloc[0]\n",
    "        print(cost_wesal, cost_alrf, cost_dualloop)\n",
    "        reduction1 = 100.0 - cost_dualloop * 100.0/cost_wesal\n",
    "        reduction2 = 100.0 - cost_dualloop * 100.0/cost_alrf\n",
    "        \n",
    "        reduction = {\n",
    "            'recall': r,\n",
    "            'reduction': reduction1\n",
    "        }\n",
    "        reductions_wesal_dualloop.append(reduction)\n",
    "        \n",
    "        reduction = {\n",
    "            'recall': r,            \n",
    "            'reduction': reduction2\n",
    "        }\n",
    "        reductions_alrf_dualloop.append(reduction)        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "cc669484",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">reduction</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>91.236675</td>\n",
       "      <td>87.123773</td>\n",
       "      <td>95.059539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>86.799226</td>\n",
       "      <td>80.037074</td>\n",
       "      <td>91.584318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>64.801994</td>\n",
       "      <td>41.950006</td>\n",
       "      <td>83.965822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>38.375465</td>\n",
       "      <td>9.428494</td>\n",
       "      <td>66.012838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        reduction                      \n",
       "             mean        min        max\n",
       "recall                                 \n",
       "70      91.236675  87.123773  95.059539\n",
       "80      86.799226  80.037074  91.584318\n",
       "90      64.801994  41.950006  83.965822\n",
       "98      38.375465   9.428494  66.012838"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduction_df_wesal_dualloop = pd.DataFrame(reductions_wesal_dualloop)\n",
    "reduction_df_wesal_dualloop.groupby(['recall']).agg({'reduction': ['mean', 'min', 'max']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "282fb641",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">reduction</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>67.228406</td>\n",
       "      <td>46.666667</td>\n",
       "      <td>78.626299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>49.028043</td>\n",
       "      <td>22.222222</td>\n",
       "      <td>74.029500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>53.625505</td>\n",
       "      <td>35.526316</td>\n",
       "      <td>64.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>32.797221</td>\n",
       "      <td>13.043478</td>\n",
       "      <td>51.169591</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        reduction                      \n",
       "             mean        min        max\n",
       "recall                                 \n",
       "70      67.228406  46.666667  78.626299\n",
       "80      49.028043  22.222222  74.029500\n",
       "90      53.625505  35.526316  64.000000\n",
       "98      32.797221  13.043478  51.169591"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduction_df_alrf_dualloop = pd.DataFrame(reductions_alrf_dualloop)\n",
    "reduction_df_alrf_dualloop.groupby(['recall']).agg({'reduction': ['mean', 'min', 'max']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38db93c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}