{
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"id": "f26eb23f",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import os\n",
"import sys\n",
"sys.path.append('../src')\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
" \n",
"sns.set(font_scale = 3) \n",
"sns.set_theme()\n",
"sns.set_style(\"whitegrid\") \n",
"\n",
"import pandas as pd\n",
"import pickle\n",
"\n",
"import dualloop as dual_loops\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "dc80de74",
"metadata": {},
"outputs": [],
"source": [
"def read_config(config_file):\n",
" with open(config_file, 'r') as f:\n",
" config = json.load(f)\n",
" return config\n",
"\n",
"cfg = read_config('../config.json')"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "73d25a55",
"metadata": {},
"outputs": [],
"source": [
"def run_experiment(cfg, dataset_name, method_name, test_case):\n",
" my_dataset_df = pd.read_csv('../results/' + dataset_name + '/' + dataset_name + '_matrix.csv')\n",
"\n",
" source_ontology = pickle.load(open('../results/' + dataset_name + '/' + dataset_name + \"_source_ontology.pk\", \"rb\"))\n",
" target_ontology = pickle.load(open('../results/' + dataset_name + '/' + dataset_name + \"_target_ontology.pk\", \"rb\"))\n",
"\n",
" dual_loops.source_ontology = source_ontology\n",
" dual_loops.target_ontology = target_ontology\n",
"\n",
" lfs_set = cfg['lfs_set']\n",
" feature_set = cfg['feature_set']\n",
" methods = cfg['methods']\n",
" method_config = methods[method_name]\n",
"\n",
" epochs = 100\n",
" balance = [0.9, 0.1]\n",
"\n",
" total_size = len(my_dataset_df)\n",
" num_iteration = int(total_size * 0.2) # explore only the first 20% of the dataset\n",
" if num_iteration > 2000:\n",
" num_iteration = 2000\n",
"\n",
" if dataset_name.startswith('conference-'):\n",
" batch_size = 10\n",
" interval_slow_loop = 20\n",
" elif dataset_name.startswith('ai4eu-'):\n",
" batch_size = 50\n",
" interval_slow_loop = 100\n",
" elif dataset_name.startswith('nasa-'):\n",
" batch_size = 100\n",
" interval_slow_loop = 200\n",
" elif dataset_name.startswith('anatomy-'):\n",
" batch_size = 200\n",
" interval_slow_loop = 500\n",
" else:\n",
" batch_size = 100\n",
" interval_slow_loop = 200\n",
"\n",
" result = dual_loops.run_experiment(method_config, my_dataset_df, lfs_set, feature_set,\n",
" total_size, num_iteration, interval_slow_loop, batch_size,\n",
" balance, epochs, False)\n",
"\n",
" pickle.dump(result, open('../results/' + dataset_name + \"/\" + test_case + \"_\" + method_name + \"_\" + dataset_name + \".pk\", \"wb\"))\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "02717d95",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4cfce7f834fc4809b0ff509ebf38bf69",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/223 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done with WeSAL\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a9641275c9b9488faecb31bf4d0448b0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/223 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"do random sampling before having some matches and non-matches in the annotated part\n",
"Done with AL-RF\n",
"no selection of lfs due to limited number of annotated samples\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1680f73f4e41492eb3827a501677f5ae",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/223 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"no selection of lfs due to limited number of annotated samples\n",
"[0.42, 0.3, 0.1, 0.35, 0.1, 0.26249999999999996, 0.4666666666666666, 0.175, 0.1, 0.1, 0.1, 0.1, 0.4666666666666666, 0.1, 0.1]\n",
"========= SLOW LOOP =============\n",
"========= random_forest ===========\n",
"1.0 0.4166666666666667 0.5882352941176471\n",
"========= logistic_regression ===========\n",
"0.6666666666666666 0.3333333333333333 0.4444444444444444\n",
"========= xgboost ===========\n",
"0.6666666666666666 0.3333333333333333 0.4444444444444444\n",
"========= mlp ===========\n",
"0.6 0.25 0.35294117647058826\n",
"===== LF_class_name_similarity_ma =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 4, 'fp': 2, 'tn': 13, 'fn': 1, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.7489004135131836 s\n",
" direc: array([[-0.74870287]])\n",
" fun: 0.0\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 66\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([3.98280589e-05])\n",
"new x0 = [3.98280589e-05]\n",
"-------NEW---------\n",
"{'x0': [array([3.98280589e-05])], 'annotated': {'tp': 5, 'fp': 15, 'tn': 0, 'fn': 0, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 1097, 'tn': 0, 'fn': 0, 'coverage': 1.0}}\n",
"LF_class_name_similarity_ma_tuned\n",
"===== LF_class_name_similarity_mb =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 4, 'fp': 2, 'tn': 13, 'fn': 1, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.6946346759796143 s\n",
" direc: array([[0.14290116]])\n",
" fun: 0.0\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 64\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([0.23135423])\n",
"new x0 = [0.23135423]\n",
"-------NEW---------\n",
"{'x0': [array([0.23135423])], 'annotated': {'tp': 5, 'fp': 13, 'tn': 2, 'fn': 0, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 697, 'tn': 400, 'fn': 0, 'coverage': 1.0}}\n",
"LF_class_name_similarity_mb_tuned\n",
"===== LF_label_similarity_ma =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.23112964630126953 s\n",
" direc: array([[1.]])\n",
" fun: 5\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 21\n",
" nit: 1\n",
" status: 0\n",
" success: True\n",
" x: array([0.97993522])\n",
"new x0 = [0.97993522]\n",
"-------NEW---------\n",
"{'x0': [array([0.97993522])], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_label_similarity_ma_tuned\n",
"===== LF_label_similarity_mb =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.2285306453704834 s\n",
" direc: array([[1.]])\n",
" fun: 5\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 21\n",
" nit: 1\n",
" status: 0\n",
" success: True\n",
" x: array([0.97993522])\n",
"new x0 = [0.97993522]\n",
"-------NEW---------\n",
"{'x0': [array([0.97993522])], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_label_similarity_mb_tuned\n",
"===== LF_comment_similarity_ma =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.2300264835357666 s\n",
" direc: array([[1.]])\n",
" fun: 5\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 21\n",
" nit: 1\n",
" status: 0\n",
" success: True\n",
" x: array([0.97993522])\n",
"new x0 = [0.97993522]\n",
"-------NEW---------\n",
"{'x0': [array([0.97993522])], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_comment_similarity_ma_tuned\n",
"===== LF_comment_similarity_mb =========\n",
"x0 [1.0]\n",
"bounds [(0.0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.2293992042541504 s\n",
" direc: array([[1.]])\n",
" fun: 5\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 21\n",
" nit: 1\n",
" status: 0\n",
" success: True\n",
" x: array([0.97993522])\n",
"new x0 = [0.97993522]\n",
"-------NEW---------\n",
"{'x0': [array([0.97993522])], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_comment_similarity_mb_tuned\n",
"===== LF_num_common_words =========\n",
"x0 [5]\n",
"bounds [(1, 5)]\n",
"-------OLD---------\n",
"{'x0': [5], 'annotated': {'tp': 5, 'fp': 10, 'tn': 5, 'fn': 0, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 620, 'tn': 477, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 620\n",
"Optimizing Time: 0.2611880302429199 s\n",
" direc: array([[1.]])\n",
" fun: 0.0\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 24\n",
" nit: 1\n",
" status: 0\n",
" success: True\n",
" x: array([4.99993758])\n",
"new x0 = [4.99993758]\n",
"-------NEW---------\n",
"{'x0': [array([4.99993758])], 'annotated': {'tp': 5, 'fp': 10, 'tn': 5, 'fn': 0, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 620, 'tn': 477, 'fn': 0, 'coverage': 1.0}}\n",
"LF_num_common_words_tuned\n",
"===== LF_random_forest_prob =========\n",
"x0 [1.0]\n",
"bounds [(0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.7052431106567383 s\n",
" direc: array([[-0.37433811]])\n",
" fun: 0.0\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 65\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([2.47456001e-05])\n",
"new x0 = [2.47456001e-05]\n",
"-------NEW---------\n",
"{'x0': [array([2.47456001e-05])], 'annotated': {'tp': 5, 'fp': 5, 'tn': 10, 'fn': 0, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 9, 'tn': 1088, 'fn': 0, 'coverage': 1.0}}\n",
"LF_random_forest_prob_tuned\n",
"===== LF_logistic_regression_prob =========\n",
"x0 [1.0]\n",
"bounds [(0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.7014923095703125 s\n",
" direc: array([[0.14290116]])\n",
" fun: 1\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 64\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([0.23135423])\n",
"new x0 = [0.23135423]\n",
"-------NEW---------\n",
"{'x0': [array([0.23135423])], 'annotated': {'tp': 4, 'fp': 4, 'tn': 11, 'fn': 1, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_logistic_regression_prob_tuned\n",
"===== LF_xgboost_prob =========\n",
"x0 [1.0]\n",
"bounds [(0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n",
"Optimizing Time: 0.6952168941497803 s\n",
" direc: array([[0.14290116]])\n",
" fun: 1\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 64\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([0.23135423])\n",
"new x0 = [0.23135423]\n",
"-------NEW---------\n",
"{'x0': [array([0.23135423])], 'annotated': {'tp': 4, 'fp': 2, 'tn': 13, 'fn': 1, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_xgboost_prob_tuned\n",
"===== LF_mlp_prob =========\n",
"x0 [1.0]\n",
"bounds [(0, 1.0)]\n",
"-------OLD---------\n",
"{'x0': [1.0], 'annotated': {'tp': 0, 'fp': 0, 'tn': 15, 'fn': 5, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"num_predicted_matches = 0\n",
"updated boundary [(0, 0.98)]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Optimizing Time: 0.6936237812042236 s\n",
" direc: array([[0.14290116]])\n",
" fun: 1\n",
" message: 'Optimization terminated successfully.'\n",
" nfev: 64\n",
" nit: 2\n",
" status: 0\n",
" success: True\n",
" x: array([0.23135423])\n",
"new x0 = [0.23135423]\n",
"-------NEW---------\n",
"{'x0': [array([0.23135423])], 'annotated': {'tp': 4, 'fp': 4, 'tn': 11, 'fn': 1, 'coverage': 1.0}, 'predicted': {'tp': 0, 'fp': 0, 'tn': 1097, 'fn': 0, 'coverage': 1.0}}\n",
"LF_mlp_prob_tuned\n",
"LF augumentation time: 7.526500225067139 s\n",
"[0.42, 0.3, 0.1, 0.35, 0.1, 0.26249999999999996, 0.4666666666666666, 0.11666666666666665, 0.1, 0.1, 0.1, 0.1, 0.4666666666666666, 0.1, 0.1, 0.7, 0.1, 0.35, 0.1, 0.1, 0.3181818181818182, 0.4666666666666666, 0.175, 0.11666666666666665, 0.42, 0.4666666666666666, 0.35, 0.1, 0.4666666666666666, 0.12962962962962962]\n",
"Unexpected exception formatting exception. Falling back to standard exception\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Traceback (most recent call last):\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3398, in run_code\n",
" exec(code_obj, self.user_global_ns, self.user_ns)\n",
" File \"/tmp/ipykernel_2252406/4232351404.py\", line 13, in <cell line: 12>\n",
" run_experiment(cfg, dataset_name, method, case_name)\n",
" File \"/tmp/ipykernel_2252406/116442902.py\", line 39, in run_experiment\n",
" result = dual_loops.run_experiment(method_config, my_dataset_df, lfs_set, feature_set,\n",
" File \"../src/dualloop.py\", line 1622, in run_experiment\n",
" File \"../src/dualloop.py\", line 846, in lfs_ensemble\n",
" df['prediction'] = model.predict(test_x)\n",
" File \"../src/dualloop.py\", line 757, in apply_snorkel_with_init_precision\n",
" from sklearn.metrics import precision_score\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/snorkel/labeling/model/label_model.py\", line 936, in fit\n",
" loss.backward()\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/torch/_tensor.py\", line 255, in backward\n",
" torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/torch/autograd/__init__.py\", line 147, in backward\n",
" Variable._execution_engine.run_backward(\n",
"KeyboardInterrupt\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 1993, in showtraceback\n",
" stb = self.InteractiveTB.structured_traceback(\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 1118, in structured_traceback\n",
" return FormattedTB.structured_traceback(\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 1012, in structured_traceback\n",
" return VerboseTB.structured_traceback(\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 865, in structured_traceback\n",
" formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 818, in format_exception_as_a_whole\n",
" frames.append(self.format_record(r))\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 736, in format_record\n",
" result += ''.join(_format_traceback_lines(frame_info.lines, Colors, self.has_colors, lvals))\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/utils.py\", line 145, in cached_property_wrapper\n",
" value = obj.__dict__[self.func.__name__] = self.func(obj)\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/core.py\", line 698, in lines\n",
" pieces = self.included_pieces\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/utils.py\", line 145, in cached_property_wrapper\n",
" value = obj.__dict__[self.func.__name__] = self.func(obj)\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/core.py\", line 649, in included_pieces\n",
" pos = scope_pieces.index(self.executing_piece)\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/utils.py\", line 145, in cached_property_wrapper\n",
" value = obj.__dict__[self.func.__name__] = self.func(obj)\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/stack_data/core.py\", line 628, in executing_piece\n",
" return only(\n",
" File \"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/executing/executing.py\", line 164, in only\n",
" raise NotOneValueFound('Expected one value, found 0')\n",
"executing.executing.NotOneValueFound: Expected one value, found 0\n"
]
}
],
"source": [
"dataset_name = 'conference-1'\n",
"case_name = 'various_lfs'\n",
"methods = [\n",
" 'WeSAL', \n",
" 'AL-RF', \n",
" 'DualLoop',\n",
"# 'DualLoop_without_slow_loop',\n",
"# 'DualLoop_with_entropy',\n",
"# 'DualLoop_with_normal_ensemble'\n",
"]\n",
"\n",
"for method in methods:\n",
" run_experiment(cfg, dataset_name, method, case_name)\n",
" print(\"Done with \" + method)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e344ec03",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "078a7509",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}