{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "8dfe8c19", "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "id": "129d60ef", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import dual_loops as dual_loops" ] }, { "cell_type": "code", "execution_count": 3, "id": "3a0299dc", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/huggingface_hub/snapshot_download.py:6: FutureWarning: snapshot_download.py has been made private and will no longer be available from version 0.11. Please use `from huggingface_hub import snapshot_download` to import the only public function in this module. Other members of the file may be changed without a deprecation notice.\n", " warnings.warn(\n" ] } ], "source": [ "dataset_name = 'conference'\n", "#dataset_name = 'nasa'\n", "#dataset_name = 'ai4eu'\n", "\n", "\n", "with_blocking = True\n", "\n", "if with_blocking:\n", " all_dataset_df = pd.read_csv(dataset_name + '_blocked.csv')\n", "else:\n", " all_dataset_df = pd.read_csv(dataset_name + '.csv')\n", "\n", "import pickle\n", "source_ontology = pickle.load( open(dataset_name + \"_source_ontology.pk\", \"rb\" ) )\n", "target_ontology = pickle.load( open(dataset_name + \"_target_ontology.pk\", \"rb\" ) )\n", "\n", "dual_loops.source_ontology = source_ontology\n", "dual_loops.target_ontology = target_ontology" ] }, { "cell_type": "code", "execution_count": 4, "id": "9b8c6f70", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...comment_distance_acomment_distance_bnum_common_words_blockedclass_long_name_distance_a_blockedlabel_long_distance_a_blockedcomment_distance_a_blockedclass_long_name_distance_b_blockedlabel_long_distance_b_blockedcomment_distance_b_blockedselected_after_blocking
0-1-1-1-1-1-1-1-1-1-1...NaNNaN11110111
1-1-1-1-1-1-100-1-1...NaNNaN10111111
2-1-1-1-1-1-1-10-1-1...NaNNaN11110111
3-1-1-1-1-1-1-10-1-1...NaNNaN11110111
4-1-1-1-1-1-100-1-1...NaNNaN10111111
..................................................................
9294-1-1-1-1-1-1-10-1-1...NaNNaN10110111
9295-1-1-1-1-1-1-10-1-1...NaNNaN10111111
9296-1-1-1-1-1-1-10-1-1...NaNNaN10110111
9297-1-1-1-1-1-1-10-1-1...NaNNaN11110111
9298-1-1-1-1-1-1-10-1-1...NaNNaN10110111
\n", "

9299 rows × 36 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "9294 -1 -1 -1 \n", "9295 -1 -1 -1 \n", "9296 -1 -1 -1 \n", "9297 -1 -1 -1 \n", "9298 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "9294 -1 -1 -1 \n", "9295 -1 -1 -1 \n", "9296 -1 -1 -1 \n", "9297 -1 -1 -1 \n", "9298 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "0 -1 -1 \n", "1 0 0 \n", "2 -1 0 \n", "3 -1 0 \n", "4 0 0 \n", "... ... ... \n", "9294 -1 0 \n", "9295 -1 0 \n", "9296 -1 0 \n", "9297 -1 0 \n", "9298 -1 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "0 -1 -1 ... \n", "1 -1 -1 ... \n", "2 -1 -1 ... \n", "3 -1 -1 ... \n", "4 -1 -1 ... \n", "... ... ... ... \n", "9294 -1 -1 ... \n", "9295 -1 -1 ... \n", "9296 -1 -1 ... \n", "9297 -1 -1 ... \n", "9298 -1 -1 ... \n", "\n", " comment_distance_a comment_distance_b num_common_words_blocked \\\n", "0 NaN NaN 1 \n", "1 NaN NaN 1 \n", "2 NaN NaN 1 \n", "3 NaN NaN 1 \n", "4 NaN NaN 1 \n", "... ... ... ... \n", "9294 NaN NaN 1 \n", "9295 NaN NaN 1 \n", "9296 NaN NaN 1 \n", "9297 NaN NaN 1 \n", "9298 NaN NaN 1 \n", "\n", " class_long_name_distance_a_blocked label_long_distance_a_blocked \\\n", "0 1 1 \n", "1 0 1 \n", "2 1 1 \n", "3 1 1 \n", "4 0 1 \n", "... ... ... \n", "9294 0 1 \n", "9295 0 1 \n", "9296 0 1 \n", "9297 1 1 \n", "9298 0 1 \n", "\n", " comment_distance_a_blocked class_long_name_distance_b_blocked \\\n", "0 1 0 \n", "1 1 1 \n", "2 1 0 \n", "3 1 0 \n", "4 1 1 \n", "... ... ... \n", "9294 1 0 \n", "9295 1 1 \n", "9296 1 0 \n", "9297 1 0 \n", "9298 1 0 \n", "\n", " label_long_distance_b_blocked comment_distance_b_blocked \\\n", "0 1 1 \n", "1 1 1 \n", "2 1 1 \n", "3 1 1 \n", "4 1 1 \n", "... ... ... \n", "9294 1 1 \n", "9295 1 1 \n", "9296 1 1 \n", "9297 1 1 \n", "9298 1 1 \n", "\n", " selected_after_blocking \n", "0 1 \n", "1 1 \n", "2 1 \n", "3 1 \n", "4 1 \n", "... ... \n", "9294 1 \n", "9295 1 \n", "9296 1 \n", "9297 1 \n", "9298 1 \n", "\n", "[9299 rows x 36 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "my_dataset_df = all_dataset_df[all_dataset_df['selected_after_blocking'] == 1].reset_index(drop=True)\n", "my_dataset_df" ] }, { "cell_type": "code", "execution_count": 5, "id": "e9a75d4c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...comment_distance_acomment_distance_bnum_common_words_blockedclass_long_name_distance_a_blockedlabel_long_distance_a_blockedcomment_distance_a_blockedclass_long_name_distance_b_blockedlabel_long_distance_b_blockedcomment_distance_b_blockedselected_after_blocking
8311-11-11111-1...NaNNaN00110111
9111-11-11111-1...NaNNaN00110111
153-1-1-1-1-1-1111-1...NaNNaN00110111
20911-11-11111-1...NaNNaN00110111
226-1-1-1-1-1-1-1-11-1...NaNNaN00110111
..................................................................
8294-1-1-1-1-1-1-1-11-1...NaNNaN00110111
836011-11-11111-1...NaNNaN00110111
853311-11-11111-1...NaNNaN00110111
9003-1-1-1-1-1-1-1-11-1...NaNNaN00110111
907511-11-11111-1...NaNNaN00110111
\n", "

65 rows × 36 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "83 1 1 -1 \n", "91 1 1 -1 \n", "153 -1 -1 -1 \n", "209 1 1 -1 \n", "226 -1 -1 -1 \n", "... ... ... ... \n", "8294 -1 -1 -1 \n", "8360 1 1 -1 \n", "8533 1 1 -1 \n", "9003 -1 -1 -1 \n", "9075 1 1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "83 1 -1 1 \n", "91 1 -1 1 \n", "153 -1 -1 -1 \n", "209 1 -1 1 \n", "226 -1 -1 -1 \n", "... ... ... ... \n", "8294 -1 -1 -1 \n", "8360 1 -1 1 \n", "8533 1 -1 1 \n", "9003 -1 -1 -1 \n", "9075 1 -1 1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "83 1 1 \n", "91 1 1 \n", "153 1 1 \n", "209 1 1 \n", "226 -1 -1 \n", "... ... ... \n", "8294 -1 -1 \n", "8360 1 1 \n", "8533 1 1 \n", "9003 -1 -1 \n", "9075 1 1 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "83 1 -1 ... \n", "91 1 -1 ... \n", "153 1 -1 ... \n", "209 1 -1 ... \n", "226 1 -1 ... \n", "... ... ... ... \n", "8294 1 -1 ... \n", "8360 1 -1 ... \n", "8533 1 -1 ... \n", "9003 1 -1 ... \n", "9075 1 -1 ... \n", "\n", " comment_distance_a comment_distance_b num_common_words_blocked \\\n", "83 NaN NaN 0 \n", "91 NaN NaN 0 \n", "153 NaN NaN 0 \n", "209 NaN NaN 0 \n", "226 NaN NaN 0 \n", "... ... ... ... \n", "8294 NaN NaN 0 \n", "8360 NaN NaN 0 \n", "8533 NaN NaN 0 \n", "9003 NaN NaN 0 \n", "9075 NaN NaN 0 \n", "\n", " class_long_name_distance_a_blocked label_long_distance_a_blocked \\\n", "83 0 1 \n", "91 0 1 \n", "153 0 1 \n", "209 0 1 \n", "226 0 1 \n", "... ... ... \n", "8294 0 1 \n", "8360 0 1 \n", "8533 0 1 \n", "9003 0 1 \n", "9075 0 1 \n", "\n", " comment_distance_a_blocked class_long_name_distance_b_blocked \\\n", "83 1 0 \n", "91 1 0 \n", "153 1 0 \n", "209 1 0 \n", "226 1 0 \n", "... ... ... \n", "8294 1 0 \n", "8360 1 0 \n", "8533 1 0 \n", "9003 1 0 \n", "9075 1 0 \n", "\n", " label_long_distance_b_blocked comment_distance_b_blocked \\\n", "83 1 1 \n", "91 1 1 \n", "153 1 1 \n", "209 1 1 \n", "226 1 1 \n", "... ... ... \n", "8294 1 1 \n", "8360 1 1 \n", "8533 1 1 \n", "9003 1 1 \n", "9075 1 1 \n", "\n", " selected_after_blocking \n", "83 1 \n", "91 1 \n", "153 1 \n", "209 1 \n", "226 1 \n", "... ... \n", "8294 1 \n", "8360 1 \n", "8533 1 \n", "9003 1 \n", "9075 1 \n", "\n", "[65 rows x 36 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "my_dataset_df[my_dataset_df['label']==1]" ] }, { "cell_type": "markdown", "id": "8314da1d", "metadata": {}, "source": [ "# define the configuration of all experiments" ] }, { "cell_type": "code", "execution_count": 7, "id": "1776e603", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\r\n", "\r\n", "************* WeSAL *************************\n" ] }, { "ename": "TypeError", "evalue": "len() of unsized object", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m************* \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m exp \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m *************************\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 86\u001b[0m experiment_config \u001b[38;5;241m=\u001b[39m configuration_options[exp]\n\u001b[0;32m---> 87\u001b[0m result, result_df \u001b[38;5;241m=\u001b[39m \u001b[43mdual_loops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_experiment\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexperiment_config\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmy_dataset_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlfs_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeature_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_iteration\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minterval_slow_loop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbudget\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 89\u001b[0m results[exp] \u001b[38;5;241m=\u001b[39m result\n", "File \u001b[0;32m~/aaai22/dual_loops.py:1481\u001b[0m, in \u001b[0;36mrun_experiment\u001b[0;34m(experiment_conf, df, pool_lfs, feature_set, total_size, num_iteration, interval_slow_loop, balance, epochs, debug)\u001b[0m\n\u001b[1;32m 1478\u001b[0m new_lfs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 1480\u001b[0m \u001b[38;5;66;03m# initialize the prediction result given by weak supervision without human annotations \u001b[39;00m\n\u001b[0;32m-> 1481\u001b[0m \u001b[43mlfs_ensemble\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexperiment_conf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlf_ensemble\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmy_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpool_lfs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeature_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 1482\u001b[0m r \u001b[38;5;241m=\u001b[39m evaluate_result(my_df) \n\u001b[1;32m 1483\u001b[0m measurements[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m r\n", "File \u001b[0;32m~/aaai22/dual_loops.py:794\u001b[0m, in \u001b[0;36mlfs_ensemble\u001b[0;34m(strategy_name, df, lfs, features, balance, epochs)\u001b[0m\n\u001b[1;32m 792\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m apply_snorkel(df, lfs, balance, epochs) \n\u001b[1;32m 793\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m strategy_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msnorkel_with_corrected_votes\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 794\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapply_snorkel_with_vote_correction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlfs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 795\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m strategy_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msnorkel_with_annotated_labels\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 796\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m apply_snorkel_with_annotated_labels(df, lfs, balance, epochs)\n", "File \u001b[0;32m~/aaai22/dual_loops.py:630\u001b[0m, in \u001b[0;36mapply_snorkel_with_vote_correction\u001b[0;34m(df, lfs, balance, epochs)\u001b[0m\n\u001b[1;32m 628\u001b[0m \u001b[38;5;66;03m# train the generative model\u001b[39;00m\n\u001b[1;32m 629\u001b[0m label_model \u001b[38;5;241m=\u001b[39m LabelModel(cardinality\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m--> 630\u001b[0m \u001b[43mlabel_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mL_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel_matrix\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mclass_balance\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 631\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog_freq\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m123\u001b[39;49m\u001b[43m)\u001b[49m \n\u001b[1;32m 633\u001b[0m \u001b[38;5;66;03m# apply the trained generative model to predict the result\u001b[39;00m\n\u001b[1;32m 634\u001b[0m Y_pred, Y_prob \u001b[38;5;241m=\u001b[39m label_model\u001b[38;5;241m.\u001b[39mpredict(L\u001b[38;5;241m=\u001b[39mlabel_matrix, return_probs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, tie_break_policy\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mabstain\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", "File \u001b[0;32m~/aaai22/snorkel/labeling/model/label_model.py:896\u001b[0m, in \u001b[0;36mLabelModel.fit\u001b[0;34m(self, L_train, Y_dev, class_balance, progress_bar, **kwargs)\u001b[0m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 892\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mL_train has cardinality \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mL_shift\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, cardinality=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m passed in.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 893\u001b[0m )\n\u001b[1;32m 895\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_constants(L_shift)\n\u001b[0;32m--> 896\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_set_class_balance\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclass_balance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mY_dev\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 897\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_tree()\n\u001b[1;32m 898\u001b[0m lf_analysis \u001b[38;5;241m=\u001b[39m LFAnalysis(L_train)\n", "File \u001b[0;32m~/aaai22/snorkel/labeling/model/label_model.py:571\u001b[0m, in \u001b[0;36mLabelModel._set_class_balance\u001b[0;34m(self, class_balance, Y_dev)\u001b[0m\n\u001b[1;32m 569\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m class_balance \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 570\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mp \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray(class_balance)\n\u001b[0;32m--> 571\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality:\n\u001b[1;32m 572\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 573\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclass_balance has \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mp)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m entries. Does not match LabelModel cardinality \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 574\u001b[0m )\n\u001b[1;32m 575\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m Y_dev \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", "\u001b[0;31mTypeError\u001b[0m: len() of unsized object" ] } ], "source": [ "configuration_options = { \n", " 'WeSAL': {\n", " 'datapoint_grouping': 'disagreement', \n", " 'group_selection': 'max',\n", " 'datapoint_selection': 'entropy', # best option for uncertainty sampling\n", " 'batch_size': 100, \n", " \n", " 'lf_ensemble': 'snorkel_with_corrected_votes',\n", " \n", " 'lf_selection': 'all', \n", " 'with_slow_loop': False \n", " }, \n", " 'ActiveLearning': {\n", " 'datapoint_grouping': 'none', \n", " 'group_selection': 'none',\n", " 'datapoint_selection': 'entropy', #entropy, least_confidence, margin\n", " 'batch_size': 100, \n", " \n", " 'lf_ensemble': 'normal_active_learning_rf',\n", " \n", " 'lf_selection': 'all', \n", " 'with_slow_loop': False \n", " }, \n", " 'DualLoops': {\n", " 'datapoint_grouping': 'num_positive_votes', \n", " 'group_selection': 'max',\n", " 'datapoint_selection': 'match_confidence', #entropy, least_confidence, margin\n", " 'batch_size': 100, \n", " \n", " 'lf_ensemble': 'snorkel_with_init_precision', #snorkel_with_init_precision\n", " \n", " 'lf_selection': 'f1_first', \n", " 'with_slow_loop': True\n", " }, \n", "}\n", "\n", "\n", "\n", "lfs_set = [\n", "# 'LF_aml', \n", "# 'LF_logmap', \n", "# 'LF_yam',\n", " \n", " 'LF_class_name_equal', \n", " 'LF_class_name_stemmed_equal',\n", " 'LF_acronyms', \n", " 'LF_class_name_synonyms',\n", " 'LF_label_equal', \n", " 'LF_root_nouns_equal', \n", " \n", " 'LF_class_name_spacy_distance', \n", " 'LF_class_name_distance', \n", " \n", " 'LF_name_segment_overlap', \n", " 'LF_label_words_overlap',\n", " 'LF_subclasses_overlap',\n", " 'LF_superclasses_overlap',\n", " 'LF_properties_overlap',\n", " ]\n", "\n", "feature_set = [\n", " 'shared_word', \n", " 'levenshtein_distance', \n", " 'hamming_distance',\n", " 'class_name_embedding_distance'\n", " ]\n", "\n", "\n", "epochs = 100\n", "balance=[0.9, 0.1]\n", "\n", "num_iteration = 200\n", "interval_slow_loop = 10\n", "budget = 2000\n", "\n", "experiments = [ \n", " 'WeSAL',\n", " 'ActiveLearning', \n", " 'DualLoops', \n", "]\n", "\n", "results = {}\n", "\n", "for exp in experiments:\n", " print(\"\\r\\n\\r\\n************* \" + exp + \" *************************\")\n", " experiment_config = configuration_options[exp]\n", " result, result_df = dual_loops.run_experiment(experiment_config, my_dataset_df, lfs_set, feature_set, num_iteration, interval_slow_loop, balance, epochs, budget, False)\n", " \n", " results[exp] = result \n" ] }, { "cell_type": "markdown", "id": "b4651040", "metadata": {}, "source": [ "# draw the figure" ] }, { "cell_type": "code", "execution_count": null, "id": "816ede8c", "metadata": {}, "outputs": [], "source": [ "interval = 20\n", "methods = ['WeSAL', 'ActiveLearning', 'DualLoops-3']\n", "labels = ['WeSAL', 'Active Learning', 'DualLoops (fastloop and slowloop)' ]\n", "filename = dataset_name + '_dualloop_result.png'\n", "\n", "dual_loops.plot_comparison_result(results, methods, labels, interval)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 5 }