{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "8dfe8c19",
"metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "129d60ef",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import dual_loops as dual_loops"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "3a0299dc",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/huggingface_hub/snapshot_download.py:6: FutureWarning: snapshot_download.py has been made private and will no longer be available from version 0.11. Please use `from huggingface_hub import snapshot_download` to import the only public function in this module. Other members of the file may be changed without a deprecation notice.\n",
" warnings.warn(\n"
]
}
],
"source": [
"dataset_name = 'conference'\n",
"#dataset_name = 'nasa'\n",
"#dataset_name = 'ai4eu'\n",
"\n",
"\n",
"with_blocking = True\n",
"\n",
"if with_blocking:\n",
" all_dataset_df = pd.read_csv(dataset_name + '_blocked.csv')\n",
"else:\n",
" all_dataset_df = pd.read_csv(dataset_name + '.csv')\n",
"\n",
"import pickle\n",
"source_ontology = pickle.load( open(dataset_name + \"_source_ontology.pk\", \"rb\" ) )\n",
"target_ontology = pickle.load( open(dataset_name + \"_target_ontology.pk\", \"rb\" ) )\n",
"\n",
"dual_loops.source_ontology = source_ontology\n",
"dual_loops.target_ontology = target_ontology"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "9b8c6f70",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>LF_class_name_equal</th>\n",
" <th>LF_class_name_stemmed_equal</th>\n",
" <th>LF_acronyms</th>\n",
" <th>LF_class_name_synonyms</th>\n",
" <th>LF_label_equal</th>\n",
" <th>LF_root_nouns_equal</th>\n",
" <th>LF_class_name_spacy_distance</th>\n",
" <th>LF_class_name_distance</th>\n",
" <th>LF_name_segment_overlap</th>\n",
" <th>LF_label_words_overlap</th>\n",
" <th>...</th>\n",
" <th>comment_distance_a</th>\n",
" <th>comment_distance_b</th>\n",
" <th>num_common_words_blocked</th>\n",
" <th>class_long_name_distance_a_blocked</th>\n",
" <th>label_long_distance_a_blocked</th>\n",
" <th>comment_distance_a_blocked</th>\n",
" <th>class_long_name_distance_b_blocked</th>\n",
" <th>label_long_distance_b_blocked</th>\n",
" <th>comment_distance_b_blocked</th>\n",
" <th>selected_after_blocking</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9294</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9295</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9296</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9297</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9298</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>9299 rows × 36 columns</p>\n",
"</div>"
],
"text/plain": [
" LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n",
"0 -1 -1 -1 \n",
"1 -1 -1 -1 \n",
"2 -1 -1 -1 \n",
"3 -1 -1 -1 \n",
"4 -1 -1 -1 \n",
"... ... ... ... \n",
"9294 -1 -1 -1 \n",
"9295 -1 -1 -1 \n",
"9296 -1 -1 -1 \n",
"9297 -1 -1 -1 \n",
"9298 -1 -1 -1 \n",
"\n",
" LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n",
"0 -1 -1 -1 \n",
"1 -1 -1 -1 \n",
"2 -1 -1 -1 \n",
"3 -1 -1 -1 \n",
"4 -1 -1 -1 \n",
"... ... ... ... \n",
"9294 -1 -1 -1 \n",
"9295 -1 -1 -1 \n",
"9296 -1 -1 -1 \n",
"9297 -1 -1 -1 \n",
"9298 -1 -1 -1 \n",
"\n",
" LF_class_name_spacy_distance LF_class_name_distance \\\n",
"0 -1 -1 \n",
"1 0 0 \n",
"2 -1 0 \n",
"3 -1 0 \n",
"4 0 0 \n",
"... ... ... \n",
"9294 -1 0 \n",
"9295 -1 0 \n",
"9296 -1 0 \n",
"9297 -1 0 \n",
"9298 -1 0 \n",
"\n",
" LF_name_segment_overlap LF_label_words_overlap ... \\\n",
"0 -1 -1 ... \n",
"1 -1 -1 ... \n",
"2 -1 -1 ... \n",
"3 -1 -1 ... \n",
"4 -1 -1 ... \n",
"... ... ... ... \n",
"9294 -1 -1 ... \n",
"9295 -1 -1 ... \n",
"9296 -1 -1 ... \n",
"9297 -1 -1 ... \n",
"9298 -1 -1 ... \n",
"\n",
" comment_distance_a comment_distance_b num_common_words_blocked \\\n",
"0 NaN NaN 1 \n",
"1 NaN NaN 1 \n",
"2 NaN NaN 1 \n",
"3 NaN NaN 1 \n",
"4 NaN NaN 1 \n",
"... ... ... ... \n",
"9294 NaN NaN 1 \n",
"9295 NaN NaN 1 \n",
"9296 NaN NaN 1 \n",
"9297 NaN NaN 1 \n",
"9298 NaN NaN 1 \n",
"\n",
" class_long_name_distance_a_blocked label_long_distance_a_blocked \\\n",
"0 1 1 \n",
"1 0 1 \n",
"2 1 1 \n",
"3 1 1 \n",
"4 0 1 \n",
"... ... ... \n",
"9294 0 1 \n",
"9295 0 1 \n",
"9296 0 1 \n",
"9297 1 1 \n",
"9298 0 1 \n",
"\n",
" comment_distance_a_blocked class_long_name_distance_b_blocked \\\n",
"0 1 0 \n",
"1 1 1 \n",
"2 1 0 \n",
"3 1 0 \n",
"4 1 1 \n",
"... ... ... \n",
"9294 1 0 \n",
"9295 1 1 \n",
"9296 1 0 \n",
"9297 1 0 \n",
"9298 1 0 \n",
"\n",
" label_long_distance_b_blocked comment_distance_b_blocked \\\n",
"0 1 1 \n",
"1 1 1 \n",
"2 1 1 \n",
"3 1 1 \n",
"4 1 1 \n",
"... ... ... \n",
"9294 1 1 \n",
"9295 1 1 \n",
"9296 1 1 \n",
"9297 1 1 \n",
"9298 1 1 \n",
"\n",
" selected_after_blocking \n",
"0 1 \n",
"1 1 \n",
"2 1 \n",
"3 1 \n",
"4 1 \n",
"... ... \n",
"9294 1 \n",
"9295 1 \n",
"9296 1 \n",
"9297 1 \n",
"9298 1 \n",
"\n",
"[9299 rows x 36 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"my_dataset_df = all_dataset_df[all_dataset_df['selected_after_blocking'] == 1].reset_index(drop=True)\n",
"my_dataset_df"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "e9a75d4c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>LF_class_name_equal</th>\n",
" <th>LF_class_name_stemmed_equal</th>\n",
" <th>LF_acronyms</th>\n",
" <th>LF_class_name_synonyms</th>\n",
" <th>LF_label_equal</th>\n",
" <th>LF_root_nouns_equal</th>\n",
" <th>LF_class_name_spacy_distance</th>\n",
" <th>LF_class_name_distance</th>\n",
" <th>LF_name_segment_overlap</th>\n",
" <th>LF_label_words_overlap</th>\n",
" <th>...</th>\n",
" <th>comment_distance_a</th>\n",
" <th>comment_distance_b</th>\n",
" <th>num_common_words_blocked</th>\n",
" <th>class_long_name_distance_a_blocked</th>\n",
" <th>label_long_distance_a_blocked</th>\n",
" <th>comment_distance_a_blocked</th>\n",
" <th>class_long_name_distance_b_blocked</th>\n",
" <th>label_long_distance_b_blocked</th>\n",
" <th>comment_distance_b_blocked</th>\n",
" <th>selected_after_blocking</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>83</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>91</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>153</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>209</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>226</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8294</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8360</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8533</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9003</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9075</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>65 rows × 36 columns</p>\n",
"</div>"
],
"text/plain": [
" LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n",
"83 1 1 -1 \n",
"91 1 1 -1 \n",
"153 -1 -1 -1 \n",
"209 1 1 -1 \n",
"226 -1 -1 -1 \n",
"... ... ... ... \n",
"8294 -1 -1 -1 \n",
"8360 1 1 -1 \n",
"8533 1 1 -1 \n",
"9003 -1 -1 -1 \n",
"9075 1 1 -1 \n",
"\n",
" LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n",
"83 1 -1 1 \n",
"91 1 -1 1 \n",
"153 -1 -1 -1 \n",
"209 1 -1 1 \n",
"226 -1 -1 -1 \n",
"... ... ... ... \n",
"8294 -1 -1 -1 \n",
"8360 1 -1 1 \n",
"8533 1 -1 1 \n",
"9003 -1 -1 -1 \n",
"9075 1 -1 1 \n",
"\n",
" LF_class_name_spacy_distance LF_class_name_distance \\\n",
"83 1 1 \n",
"91 1 1 \n",
"153 1 1 \n",
"209 1 1 \n",
"226 -1 -1 \n",
"... ... ... \n",
"8294 -1 -1 \n",
"8360 1 1 \n",
"8533 1 1 \n",
"9003 -1 -1 \n",
"9075 1 1 \n",
"\n",
" LF_name_segment_overlap LF_label_words_overlap ... \\\n",
"83 1 -1 ... \n",
"91 1 -1 ... \n",
"153 1 -1 ... \n",
"209 1 -1 ... \n",
"226 1 -1 ... \n",
"... ... ... ... \n",
"8294 1 -1 ... \n",
"8360 1 -1 ... \n",
"8533 1 -1 ... \n",
"9003 1 -1 ... \n",
"9075 1 -1 ... \n",
"\n",
" comment_distance_a comment_distance_b num_common_words_blocked \\\n",
"83 NaN NaN 0 \n",
"91 NaN NaN 0 \n",
"153 NaN NaN 0 \n",
"209 NaN NaN 0 \n",
"226 NaN NaN 0 \n",
"... ... ... ... \n",
"8294 NaN NaN 0 \n",
"8360 NaN NaN 0 \n",
"8533 NaN NaN 0 \n",
"9003 NaN NaN 0 \n",
"9075 NaN NaN 0 \n",
"\n",
" class_long_name_distance_a_blocked label_long_distance_a_blocked \\\n",
"83 0 1 \n",
"91 0 1 \n",
"153 0 1 \n",
"209 0 1 \n",
"226 0 1 \n",
"... ... ... \n",
"8294 0 1 \n",
"8360 0 1 \n",
"8533 0 1 \n",
"9003 0 1 \n",
"9075 0 1 \n",
"\n",
" comment_distance_a_blocked class_long_name_distance_b_blocked \\\n",
"83 1 0 \n",
"91 1 0 \n",
"153 1 0 \n",
"209 1 0 \n",
"226 1 0 \n",
"... ... ... \n",
"8294 1 0 \n",
"8360 1 0 \n",
"8533 1 0 \n",
"9003 1 0 \n",
"9075 1 0 \n",
"\n",
" label_long_distance_b_blocked comment_distance_b_blocked \\\n",
"83 1 1 \n",
"91 1 1 \n",
"153 1 1 \n",
"209 1 1 \n",
"226 1 1 \n",
"... ... ... \n",
"8294 1 1 \n",
"8360 1 1 \n",
"8533 1 1 \n",
"9003 1 1 \n",
"9075 1 1 \n",
"\n",
" selected_after_blocking \n",
"83 1 \n",
"91 1 \n",
"153 1 \n",
"209 1 \n",
"226 1 \n",
"... ... \n",
"8294 1 \n",
"8360 1 \n",
"8533 1 \n",
"9003 1 \n",
"9075 1 \n",
"\n",
"[65 rows x 36 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"my_dataset_df[my_dataset_df['label']==1]"
]
},
{
"cell_type": "markdown",
"id": "8314da1d",
"metadata": {},
"source": [
"# define the configuration of all experiments"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "1776e603",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r\n",
"\r\n",
"************* WeSAL *************************\n"
]
},
{
"ename": "TypeError",
"evalue": "len() of unsized object",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 84>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m************* \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m exp \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m *************************\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 86\u001b[0m experiment_config \u001b[38;5;241m=\u001b[39m configuration_options[exp]\n\u001b[0;32m---> 87\u001b[0m result, result_df \u001b[38;5;241m=\u001b[39m \u001b[43mdual_loops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_experiment\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexperiment_config\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmy_dataset_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlfs_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeature_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_iteration\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minterval_slow_loop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbudget\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 89\u001b[0m results[exp] \u001b[38;5;241m=\u001b[39m result\n",
"File \u001b[0;32m~/aaai22/dual_loops.py:1481\u001b[0m, in \u001b[0;36mrun_experiment\u001b[0;34m(experiment_conf, df, pool_lfs, feature_set, total_size, num_iteration, interval_slow_loop, balance, epochs, debug)\u001b[0m\n\u001b[1;32m 1478\u001b[0m new_lfs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 1480\u001b[0m \u001b[38;5;66;03m# initialize the prediction result given by weak supervision without human annotations \u001b[39;00m\n\u001b[0;32m-> 1481\u001b[0m \u001b[43mlfs_ensemble\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexperiment_conf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlf_ensemble\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmy_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpool_lfs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeature_set\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 1482\u001b[0m r \u001b[38;5;241m=\u001b[39m evaluate_result(my_df) \n\u001b[1;32m 1483\u001b[0m measurements[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m r\n",
"File \u001b[0;32m~/aaai22/dual_loops.py:794\u001b[0m, in \u001b[0;36mlfs_ensemble\u001b[0;34m(strategy_name, df, lfs, features, balance, epochs)\u001b[0m\n\u001b[1;32m 792\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m apply_snorkel(df, lfs, balance, epochs) \n\u001b[1;32m 793\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m strategy_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msnorkel_with_corrected_votes\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 794\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapply_snorkel_with_vote_correction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlfs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 795\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m strategy_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msnorkel_with_annotated_labels\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 796\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m apply_snorkel_with_annotated_labels(df, lfs, balance, epochs)\n",
"File \u001b[0;32m~/aaai22/dual_loops.py:630\u001b[0m, in \u001b[0;36mapply_snorkel_with_vote_correction\u001b[0;34m(df, lfs, balance, epochs)\u001b[0m\n\u001b[1;32m 628\u001b[0m \u001b[38;5;66;03m# train the generative model\u001b[39;00m\n\u001b[1;32m 629\u001b[0m label_model \u001b[38;5;241m=\u001b[39m LabelModel(cardinality\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m--> 630\u001b[0m \u001b[43mlabel_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mL_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabel_matrix\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mclass_balance\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbalance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 631\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog_freq\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m123\u001b[39;49m\u001b[43m)\u001b[49m \n\u001b[1;32m 633\u001b[0m \u001b[38;5;66;03m# apply the trained generative model to predict the result\u001b[39;00m\n\u001b[1;32m 634\u001b[0m Y_pred, Y_prob \u001b[38;5;241m=\u001b[39m label_model\u001b[38;5;241m.\u001b[39mpredict(L\u001b[38;5;241m=\u001b[39mlabel_matrix, return_probs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, tie_break_policy\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mabstain\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
"File \u001b[0;32m~/aaai22/snorkel/labeling/model/label_model.py:896\u001b[0m, in \u001b[0;36mLabelModel.fit\u001b[0;34m(self, L_train, Y_dev, class_balance, progress_bar, **kwargs)\u001b[0m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 892\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mL_train has cardinality \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mL_shift\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, cardinality=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m passed in.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 893\u001b[0m )\n\u001b[1;32m 895\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_constants(L_shift)\n\u001b[0;32m--> 896\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_set_class_balance\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclass_balance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mY_dev\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 897\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_tree()\n\u001b[1;32m 898\u001b[0m lf_analysis \u001b[38;5;241m=\u001b[39m LFAnalysis(L_train)\n",
"File \u001b[0;32m~/aaai22/snorkel/labeling/model/label_model.py:571\u001b[0m, in \u001b[0;36mLabelModel._set_class_balance\u001b[0;34m(self, class_balance, Y_dev)\u001b[0m\n\u001b[1;32m 569\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m class_balance \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 570\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mp \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray(class_balance)\n\u001b[0;32m--> 571\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality:\n\u001b[1;32m 572\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 573\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclass_balance has \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mp)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m entries. Does not match LabelModel cardinality \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcardinality\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 574\u001b[0m )\n\u001b[1;32m 575\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m Y_dev \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
"\u001b[0;31mTypeError\u001b[0m: len() of unsized object"
]
}
],
"source": [
"configuration_options = { \n",
" 'WeSAL': {\n",
" 'datapoint_grouping': 'disagreement', \n",
" 'group_selection': 'max',\n",
" 'datapoint_selection': 'entropy', # best option for uncertainty sampling\n",
" 'batch_size': 100, \n",
" \n",
" 'lf_ensemble': 'snorkel_with_corrected_votes',\n",
" \n",
" 'lf_selection': 'all', \n",
" 'with_slow_loop': False \n",
" }, \n",
" 'ActiveLearning': {\n",
" 'datapoint_grouping': 'none', \n",
" 'group_selection': 'none',\n",
" 'datapoint_selection': 'entropy', #entropy, least_confidence, margin\n",
" 'batch_size': 100, \n",
" \n",
" 'lf_ensemble': 'normal_active_learning_rf',\n",
" \n",
" 'lf_selection': 'all', \n",
" 'with_slow_loop': False \n",
" }, \n",
" 'DualLoops': {\n",
" 'datapoint_grouping': 'num_positive_votes', \n",
" 'group_selection': 'max',\n",
" 'datapoint_selection': 'match_confidence', #entropy, least_confidence, margin\n",
" 'batch_size': 100, \n",
" \n",
" 'lf_ensemble': 'snorkel_with_init_precision', #snorkel_with_init_precision\n",
" \n",
" 'lf_selection': 'f1_first', \n",
" 'with_slow_loop': True\n",
" }, \n",
"}\n",
"\n",
"\n",
"\n",
"lfs_set = [\n",
"# 'LF_aml', \n",
"# 'LF_logmap', \n",
"# 'LF_yam',\n",
" \n",
" 'LF_class_name_equal', \n",
" 'LF_class_name_stemmed_equal',\n",
" 'LF_acronyms', \n",
" 'LF_class_name_synonyms',\n",
" 'LF_label_equal', \n",
" 'LF_root_nouns_equal', \n",
" \n",
" 'LF_class_name_spacy_distance', \n",
" 'LF_class_name_distance', \n",
" \n",
" 'LF_name_segment_overlap', \n",
" 'LF_label_words_overlap',\n",
" 'LF_subclasses_overlap',\n",
" 'LF_superclasses_overlap',\n",
" 'LF_properties_overlap',\n",
" ]\n",
"\n",
"feature_set = [\n",
" 'shared_word', \n",
" 'levenshtein_distance', \n",
" 'hamming_distance',\n",
" 'class_name_embedding_distance'\n",
" ]\n",
"\n",
"\n",
"epochs = 100\n",
"balance=[0.9, 0.1]\n",
"\n",
"num_iteration = 200\n",
"interval_slow_loop = 10\n",
"budget = 2000\n",
"\n",
"experiments = [ \n",
" 'WeSAL',\n",
" 'ActiveLearning', \n",
" 'DualLoops', \n",
"]\n",
"\n",
"results = {}\n",
"\n",
"for exp in experiments:\n",
" print(\"\\r\\n\\r\\n************* \" + exp + \" *************************\")\n",
" experiment_config = configuration_options[exp]\n",
" result, result_df = dual_loops.run_experiment(experiment_config, my_dataset_df, lfs_set, feature_set, num_iteration, interval_slow_loop, balance, epochs, budget, False)\n",
" \n",
" results[exp] = result \n"
]
},
{
"cell_type": "markdown",
"id": "b4651040",
"metadata": {},
"source": [
"# draw the figure"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "816ede8c",
"metadata": {},
"outputs": [],
"source": [
"interval = 20\n",
"methods = ['WeSAL', 'ActiveLearning', 'DualLoops-3']\n",
"labels = ['WeSAL', 'Active Learning', 'DualLoops (fastloop and slowloop)' ]\n",
"filename = dataset_name + '_dualloop_result.png'\n",
"\n",
"dual_loops.plot_comparison_result(results, methods, labels, interval)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}