{ "cells": [ { "cell_type": "markdown", "id": "f7e80ce6", "metadata": {}, "source": [ "# apply the blocking algorithm" ] }, { "cell_type": "code", "execution_count": 1, "id": "97789d42", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import pickle\n", "\n", "from change_finder import ChangeFinder, SDAR\n", "import matplotlib.pyplot as plt\n", "import ruptures as rpt # our package\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "974b97c6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/huggingface_hub/snapshot_download.py:6: FutureWarning: snapshot_download.py has been made private and will no longer be available from version 0.11. Please use `from huggingface_hub import snapshot_download` to import the only public function in this module. Other members of the file may be changed without a deprecation notice.\n", " warnings.warn(\n" ] } ], "source": [ "#dataset_name = 'nasa'\n", "#dataset_name = 'conference'\n", "#dataset_name = 'ai4eu'\n", "\n", "all_dataset_df = pd.read_csv(dataset_name + '.csv')\n", "\n", "import pickle\n", "source_ontology = pickle.load( open(dataset_name + \"_source_ontology.pk\", \"rb\" ) )\n", "target_ontology = pickle.load( open(dataset_name + \"_target_ontology.pk\", \"rb\" ) )" ] }, { "cell_type": "code", "execution_count": 3, "id": "a959c5b4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "915" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(target_ontology)" ] }, { "cell_type": "code", "execution_count": 4, "id": "28f9bd1c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "154" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(source_ontology)" ] }, { "cell_type": "code", "execution_count": 5, "id": "f277de7e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...LF_superclasses_overlapLF_properties_overlaplabelxyshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distance
306111-1-1-11111-1...-1-1133161600[CLS]WeatherCondition[SEP]WeatherCondition[SEP]1.000000
5442-1-1-1-1-1-1-1-11-1...-1-11586791620[CLS]SurfaceWindCondition[SEP]Wind[SEP]0.789100
5491-1-1-11-1-1-1-1-1-1...-1-1161466[CLS]Airport[SEP]Aerodrome[SEP]0.508479
8752-1-1-1-1-1-1-1-11-1...-1-1195178814[CLS]PhysicalRunway[SEP]Runway[SEP]0.679397
10121-1-1-1-1-1-1-1-1-1-1...-1-11115691315[CLS]NavigationFix[SEP]SignificantPoint[SEP]0.319003
11271-1-1-1-1-1-1-1-11-1...-1-1112291171719[CLS]VisibilityCondition[SEP]HorizontalVisibil...0.810132
14389-1-1-1-1-1-1-1-11-1...-1-1115664241419[CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV...0.881630
15853-1-1-1-1-1-1-1-11-1...-1-111729812816[CLS]AircraftCapacity[SEP]Capacity[SEP]0.825409
23442-1-1-1-1-1-1-10-1-1...-1-112556792530[CLS]AircraftFlow[SEP]OccupancyTrafficMonitori...0.628079
36940-1-1-1-1-1-1-1-11-1...-1-1140340779[CLS]TimeInterval[SEP]TimeRangeType[SEP]0.760346
4496911-11-11111-1...-1-1149134800[CLS]Aircraft[SEP]Aircraft[SEP]1.000000
4680311-1-1-11111-1...-1-11511381200[CLS]AircraftType[SEP]AircraftType[SEP]1.000000
51377-1-1-1-1-1-1-1-11-1...-1-1156137131013[CLS]AircraftModel[SEP]AircraftMakeModelSeries...0.818485
53695-1-1-1-1-1-1-1-11-1...-1-115862513516[CLS]NavigationAid[SEP]RadioNavigationAid[SEP]0.899368
54340-1-1-1-1-1-1-1-11-1...-1-11593557814[CLS]AircraftEngine[SEP]Engine[SEP]0.898687
58021-1-1-1-1-1-1-1-11-1...-1-1163376101222[CLS]EngineType[SEP]CodeAircraftEngineType[SEP]0.880238
6073511-11-11111-1...-1-1166345700[CLS]Taxiway[SEP]Taxiway[SEP]1.000000
63306-1-1-1-1-1-1-1-1-1-1...-1-116917191013[CLS]AirCarrier[SEP]AircraftOperator[SEP]0.614038
6868411-1-1-11111-1...-1-1175591300[CLS]AirspaceLayer[SEP]AirspaceLayer[SEP]1.000000
70728-1-1-1-1-1-100-1-1...-1-117727322324[CLS]STAR[SEP]StandardInstrumentArrival[SEP]0.178293
75074-1-1-1-1-1-1-1-11-1...-1-118244141516[CLS]OperationalRunway[SEP]RunwayDirection[SEP]0.835488
86354-1-1-1-1-1-1-1-11-1...-1-1194344844[CLS]DeicingPad[SEP]DeicingArea[SEP]0.339917
8700411-11-11111-1...-1-119579800[CLS]Terminal[SEP]Terminal[SEP]1.000000
94605-1-11-1-1-100-1-1...-1-1110336032426[CLS]SID[SEP]StandardInstrumentDeparture[SEP]-0.116515
9734611-11-11111-1...-1-11106356400[CLS]Gate[SEP]Gate[SEP]1.000000
104410-1-1-1-1-1-1-1-1-1-1...-1-11114100789[CLS]AirportSpec[SEP]AerodromeSet[SEP]0.415925
10555911-1-1-11111-1...-1-111153341000[CLS]CrewMember[SEP]CrewMember[SEP]1.000000
10614211-11-11111-1...-1-111162600[CLS]Flight[SEP]Flight[SEP]1.000000
112057-1-1-1-1-1-1-1-11-1...-1-1112242762020[CLS]FlightSpec[SEP]FlightConditionCombination...0.760592
116390-1-1-1-1-1-1-1-11-1...-1-111271851399[CLS]AircraftTrackPoint[SEP]AircraftState[SEP]0.710066
128868-1-1-1-1-1-1-1-11-1...-1-11140768645[CLS]Taxipath[SEP]TaxiRoute[SEP]0.804810
131434-1-1-1-1-1-1-10-1-1...-1-1114358961213[CLS]RelativeFix[SEP]PointReference[SEP]0.449235
\n", "

32 rows × 21 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "3061 1 1 -1 \n", "5442 -1 -1 -1 \n", "5491 -1 -1 -1 \n", "8752 -1 -1 -1 \n", "10121 -1 -1 -1 \n", "11271 -1 -1 -1 \n", "14389 -1 -1 -1 \n", "15853 -1 -1 -1 \n", "23442 -1 -1 -1 \n", "36940 -1 -1 -1 \n", "44969 1 1 -1 \n", "46803 1 1 -1 \n", "51377 -1 -1 -1 \n", "53695 -1 -1 -1 \n", "54340 -1 -1 -1 \n", "58021 -1 -1 -1 \n", "60735 1 1 -1 \n", "63306 -1 -1 -1 \n", "68684 1 1 -1 \n", "70728 -1 -1 -1 \n", "75074 -1 -1 -1 \n", "86354 -1 -1 -1 \n", "87004 1 1 -1 \n", "94605 -1 -1 1 \n", "97346 1 1 -1 \n", "104410 -1 -1 -1 \n", "105559 1 1 -1 \n", "106142 1 1 -1 \n", "112057 -1 -1 -1 \n", "116390 -1 -1 -1 \n", "128868 -1 -1 -1 \n", "131434 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "3061 -1 -1 1 \n", "5442 -1 -1 -1 \n", "5491 1 -1 -1 \n", "8752 -1 -1 -1 \n", "10121 -1 -1 -1 \n", "11271 -1 -1 -1 \n", "14389 -1 -1 -1 \n", "15853 -1 -1 -1 \n", "23442 -1 -1 -1 \n", "36940 -1 -1 -1 \n", "44969 1 -1 1 \n", "46803 -1 -1 1 \n", "51377 -1 -1 -1 \n", "53695 -1 -1 -1 \n", "54340 -1 -1 -1 \n", "58021 -1 -1 -1 \n", "60735 1 -1 1 \n", "63306 -1 -1 -1 \n", "68684 -1 -1 1 \n", "70728 -1 -1 -1 \n", "75074 -1 -1 -1 \n", "86354 -1 -1 -1 \n", "87004 1 -1 1 \n", "94605 -1 -1 -1 \n", "97346 1 -1 1 \n", "104410 -1 -1 -1 \n", "105559 -1 -1 1 \n", "106142 1 -1 1 \n", "112057 -1 -1 -1 \n", "116390 -1 -1 -1 \n", "128868 -1 -1 -1 \n", "131434 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "3061 1 1 \n", "5442 -1 -1 \n", "5491 -1 -1 \n", "8752 -1 -1 \n", "10121 -1 -1 \n", "11271 -1 -1 \n", "14389 -1 -1 \n", "15853 -1 -1 \n", "23442 -1 0 \n", "36940 -1 -1 \n", "44969 1 1 \n", "46803 1 1 \n", "51377 -1 -1 \n", "53695 -1 -1 \n", "54340 -1 -1 \n", "58021 -1 -1 \n", "60735 1 1 \n", "63306 -1 -1 \n", "68684 1 1 \n", "70728 0 0 \n", "75074 -1 -1 \n", "86354 -1 -1 \n", "87004 1 1 \n", "94605 0 0 \n", "97346 1 1 \n", "104410 -1 -1 \n", "105559 1 1 \n", "106142 1 1 \n", "112057 -1 -1 \n", "116390 -1 -1 \n", "128868 -1 -1 \n", "131434 -1 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "3061 1 -1 ... \n", "5442 1 -1 ... \n", "5491 -1 -1 ... \n", "8752 1 -1 ... \n", "10121 -1 -1 ... \n", "11271 1 -1 ... \n", "14389 1 -1 ... \n", "15853 1 -1 ... \n", "23442 -1 -1 ... \n", "36940 1 -1 ... \n", "44969 1 -1 ... \n", "46803 1 -1 ... \n", "51377 1 -1 ... \n", "53695 1 -1 ... \n", "54340 1 -1 ... \n", "58021 1 -1 ... \n", "60735 1 -1 ... \n", "63306 -1 -1 ... \n", "68684 1 -1 ... \n", "70728 -1 -1 ... \n", "75074 1 -1 ... \n", "86354 1 -1 ... \n", "87004 1 -1 ... \n", "94605 -1 -1 ... \n", "97346 1 -1 ... \n", "104410 -1 -1 ... \n", "105559 1 -1 ... \n", "106142 1 -1 ... \n", "112057 1 -1 ... \n", "116390 1 -1 ... \n", "128868 1 -1 ... \n", "131434 -1 -1 ... \n", "\n", " LF_superclasses_overlap LF_properties_overlap label x y \\\n", "3061 -1 -1 1 3 316 \n", "5442 -1 -1 1 5 867 \n", "5491 -1 -1 1 6 1 \n", "8752 -1 -1 1 9 517 \n", "10121 -1 -1 1 11 56 \n", "11271 -1 -1 1 12 291 \n", "14389 -1 -1 1 15 664 \n", "15853 -1 -1 1 17 298 \n", "23442 -1 -1 1 25 567 \n", "36940 -1 -1 1 40 340 \n", "44969 -1 -1 1 49 134 \n", "46803 -1 -1 1 51 138 \n", "51377 -1 -1 1 56 137 \n", "53695 -1 -1 1 58 625 \n", "54340 -1 -1 1 59 355 \n", "58021 -1 -1 1 63 376 \n", "60735 -1 -1 1 66 345 \n", "63306 -1 -1 1 69 171 \n", "68684 -1 -1 1 75 59 \n", "70728 -1 -1 1 77 273 \n", "75074 -1 -1 1 82 44 \n", "86354 -1 -1 1 94 344 \n", "87004 -1 -1 1 95 79 \n", "94605 -1 -1 1 103 360 \n", "97346 -1 -1 1 106 356 \n", "104410 -1 -1 1 114 100 \n", "105559 -1 -1 1 115 334 \n", "106142 -1 -1 1 116 2 \n", "112057 -1 -1 1 122 427 \n", "116390 -1 -1 1 127 185 \n", "128868 -1 -1 1 140 768 \n", "131434 -1 -1 1 143 589 \n", "\n", " shared_word levenshtein_distance hamming_distance \\\n", "3061 16 0 0 \n", "5442 9 16 20 \n", "5491 4 6 6 \n", "8752 8 8 14 \n", "10121 9 13 15 \n", "11271 17 17 19 \n", "14389 24 14 19 \n", "15853 12 8 16 \n", "23442 9 25 30 \n", "36940 7 7 9 \n", "44969 8 0 0 \n", "46803 12 0 0 \n", "51377 13 10 13 \n", "53695 13 5 16 \n", "54340 7 8 14 \n", "58021 10 12 22 \n", "60735 7 0 0 \n", "63306 9 10 13 \n", "68684 13 0 0 \n", "70728 2 23 24 \n", "75074 14 15 16 \n", "86354 8 4 4 \n", "87004 8 0 0 \n", "94605 3 24 26 \n", "97346 4 0 0 \n", "104410 7 8 9 \n", "105559 10 0 0 \n", "106142 6 0 0 \n", "112057 6 20 20 \n", "116390 13 9 9 \n", "128868 6 4 5 \n", "131434 6 12 13 \n", "\n", " text_pair \\\n", "3061 [CLS]WeatherCondition[SEP]WeatherCondition[SEP] \n", "5442 [CLS]SurfaceWindCondition[SEP]Wind[SEP] \n", "5491 [CLS]Airport[SEP]Aerodrome[SEP] \n", "8752 [CLS]PhysicalRunway[SEP]Runway[SEP] \n", "10121 [CLS]NavigationFix[SEP]SignificantPoint[SEP] \n", "11271 [CLS]VisibilityCondition[SEP]HorizontalVisibil... \n", "14389 [CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV... \n", "15853 [CLS]AircraftCapacity[SEP]Capacity[SEP] \n", "23442 [CLS]AircraftFlow[SEP]OccupancyTrafficMonitori... \n", "36940 [CLS]TimeInterval[SEP]TimeRangeType[SEP] \n", "44969 [CLS]Aircraft[SEP]Aircraft[SEP] \n", "46803 [CLS]AircraftType[SEP]AircraftType[SEP] \n", "51377 [CLS]AircraftModel[SEP]AircraftMakeModelSeries... \n", "53695 [CLS]NavigationAid[SEP]RadioNavigationAid[SEP] \n", "54340 [CLS]AircraftEngine[SEP]Engine[SEP] \n", "58021 [CLS]EngineType[SEP]CodeAircraftEngineType[SEP] \n", "60735 [CLS]Taxiway[SEP]Taxiway[SEP] \n", "63306 [CLS]AirCarrier[SEP]AircraftOperator[SEP] \n", "68684 [CLS]AirspaceLayer[SEP]AirspaceLayer[SEP] \n", "70728 [CLS]STAR[SEP]StandardInstrumentArrival[SEP] \n", "75074 [CLS]OperationalRunway[SEP]RunwayDirection[SEP] \n", "86354 [CLS]DeicingPad[SEP]DeicingArea[SEP] \n", "87004 [CLS]Terminal[SEP]Terminal[SEP] \n", "94605 [CLS]SID[SEP]StandardInstrumentDeparture[SEP] \n", "97346 [CLS]Gate[SEP]Gate[SEP] \n", "104410 [CLS]AirportSpec[SEP]AerodromeSet[SEP] \n", "105559 [CLS]CrewMember[SEP]CrewMember[SEP] \n", "106142 [CLS]Flight[SEP]Flight[SEP] \n", "112057 [CLS]FlightSpec[SEP]FlightConditionCombination... \n", "116390 [CLS]AircraftTrackPoint[SEP]AircraftState[SEP] \n", "128868 [CLS]Taxipath[SEP]TaxiRoute[SEP] \n", "131434 [CLS]RelativeFix[SEP]PointReference[SEP] \n", "\n", " class_name_embedding_distance \n", "3061 1.000000 \n", "5442 0.789100 \n", "5491 0.508479 \n", "8752 0.679397 \n", "10121 0.319003 \n", "11271 0.810132 \n", "14389 0.881630 \n", "15853 0.825409 \n", "23442 0.628079 \n", "36940 0.760346 \n", "44969 1.000000 \n", "46803 1.000000 \n", "51377 0.818485 \n", "53695 0.899368 \n", "54340 0.898687 \n", "58021 0.880238 \n", "60735 1.000000 \n", "63306 0.614038 \n", "68684 1.000000 \n", "70728 0.178293 \n", "75074 0.835488 \n", "86354 0.339917 \n", "87004 1.000000 \n", "94605 -0.116515 \n", "97346 1.000000 \n", "104410 0.415925 \n", "105559 1.000000 \n", "106142 1.000000 \n", "112057 0.760592 \n", "116390 0.710066 \n", "128868 0.804810 \n", "131434 0.449235 \n", "\n", "[32 rows x 21 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_dataset_df[all_dataset_df.label==1]" ] }, { "cell_type": "markdown", "id": "a3c7d00f", "metadata": {}, "source": [ "# common words in class name" ] }, { "cell_type": "code", "execution_count": 6, "id": "dff25b09", "metadata": {}, "outputs": [], "source": [ "def calculate_common_words(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", "\n", " s1 = source_class.class_name_words\n", " s2 = target_class.class_name_words\n", " t = list(set(s1) & set(s2))\n", " \n", " return len(t)\n", "\n", "# def calculate_common_words(row):\n", "# x = int(row['x'])\n", "# y = int(row['y']) \n", "# source_class = source_ontology[x]\n", "# target_class = target_ontology[y] \n", "\n", "# s1 = source_class.label_words\n", "# s2 = target_class.label_words\n", "# t = list(set(s1) & set(s2))\n", " \n", "# return len(t)\n", " \n", "all_dataset_df['num_common_words'] = all_dataset_df.apply(calculate_common_words, axis = 1)" ] }, { "cell_type": "code", "execution_count": 7, "id": "188fd93c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...LF_properties_overlaplabelxyshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distancenum_common_words
63-1-1-1-1-1-1-1-11-1...-10063844[CLS]AirportData[SEP]AirportSlot[SEP]0.4196751
72-1-1-1-1-1-1-1-11-1...-1007281313[CLS]AirportData[SEP]AirportGroundService[SEP]0.6405601
83-1-1-1-1-1-1-1-11-1...-10083101421[CLS]AirportData[SEP]CodeStatusAirportType[SEP]0.6330531
86-1-1-1-1-1-1-1-11-1...-10086101422[CLS]AirportData[SEP]CodeAirportWarningType[SEP]0.6569661
153-1-1-1-1-1-1-101-1...-10015383438[CLS]AirportData[SEP]CodeDatalinkCommunication...0.7882751
..................................................................
136096-1-1-1-1-1-1-1-11-1...-1014867611919[CLS]PopupFactorSequence[SEP]RunwayMixSequence...0.5532101
137115-1-1-1-1-1-1-1-11-1...-10149780141326[CLS]IntersectionFix[SEP]TaxiwayIntersectionMa...0.5416621
138706-1-1-1-1-1-1-1-11-1...-10151541333[CLS]NDBfix[SEP]NDB[SEP]NaN1
138707-1-1-1-1-1-1-101-1...-1015154231316[CLS]NDBfix[SEP]CodeNDBUsageType[SEP]0.2235251
139430-1-1-1-1-1-1-1-11-1...-101523507814[CLS]PopupFactor[SEP]EmissionFactor[SEP]0.7354631
\n", "

3262 rows × 22 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "63 -1 -1 -1 \n", "72 -1 -1 -1 \n", "83 -1 -1 -1 \n", "86 -1 -1 -1 \n", "153 -1 -1 -1 \n", "... ... ... ... \n", "136096 -1 -1 -1 \n", "137115 -1 -1 -1 \n", "138706 -1 -1 -1 \n", "138707 -1 -1 -1 \n", "139430 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "63 -1 -1 -1 \n", "72 -1 -1 -1 \n", "83 -1 -1 -1 \n", "86 -1 -1 -1 \n", "153 -1 -1 -1 \n", "... ... ... ... \n", "136096 -1 -1 -1 \n", "137115 -1 -1 -1 \n", "138706 -1 -1 -1 \n", "138707 -1 -1 -1 \n", "139430 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "63 -1 -1 \n", "72 -1 -1 \n", "83 -1 -1 \n", "86 -1 -1 \n", "153 -1 0 \n", "... ... ... \n", "136096 -1 -1 \n", "137115 -1 -1 \n", "138706 -1 -1 \n", "138707 -1 0 \n", "139430 -1 -1 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "63 1 -1 ... \n", "72 1 -1 ... \n", "83 1 -1 ... \n", "86 1 -1 ... \n", "153 1 -1 ... \n", "... ... ... ... \n", "136096 1 -1 ... \n", "137115 1 -1 ... \n", "138706 1 -1 ... \n", "138707 1 -1 ... \n", "139430 1 -1 ... \n", "\n", " LF_properties_overlap label x y shared_word \\\n", "63 -1 0 0 63 8 \n", "72 -1 0 0 72 8 \n", "83 -1 0 0 83 10 \n", "86 -1 0 0 86 10 \n", "153 -1 0 0 153 8 \n", "... ... ... ... ... ... \n", "136096 -1 0 148 676 11 \n", "137115 -1 0 149 780 14 \n", "138706 -1 0 151 541 3 \n", "138707 -1 0 151 542 3 \n", "139430 -1 0 152 350 7 \n", "\n", " levenshtein_distance hamming_distance \\\n", "63 4 4 \n", "72 13 13 \n", "83 14 21 \n", "86 14 22 \n", "153 34 38 \n", "... ... ... \n", "136096 9 19 \n", "137115 13 26 \n", "138706 3 3 \n", "138707 13 16 \n", "139430 8 14 \n", "\n", " text_pair \\\n", "63 [CLS]AirportData[SEP]AirportSlot[SEP] \n", "72 [CLS]AirportData[SEP]AirportGroundService[SEP] \n", "83 [CLS]AirportData[SEP]CodeStatusAirportType[SEP] \n", "86 [CLS]AirportData[SEP]CodeAirportWarningType[SEP] \n", "153 [CLS]AirportData[SEP]CodeDatalinkCommunication... \n", "... ... \n", "136096 [CLS]PopupFactorSequence[SEP]RunwayMixSequence... \n", "137115 [CLS]IntersectionFix[SEP]TaxiwayIntersectionMa... \n", "138706 [CLS]NDBfix[SEP]NDB[SEP] \n", "138707 [CLS]NDBfix[SEP]CodeNDBUsageType[SEP] \n", "139430 [CLS]PopupFactor[SEP]EmissionFactor[SEP] \n", "\n", " class_name_embedding_distance num_common_words \n", "63 0.419675 1 \n", "72 0.640560 1 \n", "83 0.633053 1 \n", "86 0.656966 1 \n", "153 0.788275 1 \n", "... ... ... \n", "136096 0.553210 1 \n", "137115 0.541662 1 \n", "138706 NaN 1 \n", "138707 0.223525 1 \n", "139430 0.735463 1 \n", "\n", "[3262 rows x 22 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_dataset_df.query('num_common_words >= 1')" ] }, { "cell_type": "markdown", "id": "c128e601", "metadata": {}, "source": [ "# embedding distance of selected features" ] }, { "cell_type": "code", "execution_count": 8, "id": "6d5ffb50", "metadata": {}, "outputs": [], "source": [ "from scipy import spatial\n", "import numpy as np\n", "\n", "def calculate_cosine_similarity(s_embedding, t_embedding): \n", " if s_embedding is None or t_embedding is None:\n", " return None \n", " \n", " if len(s_embedding) == 0 or len(t_embedding) == 0:\n", " return None\n", " \n", " if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:\n", " return None \n", "\n", " similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)\n", " return similarity\n" ] }, { "cell_type": "code", "execution_count": 9, "id": "4d6e4ab9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "class_long_name\n", "compute feature embedding\n", "compute embedding distance\n", "label_long\n", "compute feature embedding\n", "compute embedding distance\n", "comment\n", "compute feature embedding\n", "compute embedding distance\n" ] } ], "source": [ "# calculate the embedding for the selected feature\n", "from sentence_transformers import SentenceTransformer\n", "distilbert = SentenceTransformer('distilbert-base-nli-mean-tokens')\n", "\n", "features = ['class_long_name', 'label_long', 'comment']\n", "\n", "def computer_embedding(ontology_list, feature, embedding_feature):\n", " for idx in range(len(ontology_list)):\n", " obj = ontology_list[idx]\n", " \n", " attr = getattr(obj, feature)\n", " if attr != None and attr != \"\":\n", " em = distilbert.encode(getattr(obj, feature)) \n", " setattr(obj, embedding_feature, em)\n", "\n", "\n", "def calculate_embedding_distance(row, selected_feature):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", "\n", " if hasattr(source_class, selected_feature) and hasattr(target_class, selected_feature):\n", " em1 = getattr(source_class, selected_feature)\n", " em2 = getattr(target_class, selected_feature)\n", " \n", " return calculate_cosine_similarity(em1, em2) \n", " else:\n", " return None\n", "\n", "for feature in features: \n", " print(feature)\n", "\n", " embedding_feature = feature + '_embedding' \n", " computer_embedding(target_ontology, feature, embedding_feature) \n", " computer_embedding(source_ontology, feature, embedding_feature) \n", " print(\"compute feature embedding\")\n", "\n", " embedding_distance = feature + '_distance' \n", " all_dataset_df[embedding_distance] = all_dataset_df.apply(calculate_embedding_distance, selected_feature=embedding_feature, axis = 1)\n", " print(\"compute embedding distance\")\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "710d66cf", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...yshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distancenum_common_wordsclass_long_name_distancelabel_long_distancecomment_distance
0-1-1-1-1-1-1-1-1-1-1...091214[CLS]AirportData[SEP]ACDMIrregularity[SEP]0.35595300.310721None0.305137
1-1-1-1-1-1-1-1-1-1-1...1488[CLS]AirportData[SEP]Aerodrome[SEP]0.23730700.686413None0.647584
2-1-1-1-1-1-1-10-1-1...231011[CLS]AirportData[SEP]Flight[SEP]0.45493600.697756None0.599904
3-1-1-1-1-1-1-10-1-1...3112024[CLS]AirportData[SEP]CodeACDMIrregularityType[...0.51522000.375069None0.387794
4-1-1-1-1-1-1-1-1-1-1...4101315[CLS]AirportData[SEP]ADSBGroundStation[SEP]0.48107900.421668None0.553725
..................................................................
910-1-1-1-1-1-1-10-1-1...91092630[CLS]AirportData[SEP]CodeUomVolumeConcentratio...0.56587600.418236None0.298020
911-1-1-1-1-1-1-10-1-1...91121617[CLS]AirportData[SEP]CodeUomVolumeType[SEP]0.52907300.435312None0.417220
912-1-1-1-1-1-1-10-1-1...91241316[CLS]AirportData[SEP]EntityCollection[SEP]0.50875700.478009None0.349885
913-1-1-1-1-1-1-1-1-1-1...91392327[CLS]AirportData[SEP]LongitudinalSeparationMin...0.44062600.376916None0.516962
914-1-1-1-1-1-1-1-1-1-1...91482123[CLS]AirportData[SEP]SurveillanceGroundStation...0.56246300.505302None0.615024
\n", "

915 rows × 25 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", ".. ... ... ... \n", "910 -1 -1 -1 \n", "911 -1 -1 -1 \n", "912 -1 -1 -1 \n", "913 -1 -1 -1 \n", "914 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", ".. ... ... ... \n", "910 -1 -1 -1 \n", "911 -1 -1 -1 \n", "912 -1 -1 -1 \n", "913 -1 -1 -1 \n", "914 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "0 -1 -1 \n", "1 -1 -1 \n", "2 -1 0 \n", "3 -1 0 \n", "4 -1 -1 \n", ".. ... ... \n", "910 -1 0 \n", "911 -1 0 \n", "912 -1 0 \n", "913 -1 -1 \n", "914 -1 -1 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... y shared_word \\\n", "0 -1 -1 ... 0 9 \n", "1 -1 -1 ... 1 4 \n", "2 -1 -1 ... 2 3 \n", "3 -1 -1 ... 3 11 \n", "4 -1 -1 ... 4 10 \n", ".. ... ... ... ... ... \n", "910 -1 -1 ... 910 9 \n", "911 -1 -1 ... 911 2 \n", "912 -1 -1 ... 912 4 \n", "913 -1 -1 ... 913 9 \n", "914 -1 -1 ... 914 8 \n", "\n", " levenshtein_distance hamming_distance \\\n", "0 12 14 \n", "1 8 8 \n", "2 10 11 \n", "3 20 24 \n", "4 13 15 \n", ".. ... ... \n", "910 26 30 \n", "911 16 17 \n", "912 13 16 \n", "913 23 27 \n", "914 21 23 \n", "\n", " text_pair \\\n", "0 [CLS]AirportData[SEP]ACDMIrregularity[SEP] \n", "1 [CLS]AirportData[SEP]Aerodrome[SEP] \n", "2 [CLS]AirportData[SEP]Flight[SEP] \n", "3 [CLS]AirportData[SEP]CodeACDMIrregularityType[... \n", "4 [CLS]AirportData[SEP]ADSBGroundStation[SEP] \n", ".. ... \n", "910 [CLS]AirportData[SEP]CodeUomVolumeConcentratio... \n", "911 [CLS]AirportData[SEP]CodeUomVolumeType[SEP] \n", "912 [CLS]AirportData[SEP]EntityCollection[SEP] \n", "913 [CLS]AirportData[SEP]LongitudinalSeparationMin... \n", "914 [CLS]AirportData[SEP]SurveillanceGroundStation... \n", "\n", " class_name_embedding_distance num_common_words \\\n", "0 0.355953 0 \n", "1 0.237307 0 \n", "2 0.454936 0 \n", "3 0.515220 0 \n", "4 0.481079 0 \n", ".. ... ... \n", "910 0.565876 0 \n", "911 0.529073 0 \n", "912 0.508757 0 \n", "913 0.440626 0 \n", "914 0.562463 0 \n", "\n", " class_long_name_distance label_long_distance comment_distance \n", "0 0.310721 None 0.305137 \n", "1 0.686413 None 0.647584 \n", "2 0.697756 None 0.599904 \n", "3 0.375069 None 0.387794 \n", "4 0.421668 None 0.553725 \n", ".. ... ... ... \n", "910 0.418236 None 0.298020 \n", "911 0.435312 None 0.417220 \n", "912 0.478009 None 0.349885 \n", "913 0.376916 None 0.516962 \n", "914 0.505302 None 0.615024 \n", "\n", "[915 rows x 25 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_dataset_df.query('x==0')" ] }, { "cell_type": "markdown", "id": "530cf757", "metadata": {}, "source": [ "# blocking based on change point detection over ranked distance" ] }, { "cell_type": "code", "execution_count": 11, "id": "4e758096", "metadata": {}, "outputs": [], "source": [ "import changefinder\n", "def findChangePoints(ts, r, order, smooth):\n", " '''\n", " r: Discounting rate\n", " order: AR model order\n", " smooth: smoothing window size T\n", " '''\n", " cf = changefinder.ChangeFinder(r=r, order=order, smooth=smooth)\n", " ts_score = [cf.update(p) for p in ts]\n", " return(ts_score)" ] }, { "cell_type": "code", "execution_count": 12, "id": "06bb881a", "metadata": {}, "outputs": [], "source": [ "def print_true_match(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " print(source_class.class_name, ' => ', target_class.class_name, row['x'], row['y'])\n", " \n", "def print_included_match(row, f):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " print(source_class.class_name, ' => ', target_class.class_name, row[f], row['x'], row['y'])\n", " \n", "def blocking_per_class(xdf, ranked_feature, check_point_detection, with_delta, debug_flag):\n", " # for ranked integer feature, the last increase\n", " if ranked_feature in ['num_common_words']:\n", " return len(xdf[xdf[ranked_feature] > 0])\n", "\n", " distances = xdf[ranked_feature].values\n", " \n", " num = len(distances) \n", " points = []\n", " \n", " if with_delta == True:\n", " for i in range (num-1):\n", " cur_delta = abs(distances[i+1] - distances[i])\n", " points.append(cur_delta) \n", " else:\n", " points = distances \n", " \n", " top_x = num\n", " scores = []\n", " \n", " if check_point_detection == \"rbf\": \n", " model = \"rbf\" \n", " algo = rpt.Dynp(model=model).fit(np.array(points))\n", " scores = algo.predict(n_bkps=1) \n", " top_x = scores[0]-1\n", " elif check_point_detection == \"l2\": \n", " model = \"l2\"\n", " algo = rpt.Dynp(model=model).fit(np.array(points))\n", " scores = algo.predict(n_bkps=1) \n", " top_x = scores[0]-1\n", " elif check_point_detection == \"l1\": \n", " model = \"l1\"\n", " algo = rpt.Dynp(model=model).fit(np.array(points))\n", " scores = algo.predict(n_bkps=1) \n", " top_x = scores[0]-1\n", " elif check_point_detection == \"online1\": \n", " scores = findChangePoints(points, r = 0.01, order = 3, smooth = 5) \n", " top_x = pd.Series(scores).nlargest(3).index[2]\n", " elif check_point_detection == \"online2\": \n", " cf = ChangeFinder()\n", " \n", " try:\n", " scores = [cf.update(p) for p in points] \n", " for idx in range(len(scores)):\n", " if scores[idx] < 2:\n", " top_x = idx+1\n", " break \n", " except:\n", " top_x = 0\n", " \n", " if debug_flag == True:\n", " print(points)\n", " print(scores)\n", " print(top_x)\n", " \n", " xdf[:top_x].apply(print_included_match, f=ranked_feature, axis = 1)\n", "\n", " plt.figure(figsize=(16,4))\n", " plt.plot(distances, color='blue')\n", " \n", " plt.figure(figsize=(16,4))\n", " plt.plot(points)\n", " \n", " plt.figure(figsize=(16,4))\n", " plt.plot(scores, color='red') \n", " \n", " plt.show() \n", " \n", " return top_x\n", " \n", "\n", "def apply_blocking(df, column, algm, max_k):\n", " df[column+'_blocked'] = 1\n", " \n", " for x in range(len(source_ontology)):\n", " x_df = df.query('x==' + str(x) + ' &' + column + '.notnull()')[[column, 'x', 'y', column+'_blocked']]\n", " \n", " if len(x_df) == 0:\n", " print(x, '0, ignored') \n", " continue\n", " \n", " x_df = x_df.sort_values(column, ascending=False) \n", " top_x = blocking_per_class(x_df.head(max_k), column, algm, True, False)\n", " if top_x == 0:\n", " print(x, '0, ignored')\n", " continue\n", " \n", " # update the block_flag\n", " #print(x, top_x)\n", " df.loc[x_df[:top_x].index, column+'_blocked'] = 0\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "6ffebc5f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "50\n", "\n", "class_long_name_distance\n", "\n", "num_common_words\n", "8 0, ignored\n", "30 0, ignored\n", "33 0, ignored\n", "34 0, ignored\n", "35 0, ignored\n", "36 0, ignored\n", "37 0, ignored\n", "39 0, ignored\n", "41 0, ignored\n", "45 0, ignored\n", "46 0, ignored\n", "73 0, ignored\n", "74 0, ignored\n", "76 0, ignored\n", "77 0, ignored\n", "83 0, ignored\n", "92 0, ignored\n", "101 0, ignored\n", "103 0, ignored\n", "108 0, ignored\n", "110 0, ignored\n", "111 0, ignored\n", "112 0, ignored\n", "113 0, ignored\n", "118 0, ignored\n", "124 0, ignored\n", "139 0, ignored\n", "142 0, ignored\n", "150 0, ignored\n", "153 0, ignored\n", "\n", "label_long_distance\n", "0 0, ignored\n", "1 0, ignored\n", "2 0, ignored\n", "3 0, ignored\n", "4 0, ignored\n", "5 0, ignored\n", "6 0, ignored\n", "7 0, ignored\n", "8 0, ignored\n", "9 0, ignored\n", "10 0, ignored\n", "11 0, ignored\n", "12 0, ignored\n", "13 0, ignored\n", "14 0, ignored\n", "15 0, ignored\n", "16 0, ignored\n", "17 0, ignored\n", "18 0, ignored\n", "19 0, ignored\n", "20 0, ignored\n", "21 0, ignored\n", "22 0, ignored\n", "23 0, ignored\n", "24 0, ignored\n", "25 0, ignored\n", "26 0, ignored\n", "27 0, ignored\n", "28 0, ignored\n", "29 0, ignored\n", "30 0, ignored\n", "31 0, ignored\n", "32 0, ignored\n", "33 0, ignored\n", "34 0, ignored\n", "35 0, ignored\n", "36 0, ignored\n", "37 0, ignored\n", "38 0, ignored\n", "39 0, ignored\n", "40 0, ignored\n", "41 0, ignored\n", "42 0, ignored\n", "43 0, ignored\n", "44 0, ignored\n", "45 0, ignored\n", "46 0, ignored\n", "47 0, ignored\n", "48 0, ignored\n", "49 0, ignored\n", "50 0, ignored\n", "51 0, ignored\n", "52 0, ignored\n", "53 0, ignored\n", "54 0, ignored\n", "55 0, ignored\n", "56 0, ignored\n", "57 0, ignored\n", "58 0, ignored\n", "59 0, ignored\n", "60 0, ignored\n", "61 0, ignored\n", "62 0, ignored\n", "63 0, ignored\n", "64 0, ignored\n", "65 0, ignored\n", "66 0, ignored\n", "67 0, ignored\n", "68 0, ignored\n", "69 0, ignored\n", "70 0, ignored\n", "71 0, ignored\n", "72 0, ignored\n", "73 0, ignored\n", "74 0, ignored\n", "75 0, ignored\n", "76 0, ignored\n", "77 0, ignored\n", "78 0, ignored\n", "79 0, ignored\n", "80 0, ignored\n", "81 0, ignored\n", "82 0, ignored\n", "83 0, ignored\n", "84 0, ignored\n", "85 0, ignored\n", "86 0, ignored\n", "87 0, ignored\n", "88 0, ignored\n", "89 0, ignored\n", "90 0, ignored\n", "91 0, ignored\n", "92 0, ignored\n", "93 0, ignored\n", "94 0, ignored\n", "95 0, ignored\n", "96 0, ignored\n", "97 0, ignored\n", "98 0, ignored\n", "99 0, ignored\n", "100 0, ignored\n", "101 0, ignored\n", "102 0, ignored\n", "103 0, ignored\n", "104 0, ignored\n", "105 0, ignored\n", "106 0, ignored\n", "107 0, ignored\n", "108 0, ignored\n", "109 0, ignored\n", "110 0, ignored\n", "111 0, ignored\n", "112 0, ignored\n", "113 0, ignored\n", "114 0, ignored\n", "115 0, ignored\n", "116 0, ignored\n", "117 0, ignored\n", "118 0, ignored\n", "119 0, ignored\n", "120 0, ignored\n", "121 0, ignored\n", "122 0, ignored\n", "123 0, ignored\n", "124 0, ignored\n", "125 0, ignored\n", "126 0, ignored\n", "127 0, ignored\n", "128 0, ignored\n", "129 0, ignored\n", "130 0, ignored\n", "131 0, ignored\n", "132 0, ignored\n", "133 0, ignored\n", "134 0, ignored\n", "135 0, ignored\n", "136 0, ignored\n", "137 0, ignored\n", "138 0, ignored\n", "139 0, ignored\n", "140 0, ignored\n", "141 0, ignored\n", "142 0, ignored\n", "143 0, ignored\n", "144 0, ignored\n", "145 0, ignored\n", "146 0, ignored\n", "147 0, ignored\n", "148 0, ignored\n", "149 0, ignored\n", "150 0, ignored\n", "151 0, ignored\n", "152 0, ignored\n", "153 0, ignored\n", "\n", "comment_distance\n" ] } ], "source": [ "columns = ['class_long_name_distance', 'num_common_words', 'label_long_distance', 'comment_distance' ]\n", "\n", "MAX_NUM_CANDIDATES = 50\n", "target_size = len(target_ontology)\n", "max_k = target_size\n", "if target_size > MAX_NUM_CANDIDATES:\n", " max_k = MAX_NUM_CANDIDATES\n", "print(max_k)\n", "\n", "changing_point_detection_algorithm = 'online1' # 'rdf', 'l1', 'l2', 'online1', 'online2'\n", "print_matches = True\n", "\n", "for blocked_column in columns:\n", " print(\"\\r\\n\" + blocked_column)\n", " blocked_key = blocked_column+'_blocked'\n", " apply_blocking(all_dataset_df, blocked_column, changing_point_detection_algorithm, max_k)\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "492e8068", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\r\n", "class_long_name_distance\n", "WeatherCondition => WeatherCondition 1.0 3 316\n", "SurfaceWindCondition => Wind 0.8790944218635559 5 867\n", "Airport => Aerodrome 0.840453565120697 6 1\n", "PhysicalRunway => Runway 0.8516059517860413 9 517\n", "RunwayVisibleRangeMeasurement => RunwayVisualRange 0.9156750440597534 15 664\n", "TimeInterval => TimeRangeType 0.8980520367622375 40 340\n", "Aircraft => Aircraft 1.0 49 134\n", "AircraftType => AircraftType 1.0 51 138\n", "AircraftModel => AircraftMakeModelSeries 0.8626878261566162 56 137\n", "NavigationAid => RadioNavigationAid 0.7773143649101257 58 625\n", "EngineType => CodeAircraftEngineType 0.8093550205230713 63 376\n", "Taxiway => Taxiway 1.0 66 345\n", "AirCarrier => AircraftOperator 0.8524434566497803 69 171\n", "AirspaceLayer => AirspaceLayer 1.0 75 59\n", "OperationalRunway => RunwayDirection 0.8838163614273071 82 44\n", "DeicingPad => DeicingArea 0.9137077927589417 94 344\n", "Terminal => Terminal 1.0 95 79\n", "Gate => Gate 1.0 106 356\n", "AirportSpec => AerodromeSet 0.820888102054596 114 100\n", "CrewMember => CrewMember 1.0 115 334\n", "Flight => Flight 1.0 116 2\n", "FlightSpec => FlightConditionCombination 0.8406736254692078 122 427\n", "Taxipath => TaxiRoute 0.9633474946022034 140 768\n", "RelativeFix => PointReference 0.7955299615859985 143 589\n", "2968 out of 140910 2.106308991554893\n", "24 out of 32 75.0\n", "\r\n", "num_common_words\n", "WeatherCondition => WeatherCondition 2 3 316\n", "SurfaceWindCondition => Wind 1 5 867\n", "PhysicalRunway => Runway 1 9 517\n", "VisibilityCondition => HorizontalVisibility 1 12 291\n", "RunwayVisibleRangeMeasurement => RunwayVisualRange 2 15 664\n", "AircraftCapacity => Capacity 1 17 298\n", "TimeInterval => TimeRangeType 1 40 340\n", "Aircraft => Aircraft 1 49 134\n", "AircraftType => AircraftType 2 51 138\n", "AircraftModel => AircraftMakeModelSeries 2 56 137\n", "NavigationAid => RadioNavigationAid 1 58 625\n", "AircraftEngine => Engine 1 59 355\n", "EngineType => CodeAircraftEngineType 2 63 376\n", "Taxiway => Taxiway 1 66 345\n", "AirspaceLayer => AirspaceLayer 2 75 59\n", "OperationalRunway => RunwayDirection 1 82 44\n", "DeicingPad => DeicingArea 1 94 344\n", "Terminal => Terminal 1 95 79\n", "Gate => Gate 1 106 356\n", "CrewMember => CrewMember 2 115 334\n", "Flight => Flight 1 116 2\n", "FlightSpec => FlightConditionCombination 1 122 427\n", "AircraftTrackPoint => AircraftState 1 127 185\n", "Taxipath => TaxiRoute 1 140 768\n", "2382 out of 140910 1.6904407068341494\n", "24 out of 32 75.0\n", "\r\n", "label_long_distance\n", "0 out of 140910 0.0\n", "0 out of 32 0.0\n", "\r\n", "comment_distance\n", "WeatherCondition => WeatherCondition 0.7918438911437988 3 316\n", "PhysicalRunway => Runway 0.7813198566436768 9 517\n", "VisibilityCondition => HorizontalVisibility 0.8003379702568054 12 291\n", "RunwayVisibleRangeMeasurement => RunwayVisualRange 0.9089569449424744 15 664\n", "AircraftCapacity => Capacity 0.8342739939689636 17 298\n", "AircraftFlow => OccupancyTrafficMonitoringValue 0.8646406531333923 25 567\n", "TimeInterval => TimeRangeType 0.7229282855987549 40 340\n", "AircraftModel => AircraftMakeModelSeries 0.8741695284843445 56 137\n", "NavigationAid => RadioNavigationAid 0.7332932353019714 58 625\n", "EngineType => CodeAircraftEngineType 0.8456313014030457 63 376\n", "AirCarrier => AircraftOperator 0.9076859354972839 69 171\n", "AirspaceLayer => AirspaceLayer 0.7635785937309265 75 59\n", "OperationalRunway => RunwayDirection 0.7357498407363892 82 44\n", "Terminal => Terminal 0.849309504032135 95 79\n", "SID => StandardInstrumentDeparture 0.7947181463241577 103 360\n", "Gate => Gate 0.7989940047264099 106 356\n", "AirportSpec => AerodromeSet 0.8796765804290771 114 100\n", "CrewMember => CrewMember 0.8921281695365906 115 334\n", "Taxipath => TaxiRoute 0.865944504737854 140 768\n", "2978 out of 140910 2.113405719963097\n", "19 out of 32 59.375\n", "\r\n", "\n", "7151 out of 140910 5.07487048470655\n", "30 out of 32 93.75\n", "\r\n", "=======list of all true matches========\n", "WeatherCondition => WeatherCondition 3 316\n", "SurfaceWindCondition => Wind 5 867\n", "Airport => Aerodrome 6 1\n", "PhysicalRunway => Runway 9 517\n", "NavigationFix => SignificantPoint 11 56\n", "VisibilityCondition => HorizontalVisibility 12 291\n", "RunwayVisibleRangeMeasurement => RunwayVisualRange 15 664\n", "AircraftCapacity => Capacity 17 298\n", "AircraftFlow => OccupancyTrafficMonitoringValue 25 567\n", "TimeInterval => TimeRangeType 40 340\n", "Aircraft => Aircraft 49 134\n", "AircraftType => AircraftType 51 138\n", "AircraftModel => AircraftMakeModelSeries 56 137\n", "NavigationAid => RadioNavigationAid 58 625\n", "AircraftEngine => Engine 59 355\n", "EngineType => CodeAircraftEngineType 63 376\n", "Taxiway => Taxiway 66 345\n", "AirCarrier => AircraftOperator 69 171\n", "AirspaceLayer => AirspaceLayer 75 59\n", "STAR => StandardInstrumentArrival 77 273\n", "OperationalRunway => RunwayDirection 82 44\n", "DeicingPad => DeicingArea 94 344\n", "Terminal => Terminal 95 79\n", "SID => StandardInstrumentDeparture 103 360\n", "Gate => Gate 106 356\n", "AirportSpec => AerodromeSet 114 100\n", "CrewMember => CrewMember 115 334\n", "Flight => Flight 116 2\n", "FlightSpec => FlightConditionCombination 122 427\n", "AircraftTrackPoint => AircraftState 127 185\n", "Taxipath => TaxiRoute 140 768\n", "RelativeFix => PointReference 143 589\n", "\r\n", "***** MISSING ****\n", "NavigationFix => SignificantPoint 11 56\n", "STAR => StandardInstrumentArrival 77 273\n" ] } ], "source": [ "# show the matches included by different blocking indexes\n", "combination_statement = \"\";\n", "missing_statement = \"\"\n", "for blocked_column in columns:\n", " print(\"\\r\\n\" + blocked_column)\n", " blocked_key = blocked_column+'_blocked'\n", " \n", " all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis = 1)\n", " \n", " total_num = len(all_dataset_df)\n", " num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))\n", "\n", " print(num_after_blocking, ' out of ', total_num, num_after_blocking*100.0/total_num)\n", "\n", " total_true_matches = len(all_dataset_df[all_dataset_df['label']==1])\n", " captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))\n", "\n", " print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches*100.0/total_true_matches)\n", " \n", " if combination_statement == \"\":\n", " combination_statement = \" (\" + blocked_key + '==0)'\n", " missing_statement = \" (\" + blocked_key + '==1)' \n", " else:\n", " combination_statement = combination_statement + \" | (\" + blocked_key + '==0)'\n", " missing_statement = missing_statement + \" & (\" + blocked_key + '==1)'\n", "\n", "all_dataset_df['selected_after_blocking'] = 0\n", " \n", "# calculate the union of all selected items by the committee of blockers\n", "print(\"\\r\\n\")\n", "#print(combination_statement)\n", "selected_items = all_dataset_df.query(combination_statement)\n", "all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1\n", "total_selected_items = len(selected_items)\n", "\n", "\n", "query = '(label==1) & (' + combination_statement + ')'\n", "#print(query)\n", "total_selected_matches = len(all_dataset_df.query(query))\n", "\n", "query = 'label==1' \n", "#print(query)\n", "total_matches = len(all_dataset_df.query(query))\n", "\n", "total_num = len(all_dataset_df)\n", "\n", "\n", "print(total_selected_items, ' out of ', total_num, total_selected_items*100.0/total_num)\n", "print(total_selected_matches, ' out of ', total_matches, total_selected_matches*100.0/total_matches)\n", "\n", "print(\"\\r\\n=======list of all true matches========\")\n", "true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis = 1)\n", "\n", "print(\"\\r\\n***** MISSING ****\")\n", "\n", "query = '(label==1) & (' + missing_statement + ')'\n", "#print(query)\n", "true_matches = all_dataset_df.query(query).apply(print_true_match, axis = 1)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e19726d3", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 5 }