{
"cells": [
{
"cell_type": "markdown",
"id": "f7e80ce6",
"metadata": {},
"source": [
"# apply the blocking algorithm"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "97789d42",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import pickle\n",
"\n",
"from change_finder import ChangeFinder, SDAR\n",
"import matplotlib.pyplot as plt\n",
"import ruptures as rpt # our package\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "974b97c6",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/huggingface_hub/snapshot_download.py:6: FutureWarning: snapshot_download.py has been made private and will no longer be available from version 0.11. Please use `from huggingface_hub import snapshot_download` to import the only public function in this module. Other members of the file may be changed without a deprecation notice.\n",
" warnings.warn(\n"
]
}
],
"source": [
"#dataset_name = 'nasa'\n",
"#dataset_name = 'conference'\n",
"#dataset_name = 'ai4eu'\n",
"\n",
"all_dataset_df = pd.read_csv(dataset_name + '.csv')\n",
"\n",
"import pickle\n",
"source_ontology = pickle.load( open(dataset_name + \"_source_ontology.pk\", \"rb\" ) )\n",
"target_ontology = pickle.load( open(dataset_name + \"_target_ontology.pk\", \"rb\" ) )"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a959c5b4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"915"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(target_ontology)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "28f9bd1c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"154"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(source_ontology)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "f277de7e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>LF_class_name_equal</th>\n",
" <th>LF_class_name_stemmed_equal</th>\n",
" <th>LF_acronyms</th>\n",
" <th>LF_class_name_synonyms</th>\n",
" <th>LF_label_equal</th>\n",
" <th>LF_root_nouns_equal</th>\n",
" <th>LF_class_name_spacy_distance</th>\n",
" <th>LF_class_name_distance</th>\n",
" <th>LF_name_segment_overlap</th>\n",
" <th>LF_label_words_overlap</th>\n",
" <th>...</th>\n",
" <th>LF_superclasses_overlap</th>\n",
" <th>LF_properties_overlap</th>\n",
" <th>label</th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>shared_word</th>\n",
" <th>levenshtein_distance</th>\n",
" <th>hamming_distance</th>\n",
" <th>text_pair</th>\n",
" <th>class_name_embedding_distance</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3061</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>3</td>\n",
" <td>316</td>\n",
" <td>16</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]WeatherCondition[SEP]WeatherCondition[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5442</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>5</td>\n",
" <td>867</td>\n",
" <td>9</td>\n",
" <td>16</td>\n",
" <td>20</td>\n",
" <td>[CLS]SurfaceWindCondition[SEP]Wind[SEP]</td>\n",
" <td>0.789100</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5491</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>6</td>\n",
" <td>1</td>\n",
" <td>4</td>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>[CLS]Airport[SEP]Aerodrome[SEP]</td>\n",
" <td>0.508479</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8752</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>9</td>\n",
" <td>517</td>\n",
" <td>8</td>\n",
" <td>8</td>\n",
" <td>14</td>\n",
" <td>[CLS]PhysicalRunway[SEP]Runway[SEP]</td>\n",
" <td>0.679397</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10121</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>11</td>\n",
" <td>56</td>\n",
" <td>9</td>\n",
" <td>13</td>\n",
" <td>15</td>\n",
" <td>[CLS]NavigationFix[SEP]SignificantPoint[SEP]</td>\n",
" <td>0.319003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11271</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" <td>291</td>\n",
" <td>17</td>\n",
" <td>17</td>\n",
" <td>19</td>\n",
" <td>[CLS]VisibilityCondition[SEP]HorizontalVisibil...</td>\n",
" <td>0.810132</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14389</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>15</td>\n",
" <td>664</td>\n",
" <td>24</td>\n",
" <td>14</td>\n",
" <td>19</td>\n",
" <td>[CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV...</td>\n",
" <td>0.881630</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15853</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>17</td>\n",
" <td>298</td>\n",
" <td>12</td>\n",
" <td>8</td>\n",
" <td>16</td>\n",
" <td>[CLS]AircraftCapacity[SEP]Capacity[SEP]</td>\n",
" <td>0.825409</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23442</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>25</td>\n",
" <td>567</td>\n",
" <td>9</td>\n",
" <td>25</td>\n",
" <td>30</td>\n",
" <td>[CLS]AircraftFlow[SEP]OccupancyTrafficMonitori...</td>\n",
" <td>0.628079</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36940</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>40</td>\n",
" <td>340</td>\n",
" <td>7</td>\n",
" <td>7</td>\n",
" <td>9</td>\n",
" <td>[CLS]TimeInterval[SEP]TimeRangeType[SEP]</td>\n",
" <td>0.760346</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44969</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>49</td>\n",
" <td>134</td>\n",
" <td>8</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]Aircraft[SEP]Aircraft[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46803</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>51</td>\n",
" <td>138</td>\n",
" <td>12</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]AircraftType[SEP]AircraftType[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>51377</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>56</td>\n",
" <td>137</td>\n",
" <td>13</td>\n",
" <td>10</td>\n",
" <td>13</td>\n",
" <td>[CLS]AircraftModel[SEP]AircraftMakeModelSeries...</td>\n",
" <td>0.818485</td>\n",
" </tr>\n",
" <tr>\n",
" <th>53695</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>58</td>\n",
" <td>625</td>\n",
" <td>13</td>\n",
" <td>5</td>\n",
" <td>16</td>\n",
" <td>[CLS]NavigationAid[SEP]RadioNavigationAid[SEP]</td>\n",
" <td>0.899368</td>\n",
" </tr>\n",
" <tr>\n",
" <th>54340</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>59</td>\n",
" <td>355</td>\n",
" <td>7</td>\n",
" <td>8</td>\n",
" <td>14</td>\n",
" <td>[CLS]AircraftEngine[SEP]Engine[SEP]</td>\n",
" <td>0.898687</td>\n",
" </tr>\n",
" <tr>\n",
" <th>58021</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>63</td>\n",
" <td>376</td>\n",
" <td>10</td>\n",
" <td>12</td>\n",
" <td>22</td>\n",
" <td>[CLS]EngineType[SEP]CodeAircraftEngineType[SEP]</td>\n",
" <td>0.880238</td>\n",
" </tr>\n",
" <tr>\n",
" <th>60735</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>66</td>\n",
" <td>345</td>\n",
" <td>7</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]Taxiway[SEP]Taxiway[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63306</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>69</td>\n",
" <td>171</td>\n",
" <td>9</td>\n",
" <td>10</td>\n",
" <td>13</td>\n",
" <td>[CLS]AirCarrier[SEP]AircraftOperator[SEP]</td>\n",
" <td>0.614038</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68684</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>75</td>\n",
" <td>59</td>\n",
" <td>13</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]AirspaceLayer[SEP]AirspaceLayer[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>70728</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>77</td>\n",
" <td>273</td>\n",
" <td>2</td>\n",
" <td>23</td>\n",
" <td>24</td>\n",
" <td>[CLS]STAR[SEP]StandardInstrumentArrival[SEP]</td>\n",
" <td>0.178293</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75074</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>82</td>\n",
" <td>44</td>\n",
" <td>14</td>\n",
" <td>15</td>\n",
" <td>16</td>\n",
" <td>[CLS]OperationalRunway[SEP]RunwayDirection[SEP]</td>\n",
" <td>0.835488</td>\n",
" </tr>\n",
" <tr>\n",
" <th>86354</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>94</td>\n",
" <td>344</td>\n",
" <td>8</td>\n",
" <td>4</td>\n",
" <td>4</td>\n",
" <td>[CLS]DeicingPad[SEP]DeicingArea[SEP]</td>\n",
" <td>0.339917</td>\n",
" </tr>\n",
" <tr>\n",
" <th>87004</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>95</td>\n",
" <td>79</td>\n",
" <td>8</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]Terminal[SEP]Terminal[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>94605</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>103</td>\n",
" <td>360</td>\n",
" <td>3</td>\n",
" <td>24</td>\n",
" <td>26</td>\n",
" <td>[CLS]SID[SEP]StandardInstrumentDeparture[SEP]</td>\n",
" <td>-0.116515</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97346</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>106</td>\n",
" <td>356</td>\n",
" <td>4</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]Gate[SEP]Gate[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>104410</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>114</td>\n",
" <td>100</td>\n",
" <td>7</td>\n",
" <td>8</td>\n",
" <td>9</td>\n",
" <td>[CLS]AirportSpec[SEP]AerodromeSet[SEP]</td>\n",
" <td>0.415925</td>\n",
" </tr>\n",
" <tr>\n",
" <th>105559</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>115</td>\n",
" <td>334</td>\n",
" <td>10</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]CrewMember[SEP]CrewMember[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>106142</th>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>116</td>\n",
" <td>2</td>\n",
" <td>6</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>[CLS]Flight[SEP]Flight[SEP]</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>112057</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>122</td>\n",
" <td>427</td>\n",
" <td>6</td>\n",
" <td>20</td>\n",
" <td>20</td>\n",
" <td>[CLS]FlightSpec[SEP]FlightConditionCombination...</td>\n",
" <td>0.760592</td>\n",
" </tr>\n",
" <tr>\n",
" <th>116390</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>127</td>\n",
" <td>185</td>\n",
" <td>13</td>\n",
" <td>9</td>\n",
" <td>9</td>\n",
" <td>[CLS]AircraftTrackPoint[SEP]AircraftState[SEP]</td>\n",
" <td>0.710066</td>\n",
" </tr>\n",
" <tr>\n",
" <th>128868</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>140</td>\n",
" <td>768</td>\n",
" <td>6</td>\n",
" <td>4</td>\n",
" <td>5</td>\n",
" <td>[CLS]Taxipath[SEP]TaxiRoute[SEP]</td>\n",
" <td>0.804810</td>\n",
" </tr>\n",
" <tr>\n",
" <th>131434</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>143</td>\n",
" <td>589</td>\n",
" <td>6</td>\n",
" <td>12</td>\n",
" <td>13</td>\n",
" <td>[CLS]RelativeFix[SEP]PointReference[SEP]</td>\n",
" <td>0.449235</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>32 rows × 21 columns</p>\n",
"</div>"
],
"text/plain": [
" LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n",
"3061 1 1 -1 \n",
"5442 -1 -1 -1 \n",
"5491 -1 -1 -1 \n",
"8752 -1 -1 -1 \n",
"10121 -1 -1 -1 \n",
"11271 -1 -1 -1 \n",
"14389 -1 -1 -1 \n",
"15853 -1 -1 -1 \n",
"23442 -1 -1 -1 \n",
"36940 -1 -1 -1 \n",
"44969 1 1 -1 \n",
"46803 1 1 -1 \n",
"51377 -1 -1 -1 \n",
"53695 -1 -1 -1 \n",
"54340 -1 -1 -1 \n",
"58021 -1 -1 -1 \n",
"60735 1 1 -1 \n",
"63306 -1 -1 -1 \n",
"68684 1 1 -1 \n",
"70728 -1 -1 -1 \n",
"75074 -1 -1 -1 \n",
"86354 -1 -1 -1 \n",
"87004 1 1 -1 \n",
"94605 -1 -1 1 \n",
"97346 1 1 -1 \n",
"104410 -1 -1 -1 \n",
"105559 1 1 -1 \n",
"106142 1 1 -1 \n",
"112057 -1 -1 -1 \n",
"116390 -1 -1 -1 \n",
"128868 -1 -1 -1 \n",
"131434 -1 -1 -1 \n",
"\n",
" LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n",
"3061 -1 -1 1 \n",
"5442 -1 -1 -1 \n",
"5491 1 -1 -1 \n",
"8752 -1 -1 -1 \n",
"10121 -1 -1 -1 \n",
"11271 -1 -1 -1 \n",
"14389 -1 -1 -1 \n",
"15853 -1 -1 -1 \n",
"23442 -1 -1 -1 \n",
"36940 -1 -1 -1 \n",
"44969 1 -1 1 \n",
"46803 -1 -1 1 \n",
"51377 -1 -1 -1 \n",
"53695 -1 -1 -1 \n",
"54340 -1 -1 -1 \n",
"58021 -1 -1 -1 \n",
"60735 1 -1 1 \n",
"63306 -1 -1 -1 \n",
"68684 -1 -1 1 \n",
"70728 -1 -1 -1 \n",
"75074 -1 -1 -1 \n",
"86354 -1 -1 -1 \n",
"87004 1 -1 1 \n",
"94605 -1 -1 -1 \n",
"97346 1 -1 1 \n",
"104410 -1 -1 -1 \n",
"105559 -1 -1 1 \n",
"106142 1 -1 1 \n",
"112057 -1 -1 -1 \n",
"116390 -1 -1 -1 \n",
"128868 -1 -1 -1 \n",
"131434 -1 -1 -1 \n",
"\n",
" LF_class_name_spacy_distance LF_class_name_distance \\\n",
"3061 1 1 \n",
"5442 -1 -1 \n",
"5491 -1 -1 \n",
"8752 -1 -1 \n",
"10121 -1 -1 \n",
"11271 -1 -1 \n",
"14389 -1 -1 \n",
"15853 -1 -1 \n",
"23442 -1 0 \n",
"36940 -1 -1 \n",
"44969 1 1 \n",
"46803 1 1 \n",
"51377 -1 -1 \n",
"53695 -1 -1 \n",
"54340 -1 -1 \n",
"58021 -1 -1 \n",
"60735 1 1 \n",
"63306 -1 -1 \n",
"68684 1 1 \n",
"70728 0 0 \n",
"75074 -1 -1 \n",
"86354 -1 -1 \n",
"87004 1 1 \n",
"94605 0 0 \n",
"97346 1 1 \n",
"104410 -1 -1 \n",
"105559 1 1 \n",
"106142 1 1 \n",
"112057 -1 -1 \n",
"116390 -1 -1 \n",
"128868 -1 -1 \n",
"131434 -1 0 \n",
"\n",
" LF_name_segment_overlap LF_label_words_overlap ... \\\n",
"3061 1 -1 ... \n",
"5442 1 -1 ... \n",
"5491 -1 -1 ... \n",
"8752 1 -1 ... \n",
"10121 -1 -1 ... \n",
"11271 1 -1 ... \n",
"14389 1 -1 ... \n",
"15853 1 -1 ... \n",
"23442 -1 -1 ... \n",
"36940 1 -1 ... \n",
"44969 1 -1 ... \n",
"46803 1 -1 ... \n",
"51377 1 -1 ... \n",
"53695 1 -1 ... \n",
"54340 1 -1 ... \n",
"58021 1 -1 ... \n",
"60735 1 -1 ... \n",
"63306 -1 -1 ... \n",
"68684 1 -1 ... \n",
"70728 -1 -1 ... \n",
"75074 1 -1 ... \n",
"86354 1 -1 ... \n",
"87004 1 -1 ... \n",
"94605 -1 -1 ... \n",
"97346 1 -1 ... \n",
"104410 -1 -1 ... \n",
"105559 1 -1 ... \n",
"106142 1 -1 ... \n",
"112057 1 -1 ... \n",
"116390 1 -1 ... \n",
"128868 1 -1 ... \n",
"131434 -1 -1 ... \n",
"\n",
" LF_superclasses_overlap LF_properties_overlap label x y \\\n",
"3061 -1 -1 1 3 316 \n",
"5442 -1 -1 1 5 867 \n",
"5491 -1 -1 1 6 1 \n",
"8752 -1 -1 1 9 517 \n",
"10121 -1 -1 1 11 56 \n",
"11271 -1 -1 1 12 291 \n",
"14389 -1 -1 1 15 664 \n",
"15853 -1 -1 1 17 298 \n",
"23442 -1 -1 1 25 567 \n",
"36940 -1 -1 1 40 340 \n",
"44969 -1 -1 1 49 134 \n",
"46803 -1 -1 1 51 138 \n",
"51377 -1 -1 1 56 137 \n",
"53695 -1 -1 1 58 625 \n",
"54340 -1 -1 1 59 355 \n",
"58021 -1 -1 1 63 376 \n",
"60735 -1 -1 1 66 345 \n",
"63306 -1 -1 1 69 171 \n",
"68684 -1 -1 1 75 59 \n",
"70728 -1 -1 1 77 273 \n",
"75074 -1 -1 1 82 44 \n",
"86354 -1 -1 1 94 344 \n",
"87004 -1 -1 1 95 79 \n",
"94605 -1 -1 1 103 360 \n",
"97346 -1 -1 1 106 356 \n",
"104410 -1 -1 1 114 100 \n",
"105559 -1 -1 1 115 334 \n",
"106142 -1 -1 1 116 2 \n",
"112057 -1 -1 1 122 427 \n",
"116390 -1 -1 1 127 185 \n",
"128868 -1 -1 1 140 768 \n",
"131434 -1 -1 1 143 589 \n",
"\n",
" shared_word levenshtein_distance hamming_distance \\\n",
"3061 16 0 0 \n",
"5442 9 16 20 \n",
"5491 4 6 6 \n",
"8752 8 8 14 \n",
"10121 9 13 15 \n",
"11271 17 17 19 \n",
"14389 24 14 19 \n",
"15853 12 8 16 \n",
"23442 9 25 30 \n",
"36940 7 7 9 \n",
"44969 8 0 0 \n",
"46803 12 0 0 \n",
"51377 13 10 13 \n",
"53695 13 5 16 \n",
"54340 7 8 14 \n",
"58021 10 12 22 \n",
"60735 7 0 0 \n",
"63306 9 10 13 \n",
"68684 13 0 0 \n",
"70728 2 23 24 \n",
"75074 14 15 16 \n",
"86354 8 4 4 \n",
"87004 8 0 0 \n",
"94605 3 24 26 \n",
"97346 4 0 0 \n",
"104410 7 8 9 \n",
"105559 10 0 0 \n",
"106142 6 0 0 \n",
"112057 6 20 20 \n",
"116390 13 9 9 \n",
"128868 6 4 5 \n",
"131434 6 12 13 \n",
"\n",
" text_pair \\\n",
"3061 [CLS]WeatherCondition[SEP]WeatherCondition[SEP] \n",
"5442 [CLS]SurfaceWindCondition[SEP]Wind[SEP] \n",
"5491 [CLS]Airport[SEP]Aerodrome[SEP] \n",
"8752 [CLS]PhysicalRunway[SEP]Runway[SEP] \n",
"10121 [CLS]NavigationFix[SEP]SignificantPoint[SEP] \n",
"11271 [CLS]VisibilityCondition[SEP]HorizontalVisibil... \n",
"14389 [CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV... \n",
"15853 [CLS]AircraftCapacity[SEP]Capacity[SEP] \n",
"23442 [CLS]AircraftFlow[SEP]OccupancyTrafficMonitori... \n",
"36940 [CLS]TimeInterval[SEP]TimeRangeType[SEP] \n",
"44969 [CLS]Aircraft[SEP]Aircraft[SEP] \n",
"46803 [CLS]AircraftType[SEP]AircraftType[SEP] \n",
"51377 [CLS]AircraftModel[SEP]AircraftMakeModelSeries... \n",
"53695 [CLS]NavigationAid[SEP]RadioNavigationAid[SEP] \n",
"54340 [CLS]AircraftEngine[SEP]Engine[SEP] \n",
"58021 [CLS]EngineType[SEP]CodeAircraftEngineType[SEP] \n",
"60735 [CLS]Taxiway[SEP]Taxiway[SEP] \n",
"63306 [CLS]AirCarrier[SEP]AircraftOperator[SEP] \n",
"68684 [CLS]AirspaceLayer[SEP]AirspaceLayer[SEP] \n",
"70728 [CLS]STAR[SEP]StandardInstrumentArrival[SEP] \n",
"75074 [CLS]OperationalRunway[SEP]RunwayDirection[SEP] \n",
"86354 [CLS]DeicingPad[SEP]DeicingArea[SEP] \n",
"87004 [CLS]Terminal[SEP]Terminal[SEP] \n",
"94605 [CLS]SID[SEP]StandardInstrumentDeparture[SEP] \n",
"97346 [CLS]Gate[SEP]Gate[SEP] \n",
"104410 [CLS]AirportSpec[SEP]AerodromeSet[SEP] \n",
"105559 [CLS]CrewMember[SEP]CrewMember[SEP] \n",
"106142 [CLS]Flight[SEP]Flight[SEP] \n",
"112057 [CLS]FlightSpec[SEP]FlightConditionCombination... \n",
"116390 [CLS]AircraftTrackPoint[SEP]AircraftState[SEP] \n",
"128868 [CLS]Taxipath[SEP]TaxiRoute[SEP] \n",
"131434 [CLS]RelativeFix[SEP]PointReference[SEP] \n",
"\n",
" class_name_embedding_distance \n",
"3061 1.000000 \n",
"5442 0.789100 \n",
"5491 0.508479 \n",
"8752 0.679397 \n",
"10121 0.319003 \n",
"11271 0.810132 \n",
"14389 0.881630 \n",
"15853 0.825409 \n",
"23442 0.628079 \n",
"36940 0.760346 \n",
"44969 1.000000 \n",
"46803 1.000000 \n",
"51377 0.818485 \n",
"53695 0.899368 \n",
"54340 0.898687 \n",
"58021 0.880238 \n",
"60735 1.000000 \n",
"63306 0.614038 \n",
"68684 1.000000 \n",
"70728 0.178293 \n",
"75074 0.835488 \n",
"86354 0.339917 \n",
"87004 1.000000 \n",
"94605 -0.116515 \n",
"97346 1.000000 \n",
"104410 0.415925 \n",
"105559 1.000000 \n",
"106142 1.000000 \n",
"112057 0.760592 \n",
"116390 0.710066 \n",
"128868 0.804810 \n",
"131434 0.449235 \n",
"\n",
"[32 rows x 21 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_dataset_df[all_dataset_df.label==1]"
]
},
{
"cell_type": "markdown",
"id": "a3c7d00f",
"metadata": {},
"source": [
"# common words in class name"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "dff25b09",
"metadata": {},
"outputs": [],
"source": [
"def calculate_common_words(row):\n",
" x = int(row['x'])\n",
" y = int(row['y']) \n",
" source_class = source_ontology[x]\n",
" target_class = target_ontology[y] \n",
"\n",
" s1 = source_class.class_name_words\n",
" s2 = target_class.class_name_words\n",
" t = list(set(s1) & set(s2))\n",
" \n",
" return len(t)\n",
"\n",
"# def calculate_common_words(row):\n",
"# x = int(row['x'])\n",
"# y = int(row['y']) \n",
"# source_class = source_ontology[x]\n",
"# target_class = target_ontology[y] \n",
"\n",
"# s1 = source_class.label_words\n",
"# s2 = target_class.label_words\n",
"# t = list(set(s1) & set(s2))\n",
" \n",
"# return len(t)\n",
" \n",
"all_dataset_df['num_common_words'] = all_dataset_df.apply(calculate_common_words, axis = 1)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "188fd93c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>LF_class_name_equal</th>\n",
" <th>LF_class_name_stemmed_equal</th>\n",
" <th>LF_acronyms</th>\n",
" <th>LF_class_name_synonyms</th>\n",
" <th>LF_label_equal</th>\n",
" <th>LF_root_nouns_equal</th>\n",
" <th>LF_class_name_spacy_distance</th>\n",
" <th>LF_class_name_distance</th>\n",
" <th>LF_name_segment_overlap</th>\n",
" <th>LF_label_words_overlap</th>\n",
" <th>...</th>\n",
" <th>LF_properties_overlap</th>\n",
" <th>label</th>\n",
" <th>x</th>\n",
" <th>y</th>\n",
" <th>shared_word</th>\n",
" <th>levenshtein_distance</th>\n",
" <th>hamming_distance</th>\n",
" <th>text_pair</th>\n",
" <th>class_name_embedding_distance</th>\n",
" <th>num_common_words</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>63</td>\n",
" <td>8</td>\n",
" <td>4</td>\n",
" <td>4</td>\n",
" <td>[CLS]AirportData[SEP]AirportSlot[SEP]</td>\n",
" <td>0.419675</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>72</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>72</td>\n",
" <td>8</td>\n",
" <td>13</td>\n",
" <td>13</td>\n",
" <td>[CLS]AirportData[SEP]AirportGroundService[SEP]</td>\n",
" <td>0.640560</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>83</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>83</td>\n",
" <td>10</td>\n",
" <td>14</td>\n",
" <td>21</td>\n",
" <td>[CLS]AirportData[SEP]CodeStatusAirportType[SEP]</td>\n",
" <td>0.633053</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>86</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>86</td>\n",
" <td>10</td>\n",
" <td>14</td>\n",
" <td>22</td>\n",
" <td>[CLS]AirportData[SEP]CodeAirportWarningType[SEP]</td>\n",
" <td>0.656966</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>153</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>153</td>\n",
" <td>8</td>\n",
" <td>34</td>\n",
" <td>38</td>\n",
" <td>[CLS]AirportData[SEP]CodeDatalinkCommunication...</td>\n",
" <td>0.788275</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>136096</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>148</td>\n",
" <td>676</td>\n",
" <td>11</td>\n",
" <td>9</td>\n",
" <td>19</td>\n",
" <td>[CLS]PopupFactorSequence[SEP]RunwayMixSequence...</td>\n",
" <td>0.553210</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>137115</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>149</td>\n",
" <td>780</td>\n",
" <td>14</td>\n",
" <td>13</td>\n",
" <td>26</td>\n",
" <td>[CLS]IntersectionFix[SEP]TaxiwayIntersectionMa...</td>\n",
" <td>0.541662</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>138706</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>151</td>\n",
" <td>541</td>\n",
" <td>3</td>\n",
" <td>3</td>\n",
" <td>3</td>\n",
" <td>[CLS]NDBfix[SEP]NDB[SEP]</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>138707</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>151</td>\n",
" <td>542</td>\n",
" <td>3</td>\n",
" <td>13</td>\n",
" <td>16</td>\n",
" <td>[CLS]NDBfix[SEP]CodeNDBUsageType[SEP]</td>\n",
" <td>0.223525</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>139430</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>152</td>\n",
" <td>350</td>\n",
" <td>7</td>\n",
" <td>8</td>\n",
" <td>14</td>\n",
" <td>[CLS]PopupFactor[SEP]EmissionFactor[SEP]</td>\n",
" <td>0.735463</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3262 rows × 22 columns</p>\n",
"</div>"
],
"text/plain": [
" LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n",
"63 -1 -1 -1 \n",
"72 -1 -1 -1 \n",
"83 -1 -1 -1 \n",
"86 -1 -1 -1 \n",
"153 -1 -1 -1 \n",
"... ... ... ... \n",
"136096 -1 -1 -1 \n",
"137115 -1 -1 -1 \n",
"138706 -1 -1 -1 \n",
"138707 -1 -1 -1 \n",
"139430 -1 -1 -1 \n",
"\n",
" LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n",
"63 -1 -1 -1 \n",
"72 -1 -1 -1 \n",
"83 -1 -1 -1 \n",
"86 -1 -1 -1 \n",
"153 -1 -1 -1 \n",
"... ... ... ... \n",
"136096 -1 -1 -1 \n",
"137115 -1 -1 -1 \n",
"138706 -1 -1 -1 \n",
"138707 -1 -1 -1 \n",
"139430 -1 -1 -1 \n",
"\n",
" LF_class_name_spacy_distance LF_class_name_distance \\\n",
"63 -1 -1 \n",
"72 -1 -1 \n",
"83 -1 -1 \n",
"86 -1 -1 \n",
"153 -1 0 \n",
"... ... ... \n",
"136096 -1 -1 \n",
"137115 -1 -1 \n",
"138706 -1 -1 \n",
"138707 -1 0 \n",
"139430 -1 -1 \n",
"\n",
" LF_name_segment_overlap LF_label_words_overlap ... \\\n",
"63 1 -1 ... \n",
"72 1 -1 ... \n",
"83 1 -1 ... \n",
"86 1 -1 ... \n",
"153 1 -1 ... \n",
"... ... ... ... \n",
"136096 1 -1 ... \n",
"137115 1 -1 ... \n",
"138706 1 -1 ... \n",
"138707 1 -1 ... \n",
"139430 1 -1 ... \n",
"\n",
" LF_properties_overlap label x y shared_word \\\n",
"63 -1 0 0 63 8 \n",
"72 -1 0 0 72 8 \n",
"83 -1 0 0 83 10 \n",
"86 -1 0 0 86 10 \n",
"153 -1 0 0 153 8 \n",
"... ... ... ... ... ... \n",
"136096 -1 0 148 676 11 \n",
"137115 -1 0 149 780 14 \n",
"138706 -1 0 151 541 3 \n",
"138707 -1 0 151 542 3 \n",
"139430 -1 0 152 350 7 \n",
"\n",
" levenshtein_distance hamming_distance \\\n",
"63 4 4 \n",
"72 13 13 \n",
"83 14 21 \n",
"86 14 22 \n",
"153 34 38 \n",
"... ... ... \n",
"136096 9 19 \n",
"137115 13 26 \n",
"138706 3 3 \n",
"138707 13 16 \n",
"139430 8 14 \n",
"\n",
" text_pair \\\n",
"63 [CLS]AirportData[SEP]AirportSlot[SEP] \n",
"72 [CLS]AirportData[SEP]AirportGroundService[SEP] \n",
"83 [CLS]AirportData[SEP]CodeStatusAirportType[SEP] \n",
"86 [CLS]AirportData[SEP]CodeAirportWarningType[SEP] \n",
"153 [CLS]AirportData[SEP]CodeDatalinkCommunication... \n",
"... ... \n",
"136096 [CLS]PopupFactorSequence[SEP]RunwayMixSequence... \n",
"137115 [CLS]IntersectionFix[SEP]TaxiwayIntersectionMa... \n",
"138706 [CLS]NDBfix[SEP]NDB[SEP] \n",
"138707 [CLS]NDBfix[SEP]CodeNDBUsageType[SEP] \n",
"139430 [CLS]PopupFactor[SEP]EmissionFactor[SEP] \n",
"\n",
" class_name_embedding_distance num_common_words \n",
"63 0.419675 1 \n",
"72 0.640560 1 \n",
"83 0.633053 1 \n",
"86 0.656966 1 \n",
"153 0.788275 1 \n",
"... ... ... \n",
"136096 0.553210 1 \n",
"137115 0.541662 1 \n",
"138706 NaN 1 \n",
"138707 0.223525 1 \n",
"139430 0.735463 1 \n",
"\n",
"[3262 rows x 22 columns]"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_dataset_df.query('num_common_words >= 1')"
]
},
{
"cell_type": "markdown",
"id": "c128e601",
"metadata": {},
"source": [
"# embedding distance of selected features"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "6d5ffb50",
"metadata": {},
"outputs": [],
"source": [
"from scipy import spatial\n",
"import numpy as np\n",
"\n",
"def calculate_cosine_similarity(s_embedding, t_embedding): \n",
" if s_embedding is None or t_embedding is None:\n",
" return None \n",
" \n",
" if len(s_embedding) == 0 or len(t_embedding) == 0:\n",
" return None\n",
" \n",
" if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:\n",
" return None \n",
"\n",
" similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)\n",
" return similarity\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "4d6e4ab9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"class_long_name\n",
"compute feature embedding\n",
"compute embedding distance\n",
"label_long\n",
"compute feature embedding\n",
"compute embedding distance\n",
"comment\n",
"compute feature embedding\n",
"compute embedding distance\n"
]
}
],
"source": [
"# calculate the embedding for the selected feature\n",
"from sentence_transformers import SentenceTransformer\n",
"distilbert = SentenceTransformer('distilbert-base-nli-mean-tokens')\n",
"\n",
"features = ['class_long_name', 'label_long', 'comment']\n",
"\n",
"def computer_embedding(ontology_list, feature, embedding_feature):\n",
" for idx in range(len(ontology_list)):\n",
" obj = ontology_list[idx]\n",
" \n",
" attr = getattr(obj, feature)\n",
" if attr != None and attr != \"\":\n",
" em = distilbert.encode(getattr(obj, feature)) \n",
" setattr(obj, embedding_feature, em)\n",
"\n",
"\n",
"def calculate_embedding_distance(row, selected_feature):\n",
" x = int(row['x'])\n",
" y = int(row['y']) \n",
" source_class = source_ontology[x]\n",
" target_class = target_ontology[y] \n",
"\n",
" if hasattr(source_class, selected_feature) and hasattr(target_class, selected_feature):\n",
" em1 = getattr(source_class, selected_feature)\n",
" em2 = getattr(target_class, selected_feature)\n",
" \n",
" return calculate_cosine_similarity(em1, em2) \n",
" else:\n",
" return None\n",
"\n",
"for feature in features: \n",
" print(feature)\n",
"\n",
" embedding_feature = feature + '_embedding' \n",
" computer_embedding(target_ontology, feature, embedding_feature) \n",
" computer_embedding(source_ontology, feature, embedding_feature) \n",
" print(\"compute feature embedding\")\n",
"\n",
" embedding_distance = feature + '_distance' \n",
" all_dataset_df[embedding_distance] = all_dataset_df.apply(calculate_embedding_distance, selected_feature=embedding_feature, axis = 1)\n",
" print(\"compute embedding distance\")\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "710d66cf",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>LF_class_name_equal</th>\n",
" <th>LF_class_name_stemmed_equal</th>\n",
" <th>LF_acronyms</th>\n",
" <th>LF_class_name_synonyms</th>\n",
" <th>LF_label_equal</th>\n",
" <th>LF_root_nouns_equal</th>\n",
" <th>LF_class_name_spacy_distance</th>\n",
" <th>LF_class_name_distance</th>\n",
" <th>LF_name_segment_overlap</th>\n",
" <th>LF_label_words_overlap</th>\n",
" <th>...</th>\n",
" <th>y</th>\n",
" <th>shared_word</th>\n",
" <th>levenshtein_distance</th>\n",
" <th>hamming_distance</th>\n",
" <th>text_pair</th>\n",
" <th>class_name_embedding_distance</th>\n",
" <th>num_common_words</th>\n",
" <th>class_long_name_distance</th>\n",
" <th>label_long_distance</th>\n",
" <th>comment_distance</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>9</td>\n",
" <td>12</td>\n",
" <td>14</td>\n",
" <td>[CLS]AirportData[SEP]ACDMIrregularity[SEP]</td>\n",
" <td>0.355953</td>\n",
" <td>0</td>\n",
" <td>0.310721</td>\n",
" <td>None</td>\n",
" <td>0.305137</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>1</td>\n",
" <td>4</td>\n",
" <td>8</td>\n",
" <td>8</td>\n",
" <td>[CLS]AirportData[SEP]Aerodrome[SEP]</td>\n",
" <td>0.237307</td>\n",
" <td>0</td>\n",
" <td>0.686413</td>\n",
" <td>None</td>\n",
" <td>0.647584</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>10</td>\n",
" <td>11</td>\n",
" <td>[CLS]AirportData[SEP]Flight[SEP]</td>\n",
" <td>0.454936</td>\n",
" <td>0</td>\n",
" <td>0.697756</td>\n",
" <td>None</td>\n",
" <td>0.599904</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>3</td>\n",
" <td>11</td>\n",
" <td>20</td>\n",
" <td>24</td>\n",
" <td>[CLS]AirportData[SEP]CodeACDMIrregularityType[...</td>\n",
" <td>0.515220</td>\n",
" <td>0</td>\n",
" <td>0.375069</td>\n",
" <td>None</td>\n",
" <td>0.387794</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>4</td>\n",
" <td>10</td>\n",
" <td>13</td>\n",
" <td>15</td>\n",
" <td>[CLS]AirportData[SEP]ADSBGroundStation[SEP]</td>\n",
" <td>0.481079</td>\n",
" <td>0</td>\n",
" <td>0.421668</td>\n",
" <td>None</td>\n",
" <td>0.553725</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>910</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>910</td>\n",
" <td>9</td>\n",
" <td>26</td>\n",
" <td>30</td>\n",
" <td>[CLS]AirportData[SEP]CodeUomVolumeConcentratio...</td>\n",
" <td>0.565876</td>\n",
" <td>0</td>\n",
" <td>0.418236</td>\n",
" <td>None</td>\n",
" <td>0.298020</td>\n",
" </tr>\n",
" <tr>\n",
" <th>911</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>911</td>\n",
" <td>2</td>\n",
" <td>16</td>\n",
" <td>17</td>\n",
" <td>[CLS]AirportData[SEP]CodeUomVolumeType[SEP]</td>\n",
" <td>0.529073</td>\n",
" <td>0</td>\n",
" <td>0.435312</td>\n",
" <td>None</td>\n",
" <td>0.417220</td>\n",
" </tr>\n",
" <tr>\n",
" <th>912</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>0</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>912</td>\n",
" <td>4</td>\n",
" <td>13</td>\n",
" <td>16</td>\n",
" <td>[CLS]AirportData[SEP]EntityCollection[SEP]</td>\n",
" <td>0.508757</td>\n",
" <td>0</td>\n",
" <td>0.478009</td>\n",
" <td>None</td>\n",
" <td>0.349885</td>\n",
" </tr>\n",
" <tr>\n",
" <th>913</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>913</td>\n",
" <td>9</td>\n",
" <td>23</td>\n",
" <td>27</td>\n",
" <td>[CLS]AirportData[SEP]LongitudinalSeparationMin...</td>\n",
" <td>0.440626</td>\n",
" <td>0</td>\n",
" <td>0.376916</td>\n",
" <td>None</td>\n",
" <td>0.516962</td>\n",
" </tr>\n",
" <tr>\n",
" <th>914</th>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>-1</td>\n",
" <td>...</td>\n",
" <td>914</td>\n",
" <td>8</td>\n",
" <td>21</td>\n",
" <td>23</td>\n",
" <td>[CLS]AirportData[SEP]SurveillanceGroundStation...</td>\n",
" <td>0.562463</td>\n",
" <td>0</td>\n",
" <td>0.505302</td>\n",
" <td>None</td>\n",
" <td>0.615024</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>915 rows × 25 columns</p>\n",
"</div>"
],
"text/plain": [
" LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n",
"0 -1 -1 -1 \n",
"1 -1 -1 -1 \n",
"2 -1 -1 -1 \n",
"3 -1 -1 -1 \n",
"4 -1 -1 -1 \n",
".. ... ... ... \n",
"910 -1 -1 -1 \n",
"911 -1 -1 -1 \n",
"912 -1 -1 -1 \n",
"913 -1 -1 -1 \n",
"914 -1 -1 -1 \n",
"\n",
" LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n",
"0 -1 -1 -1 \n",
"1 -1 -1 -1 \n",
"2 -1 -1 -1 \n",
"3 -1 -1 -1 \n",
"4 -1 -1 -1 \n",
".. ... ... ... \n",
"910 -1 -1 -1 \n",
"911 -1 -1 -1 \n",
"912 -1 -1 -1 \n",
"913 -1 -1 -1 \n",
"914 -1 -1 -1 \n",
"\n",
" LF_class_name_spacy_distance LF_class_name_distance \\\n",
"0 -1 -1 \n",
"1 -1 -1 \n",
"2 -1 0 \n",
"3 -1 0 \n",
"4 -1 -1 \n",
".. ... ... \n",
"910 -1 0 \n",
"911 -1 0 \n",
"912 -1 0 \n",
"913 -1 -1 \n",
"914 -1 -1 \n",
"\n",
" LF_name_segment_overlap LF_label_words_overlap ... y shared_word \\\n",
"0 -1 -1 ... 0 9 \n",
"1 -1 -1 ... 1 4 \n",
"2 -1 -1 ... 2 3 \n",
"3 -1 -1 ... 3 11 \n",
"4 -1 -1 ... 4 10 \n",
".. ... ... ... ... ... \n",
"910 -1 -1 ... 910 9 \n",
"911 -1 -1 ... 911 2 \n",
"912 -1 -1 ... 912 4 \n",
"913 -1 -1 ... 913 9 \n",
"914 -1 -1 ... 914 8 \n",
"\n",
" levenshtein_distance hamming_distance \\\n",
"0 12 14 \n",
"1 8 8 \n",
"2 10 11 \n",
"3 20 24 \n",
"4 13 15 \n",
".. ... ... \n",
"910 26 30 \n",
"911 16 17 \n",
"912 13 16 \n",
"913 23 27 \n",
"914 21 23 \n",
"\n",
" text_pair \\\n",
"0 [CLS]AirportData[SEP]ACDMIrregularity[SEP] \n",
"1 [CLS]AirportData[SEP]Aerodrome[SEP] \n",
"2 [CLS]AirportData[SEP]Flight[SEP] \n",
"3 [CLS]AirportData[SEP]CodeACDMIrregularityType[... \n",
"4 [CLS]AirportData[SEP]ADSBGroundStation[SEP] \n",
".. ... \n",
"910 [CLS]AirportData[SEP]CodeUomVolumeConcentratio... \n",
"911 [CLS]AirportData[SEP]CodeUomVolumeType[SEP] \n",
"912 [CLS]AirportData[SEP]EntityCollection[SEP] \n",
"913 [CLS]AirportData[SEP]LongitudinalSeparationMin... \n",
"914 [CLS]AirportData[SEP]SurveillanceGroundStation... \n",
"\n",
" class_name_embedding_distance num_common_words \\\n",
"0 0.355953 0 \n",
"1 0.237307 0 \n",
"2 0.454936 0 \n",
"3 0.515220 0 \n",
"4 0.481079 0 \n",
".. ... ... \n",
"910 0.565876 0 \n",
"911 0.529073 0 \n",
"912 0.508757 0 \n",
"913 0.440626 0 \n",
"914 0.562463 0 \n",
"\n",
" class_long_name_distance label_long_distance comment_distance \n",
"0 0.310721 None 0.305137 \n",
"1 0.686413 None 0.647584 \n",
"2 0.697756 None 0.599904 \n",
"3 0.375069 None 0.387794 \n",
"4 0.421668 None 0.553725 \n",
".. ... ... ... \n",
"910 0.418236 None 0.298020 \n",
"911 0.435312 None 0.417220 \n",
"912 0.478009 None 0.349885 \n",
"913 0.376916 None 0.516962 \n",
"914 0.505302 None 0.615024 \n",
"\n",
"[915 rows x 25 columns]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_dataset_df.query('x==0')"
]
},
{
"cell_type": "markdown",
"id": "530cf757",
"metadata": {},
"source": [
"# blocking based on change point detection over ranked distance"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "4e758096",
"metadata": {},
"outputs": [],
"source": [
"import changefinder\n",
"def findChangePoints(ts, r, order, smooth):\n",
" '''\n",
" r: Discounting rate\n",
" order: AR model order\n",
" smooth: smoothing window size T\n",
" '''\n",
" cf = changefinder.ChangeFinder(r=r, order=order, smooth=smooth)\n",
" ts_score = [cf.update(p) for p in ts]\n",
" return(ts_score)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "06bb881a",
"metadata": {},
"outputs": [],
"source": [
"def print_true_match(row):\n",
" x = int(row['x'])\n",
" y = int(row['y']) \n",
" source_class = source_ontology[x]\n",
" target_class = target_ontology[y] \n",
" print(source_class.class_name, ' => ', target_class.class_name, row['x'], row['y'])\n",
" \n",
"def print_included_match(row, f):\n",
" x = int(row['x'])\n",
" y = int(row['y']) \n",
" source_class = source_ontology[x]\n",
" target_class = target_ontology[y] \n",
" print(source_class.class_name, ' => ', target_class.class_name, row[f], row['x'], row['y'])\n",
" \n",
"def blocking_per_class(xdf, ranked_feature, check_point_detection, with_delta, debug_flag):\n",
" # for ranked integer feature, the last increase\n",
" if ranked_feature in ['num_common_words']:\n",
" return len(xdf[xdf[ranked_feature] > 0])\n",
"\n",
" distances = xdf[ranked_feature].values\n",
" \n",
" num = len(distances) \n",
" points = []\n",
" \n",
" if with_delta == True:\n",
" for i in range (num-1):\n",
" cur_delta = abs(distances[i+1] - distances[i])\n",
" points.append(cur_delta) \n",
" else:\n",
" points = distances \n",
" \n",
" top_x = num\n",
" scores = []\n",
" \n",
" if check_point_detection == \"rbf\": \n",
" model = \"rbf\" \n",
" algo = rpt.Dynp(model=model).fit(np.array(points))\n",
" scores = algo.predict(n_bkps=1) \n",
" top_x = scores[0]-1\n",
" elif check_point_detection == \"l2\": \n",
" model = \"l2\"\n",
" algo = rpt.Dynp(model=model).fit(np.array(points))\n",
" scores = algo.predict(n_bkps=1) \n",
" top_x = scores[0]-1\n",
" elif check_point_detection == \"l1\": \n",
" model = \"l1\"\n",
" algo = rpt.Dynp(model=model).fit(np.array(points))\n",
" scores = algo.predict(n_bkps=1) \n",
" top_x = scores[0]-1\n",
" elif check_point_detection == \"online1\": \n",
" scores = findChangePoints(points, r = 0.01, order = 3, smooth = 5) \n",
" top_x = pd.Series(scores).nlargest(3).index[2]\n",
" elif check_point_detection == \"online2\": \n",
" cf = ChangeFinder()\n",
" \n",
" try:\n",
" scores = [cf.update(p) for p in points] \n",
" for idx in range(len(scores)):\n",
" if scores[idx] < 2:\n",
" top_x = idx+1\n",
" break \n",
" except:\n",
" top_x = 0\n",
" \n",
" if debug_flag == True:\n",
" print(points)\n",
" print(scores)\n",
" print(top_x)\n",
" \n",
" xdf[:top_x].apply(print_included_match, f=ranked_feature, axis = 1)\n",
"\n",
" plt.figure(figsize=(16,4))\n",
" plt.plot(distances, color='blue')\n",
" \n",
" plt.figure(figsize=(16,4))\n",
" plt.plot(points)\n",
" \n",
" plt.figure(figsize=(16,4))\n",
" plt.plot(scores, color='red') \n",
" \n",
" plt.show() \n",
" \n",
" return top_x\n",
" \n",
"\n",
"def apply_blocking(df, column, algm, max_k):\n",
" df[column+'_blocked'] = 1\n",
" \n",
" for x in range(len(source_ontology)):\n",
" x_df = df.query('x==' + str(x) + ' &' + column + '.notnull()')[[column, 'x', 'y', column+'_blocked']]\n",
" \n",
" if len(x_df) == 0:\n",
" print(x, '0, ignored') \n",
" continue\n",
" \n",
" x_df = x_df.sort_values(column, ascending=False) \n",
" top_x = blocking_per_class(x_df.head(max_k), column, algm, True, False)\n",
" if top_x == 0:\n",
" print(x, '0, ignored')\n",
" continue\n",
" \n",
" # update the block_flag\n",
" #print(x, top_x)\n",
" df.loc[x_df[:top_x].index, column+'_blocked'] = 0\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6ffebc5f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"50\n",
"\n",
"class_long_name_distance\n",
"\n",
"num_common_words\n",
"8 0, ignored\n",
"30 0, ignored\n",
"33 0, ignored\n",
"34 0, ignored\n",
"35 0, ignored\n",
"36 0, ignored\n",
"37 0, ignored\n",
"39 0, ignored\n",
"41 0, ignored\n",
"45 0, ignored\n",
"46 0, ignored\n",
"73 0, ignored\n",
"74 0, ignored\n",
"76 0, ignored\n",
"77 0, ignored\n",
"83 0, ignored\n",
"92 0, ignored\n",
"101 0, ignored\n",
"103 0, ignored\n",
"108 0, ignored\n",
"110 0, ignored\n",
"111 0, ignored\n",
"112 0, ignored\n",
"113 0, ignored\n",
"118 0, ignored\n",
"124 0, ignored\n",
"139 0, ignored\n",
"142 0, ignored\n",
"150 0, ignored\n",
"153 0, ignored\n",
"\n",
"label_long_distance\n",
"0 0, ignored\n",
"1 0, ignored\n",
"2 0, ignored\n",
"3 0, ignored\n",
"4 0, ignored\n",
"5 0, ignored\n",
"6 0, ignored\n",
"7 0, ignored\n",
"8 0, ignored\n",
"9 0, ignored\n",
"10 0, ignored\n",
"11 0, ignored\n",
"12 0, ignored\n",
"13 0, ignored\n",
"14 0, ignored\n",
"15 0, ignored\n",
"16 0, ignored\n",
"17 0, ignored\n",
"18 0, ignored\n",
"19 0, ignored\n",
"20 0, ignored\n",
"21 0, ignored\n",
"22 0, ignored\n",
"23 0, ignored\n",
"24 0, ignored\n",
"25 0, ignored\n",
"26 0, ignored\n",
"27 0, ignored\n",
"28 0, ignored\n",
"29 0, ignored\n",
"30 0, ignored\n",
"31 0, ignored\n",
"32 0, ignored\n",
"33 0, ignored\n",
"34 0, ignored\n",
"35 0, ignored\n",
"36 0, ignored\n",
"37 0, ignored\n",
"38 0, ignored\n",
"39 0, ignored\n",
"40 0, ignored\n",
"41 0, ignored\n",
"42 0, ignored\n",
"43 0, ignored\n",
"44 0, ignored\n",
"45 0, ignored\n",
"46 0, ignored\n",
"47 0, ignored\n",
"48 0, ignored\n",
"49 0, ignored\n",
"50 0, ignored\n",
"51 0, ignored\n",
"52 0, ignored\n",
"53 0, ignored\n",
"54 0, ignored\n",
"55 0, ignored\n",
"56 0, ignored\n",
"57 0, ignored\n",
"58 0, ignored\n",
"59 0, ignored\n",
"60 0, ignored\n",
"61 0, ignored\n",
"62 0, ignored\n",
"63 0, ignored\n",
"64 0, ignored\n",
"65 0, ignored\n",
"66 0, ignored\n",
"67 0, ignored\n",
"68 0, ignored\n",
"69 0, ignored\n",
"70 0, ignored\n",
"71 0, ignored\n",
"72 0, ignored\n",
"73 0, ignored\n",
"74 0, ignored\n",
"75 0, ignored\n",
"76 0, ignored\n",
"77 0, ignored\n",
"78 0, ignored\n",
"79 0, ignored\n",
"80 0, ignored\n",
"81 0, ignored\n",
"82 0, ignored\n",
"83 0, ignored\n",
"84 0, ignored\n",
"85 0, ignored\n",
"86 0, ignored\n",
"87 0, ignored\n",
"88 0, ignored\n",
"89 0, ignored\n",
"90 0, ignored\n",
"91 0, ignored\n",
"92 0, ignored\n",
"93 0, ignored\n",
"94 0, ignored\n",
"95 0, ignored\n",
"96 0, ignored\n",
"97 0, ignored\n",
"98 0, ignored\n",
"99 0, ignored\n",
"100 0, ignored\n",
"101 0, ignored\n",
"102 0, ignored\n",
"103 0, ignored\n",
"104 0, ignored\n",
"105 0, ignored\n",
"106 0, ignored\n",
"107 0, ignored\n",
"108 0, ignored\n",
"109 0, ignored\n",
"110 0, ignored\n",
"111 0, ignored\n",
"112 0, ignored\n",
"113 0, ignored\n",
"114 0, ignored\n",
"115 0, ignored\n",
"116 0, ignored\n",
"117 0, ignored\n",
"118 0, ignored\n",
"119 0, ignored\n",
"120 0, ignored\n",
"121 0, ignored\n",
"122 0, ignored\n",
"123 0, ignored\n",
"124 0, ignored\n",
"125 0, ignored\n",
"126 0, ignored\n",
"127 0, ignored\n",
"128 0, ignored\n",
"129 0, ignored\n",
"130 0, ignored\n",
"131 0, ignored\n",
"132 0, ignored\n",
"133 0, ignored\n",
"134 0, ignored\n",
"135 0, ignored\n",
"136 0, ignored\n",
"137 0, ignored\n",
"138 0, ignored\n",
"139 0, ignored\n",
"140 0, ignored\n",
"141 0, ignored\n",
"142 0, ignored\n",
"143 0, ignored\n",
"144 0, ignored\n",
"145 0, ignored\n",
"146 0, ignored\n",
"147 0, ignored\n",
"148 0, ignored\n",
"149 0, ignored\n",
"150 0, ignored\n",
"151 0, ignored\n",
"152 0, ignored\n",
"153 0, ignored\n",
"\n",
"comment_distance\n"
]
}
],
"source": [
"columns = ['class_long_name_distance', 'num_common_words', 'label_long_distance', 'comment_distance' ]\n",
"\n",
"MAX_NUM_CANDIDATES = 50\n",
"target_size = len(target_ontology)\n",
"max_k = target_size\n",
"if target_size > MAX_NUM_CANDIDATES:\n",
" max_k = MAX_NUM_CANDIDATES\n",
"print(max_k)\n",
"\n",
"changing_point_detection_algorithm = 'online1' # 'rdf', 'l1', 'l2', 'online1', 'online2'\n",
"print_matches = True\n",
"\n",
"for blocked_column in columns:\n",
" print(\"\\r\\n\" + blocked_column)\n",
" blocked_key = blocked_column+'_blocked'\n",
" apply_blocking(all_dataset_df, blocked_column, changing_point_detection_algorithm, max_k)\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "492e8068",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r\n",
"class_long_name_distance\n",
"WeatherCondition => WeatherCondition 1.0 3 316\n",
"SurfaceWindCondition => Wind 0.8790944218635559 5 867\n",
"Airport => Aerodrome 0.840453565120697 6 1\n",
"PhysicalRunway => Runway 0.8516059517860413 9 517\n",
"RunwayVisibleRangeMeasurement => RunwayVisualRange 0.9156750440597534 15 664\n",
"TimeInterval => TimeRangeType 0.8980520367622375 40 340\n",
"Aircraft => Aircraft 1.0 49 134\n",
"AircraftType => AircraftType 1.0 51 138\n",
"AircraftModel => AircraftMakeModelSeries 0.8626878261566162 56 137\n",
"NavigationAid => RadioNavigationAid 0.7773143649101257 58 625\n",
"EngineType => CodeAircraftEngineType 0.8093550205230713 63 376\n",
"Taxiway => Taxiway 1.0 66 345\n",
"AirCarrier => AircraftOperator 0.8524434566497803 69 171\n",
"AirspaceLayer => AirspaceLayer 1.0 75 59\n",
"OperationalRunway => RunwayDirection 0.8838163614273071 82 44\n",
"DeicingPad => DeicingArea 0.9137077927589417 94 344\n",
"Terminal => Terminal 1.0 95 79\n",
"Gate => Gate 1.0 106 356\n",
"AirportSpec => AerodromeSet 0.820888102054596 114 100\n",
"CrewMember => CrewMember 1.0 115 334\n",
"Flight => Flight 1.0 116 2\n",
"FlightSpec => FlightConditionCombination 0.8406736254692078 122 427\n",
"Taxipath => TaxiRoute 0.9633474946022034 140 768\n",
"RelativeFix => PointReference 0.7955299615859985 143 589\n",
"2968 out of 140910 2.106308991554893\n",
"24 out of 32 75.0\n",
"\r\n",
"num_common_words\n",
"WeatherCondition => WeatherCondition 2 3 316\n",
"SurfaceWindCondition => Wind 1 5 867\n",
"PhysicalRunway => Runway 1 9 517\n",
"VisibilityCondition => HorizontalVisibility 1 12 291\n",
"RunwayVisibleRangeMeasurement => RunwayVisualRange 2 15 664\n",
"AircraftCapacity => Capacity 1 17 298\n",
"TimeInterval => TimeRangeType 1 40 340\n",
"Aircraft => Aircraft 1 49 134\n",
"AircraftType => AircraftType 2 51 138\n",
"AircraftModel => AircraftMakeModelSeries 2 56 137\n",
"NavigationAid => RadioNavigationAid 1 58 625\n",
"AircraftEngine => Engine 1 59 355\n",
"EngineType => CodeAircraftEngineType 2 63 376\n",
"Taxiway => Taxiway 1 66 345\n",
"AirspaceLayer => AirspaceLayer 2 75 59\n",
"OperationalRunway => RunwayDirection 1 82 44\n",
"DeicingPad => DeicingArea 1 94 344\n",
"Terminal => Terminal 1 95 79\n",
"Gate => Gate 1 106 356\n",
"CrewMember => CrewMember 2 115 334\n",
"Flight => Flight 1 116 2\n",
"FlightSpec => FlightConditionCombination 1 122 427\n",
"AircraftTrackPoint => AircraftState 1 127 185\n",
"Taxipath => TaxiRoute 1 140 768\n",
"2382 out of 140910 1.6904407068341494\n",
"24 out of 32 75.0\n",
"\r\n",
"label_long_distance\n",
"0 out of 140910 0.0\n",
"0 out of 32 0.0\n",
"\r\n",
"comment_distance\n",
"WeatherCondition => WeatherCondition 0.7918438911437988 3 316\n",
"PhysicalRunway => Runway 0.7813198566436768 9 517\n",
"VisibilityCondition => HorizontalVisibility 0.8003379702568054 12 291\n",
"RunwayVisibleRangeMeasurement => RunwayVisualRange 0.9089569449424744 15 664\n",
"AircraftCapacity => Capacity 0.8342739939689636 17 298\n",
"AircraftFlow => OccupancyTrafficMonitoringValue 0.8646406531333923 25 567\n",
"TimeInterval => TimeRangeType 0.7229282855987549 40 340\n",
"AircraftModel => AircraftMakeModelSeries 0.8741695284843445 56 137\n",
"NavigationAid => RadioNavigationAid 0.7332932353019714 58 625\n",
"EngineType => CodeAircraftEngineType 0.8456313014030457 63 376\n",
"AirCarrier => AircraftOperator 0.9076859354972839 69 171\n",
"AirspaceLayer => AirspaceLayer 0.7635785937309265 75 59\n",
"OperationalRunway => RunwayDirection 0.7357498407363892 82 44\n",
"Terminal => Terminal 0.849309504032135 95 79\n",
"SID => StandardInstrumentDeparture 0.7947181463241577 103 360\n",
"Gate => Gate 0.7989940047264099 106 356\n",
"AirportSpec => AerodromeSet 0.8796765804290771 114 100\n",
"CrewMember => CrewMember 0.8921281695365906 115 334\n",
"Taxipath => TaxiRoute 0.865944504737854 140 768\n",
"2978 out of 140910 2.113405719963097\n",
"19 out of 32 59.375\n",
"\r\n",
"\n",
"7151 out of 140910 5.07487048470655\n",
"30 out of 32 93.75\n",
"\r\n",
"=======list of all true matches========\n",
"WeatherCondition => WeatherCondition 3 316\n",
"SurfaceWindCondition => Wind 5 867\n",
"Airport => Aerodrome 6 1\n",
"PhysicalRunway => Runway 9 517\n",
"NavigationFix => SignificantPoint 11 56\n",
"VisibilityCondition => HorizontalVisibility 12 291\n",
"RunwayVisibleRangeMeasurement => RunwayVisualRange 15 664\n",
"AircraftCapacity => Capacity 17 298\n",
"AircraftFlow => OccupancyTrafficMonitoringValue 25 567\n",
"TimeInterval => TimeRangeType 40 340\n",
"Aircraft => Aircraft 49 134\n",
"AircraftType => AircraftType 51 138\n",
"AircraftModel => AircraftMakeModelSeries 56 137\n",
"NavigationAid => RadioNavigationAid 58 625\n",
"AircraftEngine => Engine 59 355\n",
"EngineType => CodeAircraftEngineType 63 376\n",
"Taxiway => Taxiway 66 345\n",
"AirCarrier => AircraftOperator 69 171\n",
"AirspaceLayer => AirspaceLayer 75 59\n",
"STAR => StandardInstrumentArrival 77 273\n",
"OperationalRunway => RunwayDirection 82 44\n",
"DeicingPad => DeicingArea 94 344\n",
"Terminal => Terminal 95 79\n",
"SID => StandardInstrumentDeparture 103 360\n",
"Gate => Gate 106 356\n",
"AirportSpec => AerodromeSet 114 100\n",
"CrewMember => CrewMember 115 334\n",
"Flight => Flight 116 2\n",
"FlightSpec => FlightConditionCombination 122 427\n",
"AircraftTrackPoint => AircraftState 127 185\n",
"Taxipath => TaxiRoute 140 768\n",
"RelativeFix => PointReference 143 589\n",
"\r\n",
"***** MISSING ****\n",
"NavigationFix => SignificantPoint 11 56\n",
"STAR => StandardInstrumentArrival 77 273\n"
]
}
],
"source": [
"# show the matches included by different blocking indexes\n",
"combination_statement = \"\";\n",
"missing_statement = \"\"\n",
"for blocked_column in columns:\n",
" print(\"\\r\\n\" + blocked_column)\n",
" blocked_key = blocked_column+'_blocked'\n",
" \n",
" all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis = 1)\n",
" \n",
" total_num = len(all_dataset_df)\n",
" num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))\n",
"\n",
" print(num_after_blocking, ' out of ', total_num, num_after_blocking*100.0/total_num)\n",
"\n",
" total_true_matches = len(all_dataset_df[all_dataset_df['label']==1])\n",
" captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))\n",
"\n",
" print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches*100.0/total_true_matches)\n",
" \n",
" if combination_statement == \"\":\n",
" combination_statement = \" (\" + blocked_key + '==0)'\n",
" missing_statement = \" (\" + blocked_key + '==1)' \n",
" else:\n",
" combination_statement = combination_statement + \" | (\" + blocked_key + '==0)'\n",
" missing_statement = missing_statement + \" & (\" + blocked_key + '==1)'\n",
"\n",
"all_dataset_df['selected_after_blocking'] = 0\n",
" \n",
"# calculate the union of all selected items by the committee of blockers\n",
"print(\"\\r\\n\")\n",
"#print(combination_statement)\n",
"selected_items = all_dataset_df.query(combination_statement)\n",
"all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1\n",
"total_selected_items = len(selected_items)\n",
"\n",
"\n",
"query = '(label==1) & (' + combination_statement + ')'\n",
"#print(query)\n",
"total_selected_matches = len(all_dataset_df.query(query))\n",
"\n",
"query = 'label==1' \n",
"#print(query)\n",
"total_matches = len(all_dataset_df.query(query))\n",
"\n",
"total_num = len(all_dataset_df)\n",
"\n",
"\n",
"print(total_selected_items, ' out of ', total_num, total_selected_items*100.0/total_num)\n",
"print(total_selected_matches, ' out of ', total_matches, total_selected_matches*100.0/total_matches)\n",
"\n",
"print(\"\\r\\n=======list of all true matches========\")\n",
"true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis = 1)\n",
"\n",
"print(\"\\r\\n***** MISSING ****\")\n",
"\n",
"query = '(label==1) & (' + missing_statement + ')'\n",
"#print(query)\n",
"true_matches = all_dataset_df.query(query).apply(print_true_match, axis = 1)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e19726d3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}