{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Step 0 Load all required ontologies into RDF4J\n", "\n", "See loader.py and config.py scripts, each ontology will be loaded into a running RDF4J server as seperate repository.\n", "\n", "- please check the configuration in the config.py first\n", "\n", "- There is a parameter to run \"python load.py\".\n", " 1. If you run \"python load.py clean\", this means cleaning all repositories that have been loaded into rdf4j before.\n", " 2. If you run \"python load.py load\", this means loading all ontology files located in your data folder, which is specified in the config.py file\n", " \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### The following code can reload the imported python code files if any change is made during the experiments" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Step 1 Preprocessing to construct alignment profiles per class\n", "\n", "This step prepares our data to be access by labeling functions.\n", "For each class in each ontology we construct an alignment profile that consists of useful context information about that class. For example, the class name, class comment, number of sub and super classes etc. Finally we store these alignment profiles in a pandas dataframe" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### include the dataset folder" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['/home/bcheng/aaai22', '/home/bcheng/anaconda3/envs/dualloop/lib/python38.zip', '/home/bcheng/anaconda3/envs/dualloop/lib/python3.8', '/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/lib-dynload', '', '/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages', '/home/bcheng/aaai22/data']\n" ] } ], "source": [ "import sys\n", "import os\n", "sys.path.append(os.path.abspath(\"./data\"))\n", "print(sys.path)\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/huggingface_hub/snapshot_download.py:6: FutureWarning: snapshot_download.py has been made private and will no longer be available from version 0.11. Please use `from huggingface_hub import snapshot_download` to import the only public function in this module. Other members of the file may be changed without a deprecation notice.\n", " warnings.warn(\n" ] } ], "source": [ "import os\n", "import requests\n", "\n", "import preprocessing as preprocess\n", "import idp as idp\n", "\n", "import config_conference as config\n", "import config_nasa as config\n", "#import config_iot as config\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "http://192.168.0.16:8080/rdf4j-server/repositories/atmonto\n", "http://192.168.0.16:8080/rdf4j-server/repositories/airm\n", "Alignment Construction Time: 91.49237751960754 s\n" ] } ], "source": [ "import time\n", "start = time.time()\n", "\n", "rdf4j_server_url = config.RDF4J_SERVER + \"/rdf4j-server/repositories/\" \n", "\n", "source_ontology = []\n", "target_ontology = []\n", "\n", "sourceMap = {}\n", "targetMap = {}\n", "\n", "for dataset in config.TRAIN_DATASET: \n", " #add one ontology into the source_ontology\n", " repository_name = rdf4j_server_url + dataset['source'] \n", " alignment_profiles = preprocess.build_alignment_profile(repository_name, True) \n", " for profile in alignment_profiles:\n", " uri = profile.uri.lower()\n", " if uri not in sourceMap:\n", " sourceMap[uri] = profile\n", " source_ontology.append(profile)\n", " \n", "# source_ontology = source_ontology + alignment_profiles\n", "# num_of_class_x = len(alignment_profiles)\n", " \n", "\n", " #add one ontology into the target_ontology\n", " repository_name = rdf4j_server_url + dataset['target']\n", " alignment_profiles = preprocess.build_alignment_profile(repository_name, True) \n", " for profile in alignment_profiles:\n", " uri = profile.uri.lower()\n", " if uri not in targetMap:\n", " targetMap[uri] = profile\n", " target_ontology.append(profile) \n", " \n", "# target_ontology = target_ontology + alignment_profiles\n", "# num_of_class_y = len(alignment_profiles) \n", "\n", "# dataset_size_xy.append([num_of_class_x, num_of_class_y])\n", " \n", "end = time.time()\n", "print(\"Alignment Construction Time: \", end - start, \"s\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "154" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(source_ontology)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "915" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(target_ontology)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 1.1 Create a dataframe to index all class pairs [all combinations]\n", "\n", "include all class-pair combination between source ontology and target ontology into a pandas dataframe" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialization time of all class pairs: 0.16331076622009277 s\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "start = time.time()\n", "\n", "items = []\n", "\n", "begin_x = 0\n", "begin_y = 0\n", "for i in range(len(source_ontology)):\n", " for j in range(len(target_ontology)):\n", " x = begin_x + i\n", " y = begin_y + j\n", " items.append({'x': x, 'y': y})\n", "\n", "# begin_x = 0\n", "# begin_y = 0\n", "# for xy in dataset_size_xy:\n", "# for i in range(xy[0]):\n", "# for j in range(xy[1]):\n", "# x = begin_x + i\n", "# y = begin_y + j\n", "# items.append({'x': x, 'y': y})\n", " \n", "# begin_x = begin_x + xy[0]\n", "# begin_y = begin_y + xy[1]\n", "\n", "cols = ['x', 'y'] \n", "all_combinations = pd.DataFrame(items, columns = cols) \n", " \n", "end = time.time()\n", "print(\"Initialization time of all class pairs: \", end - start, \"s\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
xy
000
101
202
303
404
.........
140905153910
140906153911
140907153912
140908153913
140909153914
\n", "

140910 rows × 2 columns

\n", "
" ], "text/plain": [ " x y\n", "0 0 0\n", "1 0 1\n", "2 0 2\n", "3 0 3\n", "4 0 4\n", "... ... ...\n", "140905 153 910\n", "140906 153 911\n", "140907 153 912\n", "140908 153 913\n", "140909 153 914\n", "\n", "[140910 rows x 2 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_combinations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 1.2 Load the groundtruth into the same dataframe\n", "\n", "Read groundtruth data and get matching classes fron alignment RDF file." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "data/groundtruth/nasa/atmonto-airm.rdf\n", "32\n", "32\n", "Time to set all ground truth labels: 0.22237014770507812 s\n" ] } ], "source": [ "def get_class_index(uri, clist):\n", " for idx in range(len(clist)):\n", " #print(clist[idx].uri)\n", " if clist[idx].uri.lower() == uri.lower():\n", " return idx\n", " \n", " return -1\n", "\n", "\n", "start = time.time()\n", "\n", "all_combinations['label'] = 0\n", "\n", "ground_truth = []\n", "for dataset in config.TRAIN_DATASET:\n", " fix_uri = config.NAME == 'conference'\n", " matches = preprocess.get_groundtruth(dataset['groundtruth'], fix_uri)\n", " \n", " print(len(matches))\n", " ground_truth = ground_truth + matches\n", "\n", " \n", "for match in ground_truth:\n", " s_URI = match['source_class']\n", " t_URI = match['target_class']\n", " \n", " # find the location of class in the class list of source ontology and target ontology\n", " sIdx = get_class_index(s_URI, source_ontology)\n", " tIdx = get_class_index(t_URI, target_ontology)\n", " \n", " if (sIdx < 0):\n", " print(\"SOURCE_CLASS MISSING:\" + s_URI)\n", " \n", " if (tIdx < 0):\n", " print(\"TARGET_CLASS MISSING:\" + t_URI) \n", " \n", " # update the dataframe\n", " if (sIdx >= 0) and (tIdx >= 0): \n", " all_combinations.label[(all_combinations.x == sIdx) & (all_combinations.y == tIdx)] = 1\n", " \n", " \n", "print(len(all_combinations[all_combinations.label == 1]))\n", "\n", "end = time.time()\n", "print(\"Time to set all ground truth labels: \", end - start, \"s\")" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
xylabel
306133161
544258671
5491611
875295171
1012111561
11271122911
14389156641
15853172981
23442255671
36940403401
44969491341
46803511381
51377561371
53695586251
54340593551
58021633761
60735663451
63306691711
6868475591
70728772731
7507482441
86354943441
8700495791
946051033601
973461063561
1044101141001
1055591153341
10614211621
1120571224271
1163901271851
1288681407681
1314341435891
\n", "
" ], "text/plain": [ " x y label\n", "3061 3 316 1\n", "5442 5 867 1\n", "5491 6 1 1\n", "8752 9 517 1\n", "10121 11 56 1\n", "11271 12 291 1\n", "14389 15 664 1\n", "15853 17 298 1\n", "23442 25 567 1\n", "36940 40 340 1\n", "44969 49 134 1\n", "46803 51 138 1\n", "51377 56 137 1\n", "53695 58 625 1\n", "54340 59 355 1\n", "58021 63 376 1\n", "60735 66 345 1\n", "63306 69 171 1\n", "68684 75 59 1\n", "70728 77 273 1\n", "75074 82 44 1\n", "86354 94 344 1\n", "87004 95 79 1\n", "94605 103 360 1\n", "97346 106 356 1\n", "104410 114 100 1\n", "105559 115 334 1\n", "106142 116 2 1\n", "112057 122 427 1\n", "116390 127 185 1\n", "128868 140 768 1\n", "131434 143 589 1" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_combinations[all_combinations.label == 1] # to show all true matches in the ground truth" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 1.3 Load the results given by existing methods\n", "\n", "Read the results (alignment RDF file) predicted by existing methods." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "aml\n", "data/baselines/nasa/AML/AMLAlignment_Auto.rdf\n", "15\n", "Uknown source class: https://data.nasa.gov/ontologies/atmonto/equipment#aircraftRegistrant\n", "Uknown target class: https://w3id.org/airm-o/ontology#Aircraft-aircraftRegistration\n", "Uknown source class: https://data.nasa.gov/ontologies/atmonto/ATM#Sector\n", "logmap\n", "data/baselines/nasa/LogMap/LogMapAlignment.rdf\n", "12\n", "Uknown source class: https://data.nasa.gov/ontologies/atmonto/equipment#aircraftRegistrant\n", "Uknown target class: https://w3id.org/airm-o/ontology#Aircraft-aircraftRegistration\n", "Uknown source class: https://data.nasa.gov/ontologies/atmonto/equipment#hasAircraftModel\n", "Uknown target class: https://w3id.org/airm-o/ontology#Aircraft-aircraftModel\n", "yam\n", "data/baselines/nasa/yam/YAMPlusPlusAlignment.rdf\n", "11\n" ] } ], "source": [ "ground_truth = []\n", "for dataset in config.TRAIN_DATASET:\n", " methods = dataset['existing_methods']\n", " for m in methods:\n", " print(m)\n", " fix_uri = config.NAME == 'conference' \n", " predicted_matches = preprocess.get_groundtruth(methods[m], fix_uri) \n", " print(len(predicted_matches))\n", " \n", " all_combinations[m] = 0\n", " \n", " for match in predicted_matches:\n", " s_URI = match['source_class']\n", " t_URI = match['target_class']\n", " \n", " # find the location of class in the class list of source ontology and target ontology\n", " sIdx = get_class_index(s_URI, source_ontology)\n", " tIdx = get_class_index(t_URI, target_ontology)\n", " \n", " # update the dataframe\n", " if (sIdx >= 0) and (tIdx >= 0):\n", " all_combinations[m][(all_combinations.x == sIdx) & (all_combinations.y == tIdx)] = 1\n", " continue\n", " else:\n", " if(sIdx < 0):\n", " print(\"Uknown source class: \" + s_URI)\n", "\n", " if(tIdx < 0):\n", " print(\"Uknown target class: \" + t_URI) \n", " " ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# of matches predicted by aml : 13\n", "# of matches predicted by logmap : 10\n", "# of matches predicted by yam : 11\n" ] } ], "source": [ "for dataset in config.TRAIN_DATASET:\n", " methods = dataset['existing_methods']\n", " for m in methods:\n", " print(\"# of matches predicted by \" + m + \" : \" + str(len(all_combinations[all_combinations[m] == 1])))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 1.4 Validate the initialized dataframe " ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
xylabelamllogmapyam
0000000
1010000
2020000
3030000
4040000
.....................
1409051539100000
1409061539110000
1409071539120000
1409081539130000
1409091539140000
\n", "

140910 rows × 6 columns

\n", "
" ], "text/plain": [ " x y label aml logmap yam\n", "0 0 0 0 0 0 0\n", "1 0 1 0 0 0 0\n", "2 0 2 0 0 0 0\n", "3 0 3 0 0 0 0\n", "4 0 4 0 0 0 0\n", "... ... ... ... ... ... ...\n", "140905 153 910 0 0 0 0\n", "140906 153 911 0 0 0 0\n", "140907 153 912 0 0 0 0\n", "140908 153 913 0 0 0 0\n", "140909 153 914 0 0 0 0\n", "\n", "[140910 rows x 6 columns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_combinations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Step 2 Labelling \n", "\n", "Labeling functions based on domain knowledge, knowledge bases etc. The goal is it to combine multiple matchers." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Step 2.1 Labeling function to compare two classes directly" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "ABSTAIN = -1\n", "UNMATCHED = 0\n", "MATCHED = 1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Step 2.1.1 Define labeling functions" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package wordnet to /home/bcheng/nltk_data...\n", "[nltk_data] Package wordnet is already up-to-date!\n" ] } ], "source": [ "from snorkel.labeling import labeling_function\n", "\n", "import nltk\n", "nltk.download('wordnet')\n", "\n", "from fuzzywuzzy import fuzz\n", "\n", "import re\n", "\n", "import wordsegment \n", "wordsegment.load()\n", "\n", "def intersection(lst1, lst2): \n", " if lst1 == None or lst2 == None:\n", " return [] \n", " \n", " lst3 = [value for value in lst1 if value in lst2] \n", " return lst3 \n", "\n", "from scipy import spatial\n", "import numpy as np\n", "\n", "def calculate_cosine_similarity(s_embedding, t_embedding): \n", " if s_embedding is None or t_embedding is None:\n", " return None \n", " \n", " if len(s_embedding) == 0 or len(t_embedding) == 0:\n", " return None\n", " \n", " if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:\n", " return None \n", "\n", " similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)\n", " return similarity\n" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ " \n", "@labeling_function()\n", "def LF_aml(row): \n", " return row.aml\n", " \n", " \n", "@labeling_function()\n", "def LF_logmap(row): \n", " return row.logmap\n", " \n", " \n", "@labeling_function()\n", "def LF_yam(row): \n", " return row.yam\n", "\n", "@labeling_function()\n", "def LF_class_name_equal(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y]\n", " \n", " if source_class.class_name.lower() == target_class.class_name.lower():\n", " return MATCHED\n", " else:\n", " return ABSTAIN\n", " \n", "@labeling_function()\n", "def LF_class_name_stemmed_equal(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.class_name_stemmed.lower() == target_class.class_name_stemmed.lower():\n", " return MATCHED\n", " else:\n", " return ABSTAIN\n", "\n", "@labeling_function()\n", "def LF_acronyms(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " name_x = source_class.class_name\n", " name_y = target_class.class_name\n", " \n", " acro_x = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \\1', re.sub('([A-Z]+)', r' \\1', name_x)).split()])\n", " acro_y = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \\1', re.sub('([A-Z]+)', r' \\1', name_y)).split()])\n", " if name_x.lower() == acro_y.lower() or name_y.lower() == acro_x.lower():\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", " \n", "@labeling_function()\n", "def LF_class_name_synonyms(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " from nltk.corpus import wordnet\n", " a_class_synonyms = []\n", " for syn in wordnet.synsets(source_class.class_name):\n", " for lm in syn.lemmas():\n", " a_class_synonyms.append(lm.name())\n", " b_class_synonyms = []\n", " for syn in wordnet.synsets(target_class.class_name):\n", " for lm in syn.lemmas():\n", " b_class_synonyms.append(lm.name())\n", " \n", " common = list(set(a_class_synonyms) & set(b_class_synonyms))\n", " if len(common) > 0:\n", " return MATCHED\n", " \n", " return ABSTAIN\n", "\n", "\n", "@labeling_function()\n", "def LF_label_equal(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.label == None or target_class.label == None:\n", " return ABSTAIN\n", " \n", " if source_class.label == '' or target_class.label == '':\n", " return ABSTAIN \n", " \n", " if source_class.label == target_class.label:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", " \n", " \n", "@labeling_function()\n", "def LF_root_nouns_equal(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y]\n", " \n", " if source_class.class_name_root == None or target_class.class_name_root == None:\n", " return ABSTAIN\n", " \n", " if source_class.class_name_root.lower() == target_class.class_name_root.lower() and fuzz.ratio(source_class.class_name_root, target_class.class_name_root) > 90:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", " \n", "\n", " \n", "@labeling_function()\n", "def LF_class_name_spacy_distance(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " s = calculate_cosine_similarity(source_class.class_long_name_embedding, target_class.class_long_name_embedding) \n", " if s == None:\n", " return ABSTAIN\n", " elif s > 0.95:\n", " return MATCHED\n", " elif s < 0.20:\n", " return UNMATCHED\n", " return ABSTAIN\n", " \n", "\n", "@labeling_function() \n", "def LF_class_name_distance(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " r = fuzz.ratio(source_class.class_name, target_class.class_name)\n", " if r > 90:\n", " return MATCHED\n", " if r < 30:\n", " return UNMATCHED\n", " else:\n", " return ABSTAIN \n", " \n", " \n", "@labeling_function()\n", "def LF_name_segment_overlap(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.class_name_words == None or target_class.class_name_words == None:\n", " return ABSTAIN \n", " \n", " overlapset = intersection(source_class.class_name_words, target_class.class_name_words) \n", " if len(overlapset) >= 1:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", " \n", "\n", "@labeling_function()\n", "def LF_label_words_overlap(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.label_words == None or target_class.label_words == None:\n", " return ABSTAIN \n", " \n", " overlapset = intersection(source_class.label_words, target_class.label_words) \n", " if len(overlapset) >= 1:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", "\n", "@labeling_function()\n", "def LF_subclasses_overlap(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.subclasses == None or target_class.subclasses == None:\n", " return ABSTAIN \n", " \n", " overlapset = intersection(source_class.subclasses, target_class.subclasses) \n", " if len(overlapset) >= 1:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", "\n", "@labeling_function()\n", "def LF_superclasses_overlap(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.superclasses == None or target_class.superclasses == None:\n", " return ABSTAIN \n", " \n", " overlapset = intersection(source_class.superclasses, target_class.superclasses) \n", " if len(overlapset) >= 1:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n", " \n", "@labeling_function()\n", "def LF_properties_overlap(row):\n", " x = int(row['x'])\n", " y = int(row['y']) \n", " source_class = source_ontology[x]\n", " target_class = target_ontology[y] \n", " \n", " if source_class.properties == None or target_class.properties == None:\n", " return ABSTAIN \n", " \n", " overlapset = intersection(source_class.properties, target_class.properties) \n", " if len(overlapset) >= 1:\n", " return MATCHED\n", " else:\n", " return ABSTAIN \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Step 2.1.2 Execute labeling functions" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 140910/140910 [00:43<00:00, 3235.04it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Labeling Time: 44.27173733711243 s\n" ] } ], "source": [ "lfs = [\n", "# LF_aml, \n", "# LF_logmap, \n", "# LF_yam, \n", " \n", " LF_class_name_equal, \n", " LF_class_name_stemmed_equal,\n", " LF_acronyms, \n", " LF_class_name_synonyms, \n", " LF_label_equal, \n", " LF_root_nouns_equal, \n", " \n", " LF_class_name_spacy_distance, \n", " LF_class_name_distance, \n", " \n", " LF_name_segment_overlap, \n", " LF_label_words_overlap,\n", " LF_subclasses_overlap,\n", " LF_superclasses_overlap,\n", " LF_properties_overlap,\n", " ]\n", "\n", "start = time.time()\n", "from snorkel.labeling import PandasLFApplier\n", "applier = PandasLFApplier(lfs=lfs)\n", "label_matrix = applier.apply(df=all_combinations)\n", "end = time.time()\n", "print(\"Labeling Time: \", end - start, \"s\")\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[-1, -1, -1, ..., -1, -1, -1],\n", " [-1, -1, -1, ..., -1, -1, -1],\n", " [-1, -1, -1, ..., -1, -1, -1],\n", " ...,\n", " [-1, -1, -1, ..., -1, -1, -1],\n", " [-1, -1, -1, ..., -1, -1, -1],\n", " [-1, -1, -1, ..., -1, -1, -1]])" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label_matrix" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Step 2.1.3 Prepare the label matrix and y" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "153 LF_class_name_equal\n", "153 LF_class_name_stemmed_equal\n", "153 LF_acronyms\n", "153 LF_class_name_synonyms\n", "153 LF_label_equal\n", "153 LF_root_nouns_equal\n", "153 LF_class_name_spacy_distance\n", "153 LF_class_name_distance\n", "153 LF_name_segment_overlap\n", "153 LF_label_words_overlap\n", "153 LF_subclasses_overlap\n", "153 LF_superclasses_overlap\n", "153 LF_properties_overlap\n" ] } ], "source": [ "label_columns = []\n", "for lf in lfs:\n", " print(i, lf.name)\n", " label_columns.append(lf.name)\n", "\n", "my_label_df = pd.DataFrame(data=label_matrix, columns=label_columns) \n", "\n", "my_label_df['label'] = all_combinations.label.values\n", "my_label_df['x'] = all_combinations.x.values\n", "my_label_df['y'] = all_combinations.y.values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Step 2.1.4 Analyze the performance of each LF" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlapLF_subclasses_overlapLF_superclasses_overlapLF_properties_overlaplabelxy
0-1-1-1-1-1-1-1-1-1-1-1-1-1000
1-1-1-1-1-1-1-1-1-1-1-1-1-1001
2-1-1-1-1-1-1-10-1-1-1-1-1002
3-1-1-1-1-1-1-10-1-1-1-1-1003
4-1-1-1-1-1-1-1-1-1-1-1-1-1004
...................................................
140905-1-1-1-1-1-100-1-1-1-1-10153910
140906-1-1-1-1-1-100-1-1-1-1-10153911
140907-1-1-1-1-1-100-1-1-1-1-10153912
140908-1-1-1-1-1-100-1-1-1-1-10153913
140909-1-1-1-1-1-100-1-1-1-1-10153914
\n", "

140910 rows × 16 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "0 -1 -1 \n", "1 -1 -1 \n", "2 -1 0 \n", "3 -1 0 \n", "4 -1 -1 \n", "... ... ... \n", "140905 0 0 \n", "140906 0 0 \n", "140907 0 0 \n", "140908 0 0 \n", "140909 0 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap \\\n", "0 -1 -1 \n", "1 -1 -1 \n", "2 -1 -1 \n", "3 -1 -1 \n", "4 -1 -1 \n", "... ... ... \n", "140905 -1 -1 \n", "140906 -1 -1 \n", "140907 -1 -1 \n", "140908 -1 -1 \n", "140909 -1 -1 \n", "\n", " LF_subclasses_overlap LF_superclasses_overlap LF_properties_overlap \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " label x y \n", "0 0 0 0 \n", "1 0 0 1 \n", "2 0 0 2 \n", "3 0 0 3 \n", "4 0 0 4 \n", "... ... ... ... \n", "140905 0 153 910 \n", "140906 0 153 911 \n", "140907 0 153 912 \n", "140908 0 153 913 \n", "140909 0 153 914 \n", "\n", "[140910 rows x 16 columns]" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "my_label_df" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
jPolarityCoverageOverlapsConflictsCorrectIncorrectEmp. Acc.True PositivesFalse PositivesFalse NegativesTrue NegativesAbstained PositivesAbstained Negatives
LF_class_name_equal0[1]0.0000640.0000640.000000901.000000900023140878
LF_class_name_stemmed_equal1[1]0.0000640.0000640.000000901.000000900023140878
LF_acronyms2[1]0.0000070.0000070.000007101.000000100031140878
LF_class_name_synonyms3[1]0.0000570.0000500.000014620.750000620026140876
LF_label_equal4[]0.0000000.0000000.000000000.000000000032140878
LF_root_nouns_equal5[1]0.0000640.0000640.000000901.000000900023140878
LF_class_name_spacy_distance6[0, 1]0.1380170.1255910.0000851944260.9996919421943321121441
LF_class_name_distance7[0, 1]0.6982400.1266550.0011509838270.999929934983731942502
LF_name_segment_overlap8[1]0.0231500.0012920.0011852432380.007357243238008137640
LF_label_words_overlap9[]0.0000000.0000000.000000000.000000000032140878
LF_subclasses_overlap10[]0.0000000.0000000.000000000.000000000032140878
LF_superclasses_overlap11[]0.0000000.0000000.000000000.000000000032140878
LF_properties_overlap12[]0.0000000.0000000.000000000.000000000032140878
\n", "
" ], "text/plain": [ " j Polarity Coverage Overlaps Conflicts \\\n", "LF_class_name_equal 0 [1] 0.000064 0.000064 0.000000 \n", "LF_class_name_stemmed_equal 1 [1] 0.000064 0.000064 0.000000 \n", "LF_acronyms 2 [1] 0.000007 0.000007 0.000007 \n", "LF_class_name_synonyms 3 [1] 0.000057 0.000050 0.000014 \n", "LF_label_equal 4 [] 0.000000 0.000000 0.000000 \n", "LF_root_nouns_equal 5 [1] 0.000064 0.000064 0.000000 \n", "LF_class_name_spacy_distance 6 [0, 1] 0.138017 0.125591 0.000085 \n", "LF_class_name_distance 7 [0, 1] 0.698240 0.126655 0.001150 \n", "LF_name_segment_overlap 8 [1] 0.023150 0.001292 0.001185 \n", "LF_label_words_overlap 9 [] 0.000000 0.000000 0.000000 \n", "LF_subclasses_overlap 10 [] 0.000000 0.000000 0.000000 \n", "LF_superclasses_overlap 11 [] 0.000000 0.000000 0.000000 \n", "LF_properties_overlap 12 [] 0.000000 0.000000 0.000000 \n", "\n", " Correct Incorrect Emp. Acc. True Positives \\\n", "LF_class_name_equal 9 0 1.000000 9 \n", "LF_class_name_stemmed_equal 9 0 1.000000 9 \n", "LF_acronyms 1 0 1.000000 1 \n", "LF_class_name_synonyms 6 2 0.750000 6 \n", "LF_label_equal 0 0 0.000000 0 \n", "LF_root_nouns_equal 9 0 1.000000 9 \n", "LF_class_name_spacy_distance 19442 6 0.999691 9 \n", "LF_class_name_distance 98382 7 0.999929 9 \n", "LF_name_segment_overlap 24 3238 0.007357 24 \n", "LF_label_words_overlap 0 0 0.000000 0 \n", "LF_subclasses_overlap 0 0 0.000000 0 \n", "LF_superclasses_overlap 0 0 0.000000 0 \n", "LF_properties_overlap 0 0 0.000000 0 \n", "\n", " False Positives False Negatives \\\n", "LF_class_name_equal 0 0 \n", "LF_class_name_stemmed_equal 0 0 \n", "LF_acronyms 0 0 \n", "LF_class_name_synonyms 2 0 \n", "LF_label_equal 0 0 \n", "LF_root_nouns_equal 0 0 \n", "LF_class_name_spacy_distance 4 2 \n", "LF_class_name_distance 3 4 \n", "LF_name_segment_overlap 3238 0 \n", "LF_label_words_overlap 0 0 \n", "LF_subclasses_overlap 0 0 \n", "LF_superclasses_overlap 0 0 \n", "LF_properties_overlap 0 0 \n", "\n", " True Negatives Abstained Positives \\\n", "LF_class_name_equal 0 23 \n", "LF_class_name_stemmed_equal 0 23 \n", "LF_acronyms 0 31 \n", "LF_class_name_synonyms 0 26 \n", "LF_label_equal 0 32 \n", "LF_root_nouns_equal 0 23 \n", "LF_class_name_spacy_distance 19433 21 \n", "LF_class_name_distance 98373 19 \n", "LF_name_segment_overlap 0 8 \n", "LF_label_words_overlap 0 32 \n", "LF_subclasses_overlap 0 32 \n", "LF_superclasses_overlap 0 32 \n", "LF_properties_overlap 0 32 \n", "\n", " Abstained Negatives \n", "LF_class_name_equal 140878 \n", "LF_class_name_stemmed_equal 140878 \n", "LF_acronyms 140878 \n", "LF_class_name_synonyms 140876 \n", "LF_label_equal 140878 \n", "LF_root_nouns_equal 140878 \n", "LF_class_name_spacy_distance 121441 \n", "LF_class_name_distance 42502 \n", "LF_name_segment_overlap 137640 \n", "LF_label_words_overlap 140878 \n", "LF_subclasses_overlap 140878 \n", "LF_superclasses_overlap 140878 \n", "LF_properties_overlap 140878 " ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "label_matrix = my_label_df[label_columns].to_numpy()\n", "y_values = my_label_df.label.to_numpy()\n", "\n", "summary = idp.get_lf_analysis_summary_with_label_matrix(label_matrix, y_values, lfs)\n", "summary" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0 0 0 ... 0 0 0]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n", "/home/bcheng/anaconda3/envs/dualloop/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass labels=[-1 0 1] as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n", " warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "0 0 0 9\n", "0 0 0 9\n", "0 0 0 1\n", "0 2 0 6\n", "0 0 0 0\n", "0 0 0 9\n", "19433 4 2 9\n", "98373 3 4 9\n", "0 3238 0 24\n", "0 0 0 0\n", "0 0 0 0\n", "0 0 0 0\n", "0 0 0 0\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
jPolarityCoverageOverlapsConflictsCorrectIncorrectEmp. Acc.True PositivesFalse PositivesFalse NegativesTrue Negatives
LF_class_name_equal0[1]0.0000640.0000640.000000901.0000009000
LF_class_name_stemmed_equal1[1]0.0000640.0000640.000000901.0000009000
LF_acronyms2[1]0.0000070.0000070.000007101.0000001000
LF_class_name_synonyms3[1]0.0000570.0000500.000014620.7500006200
LF_label_equal4[]0.0000000.0000000.000000000.0000000000
LF_root_nouns_equal5[1]0.0000640.0000640.000000901.0000009000
LF_class_name_spacy_distance6[0, 1]0.1380170.1255910.0000851944260.99969194219433
LF_class_name_distance7[0, 1]0.6982400.1266550.0011509838270.99992993498373
LF_name_segment_overlap8[1]0.0231500.0012920.0011852432380.00735724323800
LF_label_words_overlap9[]0.0000000.0000000.000000000.0000000000
LF_subclasses_overlap10[]0.0000000.0000000.000000000.0000000000
LF_superclasses_overlap11[]0.0000000.0000000.000000000.0000000000
LF_properties_overlap12[]0.0000000.0000000.000000000.0000000000
\n", "
" ], "text/plain": [ " j Polarity Coverage Overlaps Conflicts \\\n", "LF_class_name_equal 0 [1] 0.000064 0.000064 0.000000 \n", "LF_class_name_stemmed_equal 1 [1] 0.000064 0.000064 0.000000 \n", "LF_acronyms 2 [1] 0.000007 0.000007 0.000007 \n", "LF_class_name_synonyms 3 [1] 0.000057 0.000050 0.000014 \n", "LF_label_equal 4 [] 0.000000 0.000000 0.000000 \n", "LF_root_nouns_equal 5 [1] 0.000064 0.000064 0.000000 \n", "LF_class_name_spacy_distance 6 [0, 1] 0.138017 0.125591 0.000085 \n", "LF_class_name_distance 7 [0, 1] 0.698240 0.126655 0.001150 \n", "LF_name_segment_overlap 8 [1] 0.023150 0.001292 0.001185 \n", "LF_label_words_overlap 9 [] 0.000000 0.000000 0.000000 \n", "LF_subclasses_overlap 10 [] 0.000000 0.000000 0.000000 \n", "LF_superclasses_overlap 11 [] 0.000000 0.000000 0.000000 \n", "LF_properties_overlap 12 [] 0.000000 0.000000 0.000000 \n", "\n", " Correct Incorrect Emp. Acc. True Positives \\\n", "LF_class_name_equal 9 0 1.000000 9 \n", "LF_class_name_stemmed_equal 9 0 1.000000 9 \n", "LF_acronyms 1 0 1.000000 1 \n", "LF_class_name_synonyms 6 2 0.750000 6 \n", "LF_label_equal 0 0 0.000000 0 \n", "LF_root_nouns_equal 9 0 1.000000 9 \n", "LF_class_name_spacy_distance 19442 6 0.999691 9 \n", "LF_class_name_distance 98382 7 0.999929 9 \n", "LF_name_segment_overlap 24 3238 0.007357 24 \n", "LF_label_words_overlap 0 0 0.000000 0 \n", "LF_subclasses_overlap 0 0 0.000000 0 \n", "LF_superclasses_overlap 0 0 0.000000 0 \n", "LF_properties_overlap 0 0 0.000000 0 \n", "\n", " False Positives False Negatives True Negatives \n", "LF_class_name_equal 0 0 0 \n", "LF_class_name_stemmed_equal 0 0 0 \n", "LF_acronyms 0 0 0 \n", "LF_class_name_synonyms 2 0 0 \n", "LF_label_equal 0 0 0 \n", "LF_root_nouns_equal 0 0 0 \n", "LF_class_name_spacy_distance 4 2 19433 \n", "LF_class_name_distance 3 4 98373 \n", "LF_name_segment_overlap 3238 0 0 \n", "LF_label_words_overlap 0 0 0 \n", "LF_subclasses_overlap 0 0 0 \n", "LF_superclasses_overlap 0 0 0 \n", "LF_properties_overlap 0 0 0 " ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "summary = idp.get_lf_analysis_summary_with_df(my_label_df, lfs)\n", "summary" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Step 2.1.5 Analyze the performance of the generative model with all LFs" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Computing O...\n", "INFO:root:Estimating \\mu...\n", " 0%| | 0/200 [00:00\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
shared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distance
091214[CLS]AirportData[SEP]ACDMIrregularity[SEP]0.355953
1488[CLS]AirportData[SEP]Aerodrome[SEP]0.237307
231011[CLS]AirportData[SEP]Flight[SEP]0.454936
3112024[CLS]AirportData[SEP]CodeACDMIrregularityType[...0.51522
4101315[CLS]AirportData[SEP]ADSBGroundStation[SEP]0.481079
..................
14090512930[CLS]FRDfix[SEP]CodeUomVolumeConcentrationType...0.126923
14090601717[CLS]FRDfix[SEP]CodeUomVolumeType[SEP]0.171529
14090711516[CLS]FRDfix[SEP]EntityCollection[SEP]-0.071783
14090812727[CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP]0.071912
14090912425[CLS]FRDfix[SEP]SurveillanceGroundStation[SEP]-0.020724
\n", "

140910 rows × 5 columns

\n", "" ], "text/plain": [ " shared_word levenshtein_distance hamming_distance \\\n", "0 9 12 14 \n", "1 4 8 8 \n", "2 3 10 11 \n", "3 11 20 24 \n", "4 10 13 15 \n", "... ... ... ... \n", "140905 1 29 30 \n", "140906 0 17 17 \n", "140907 1 15 16 \n", "140908 1 27 27 \n", "140909 1 24 25 \n", "\n", " text_pair \\\n", "0 [CLS]AirportData[SEP]ACDMIrregularity[SEP] \n", "1 [CLS]AirportData[SEP]Aerodrome[SEP] \n", "2 [CLS]AirportData[SEP]Flight[SEP] \n", "3 [CLS]AirportData[SEP]CodeACDMIrregularityType[... \n", "4 [CLS]AirportData[SEP]ADSBGroundStation[SEP] \n", "... ... \n", "140905 [CLS]FRDfix[SEP]CodeUomVolumeConcentrationType... \n", "140906 [CLS]FRDfix[SEP]CodeUomVolumeType[SEP] \n", "140907 [CLS]FRDfix[SEP]EntityCollection[SEP] \n", "140908 [CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP] \n", "140909 [CLS]FRDfix[SEP]SurveillanceGroundStation[SEP] \n", "\n", " class_name_embedding_distance \n", "0 0.355953 \n", "1 0.237307 \n", "2 0.454936 \n", "3 0.51522 \n", "4 0.481079 \n", "... ... \n", "140905 0.126923 \n", "140906 0.171529 \n", "140907 -0.071783 \n", "140908 0.071912 \n", "140909 -0.020724 \n", "\n", "[140910 rows x 5 columns]" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "my_feature_df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Step 4 Save the prepared datasets" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "dfs = [my_label_df, my_feature_df]\n", "\n", "combined_dfs = pd.concat(dfs, axis=1)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "dataset_name = config.NAME" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'nasa'" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dataset_name" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...LF_superclasses_overlapLF_properties_overlaplabelxyshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distance
0-1-1-1-1-1-1-1-1-1-1...-1-100091214[CLS]AirportData[SEP]ACDMIrregularity[SEP]0.355953
1-1-1-1-1-1-1-1-1-1-1...-1-1001488[CLS]AirportData[SEP]Aerodrome[SEP]0.237307
2-1-1-1-1-1-1-10-1-1...-1-100231011[CLS]AirportData[SEP]Flight[SEP]0.454936
3-1-1-1-1-1-1-10-1-1...-1-1003112024[CLS]AirportData[SEP]CodeACDMIrregularityType[...0.51522
4-1-1-1-1-1-1-1-1-1-1...-1-1004101315[CLS]AirportData[SEP]ADSBGroundStation[SEP]0.481079
..................................................................
140905-1-1-1-1-1-100-1-1...-1-1015391012930[CLS]FRDfix[SEP]CodeUomVolumeConcentrationType...0.126923
140906-1-1-1-1-1-100-1-1...-1-1015391101717[CLS]FRDfix[SEP]CodeUomVolumeType[SEP]0.171529
140907-1-1-1-1-1-100-1-1...-1-1015391211516[CLS]FRDfix[SEP]EntityCollection[SEP]-0.071783
140908-1-1-1-1-1-100-1-1...-1-1015391312727[CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP]0.071912
140909-1-1-1-1-1-100-1-1...-1-1015391412425[CLS]FRDfix[SEP]SurveillanceGroundStation[SEP]-0.020724
\n", "

140910 rows × 21 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "0 -1 -1 \n", "1 -1 -1 \n", "2 -1 0 \n", "3 -1 0 \n", "4 -1 -1 \n", "... ... ... \n", "140905 0 0 \n", "140906 0 0 \n", "140907 0 0 \n", "140908 0 0 \n", "140909 0 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "0 -1 -1 ... \n", "1 -1 -1 ... \n", "2 -1 -1 ... \n", "3 -1 -1 ... \n", "4 -1 -1 ... \n", "... ... ... ... \n", "140905 -1 -1 ... \n", "140906 -1 -1 ... \n", "140907 -1 -1 ... \n", "140908 -1 -1 ... \n", "140909 -1 -1 ... \n", "\n", " LF_superclasses_overlap LF_properties_overlap label x y \\\n", "0 -1 -1 0 0 0 \n", "1 -1 -1 0 0 1 \n", "2 -1 -1 0 0 2 \n", "3 -1 -1 0 0 3 \n", "4 -1 -1 0 0 4 \n", "... ... ... ... ... ... \n", "140905 -1 -1 0 153 910 \n", "140906 -1 -1 0 153 911 \n", "140907 -1 -1 0 153 912 \n", "140908 -1 -1 0 153 913 \n", "140909 -1 -1 0 153 914 \n", "\n", " shared_word levenshtein_distance hamming_distance \\\n", "0 9 12 14 \n", "1 4 8 8 \n", "2 3 10 11 \n", "3 11 20 24 \n", "4 10 13 15 \n", "... ... ... ... \n", "140905 1 29 30 \n", "140906 0 17 17 \n", "140907 1 15 16 \n", "140908 1 27 27 \n", "140909 1 24 25 \n", "\n", " text_pair \\\n", "0 [CLS]AirportData[SEP]ACDMIrregularity[SEP] \n", "1 [CLS]AirportData[SEP]Aerodrome[SEP] \n", "2 [CLS]AirportData[SEP]Flight[SEP] \n", "3 [CLS]AirportData[SEP]CodeACDMIrregularityType[... \n", "4 [CLS]AirportData[SEP]ADSBGroundStation[SEP] \n", "... ... \n", "140905 [CLS]FRDfix[SEP]CodeUomVolumeConcentrationType... \n", "140906 [CLS]FRDfix[SEP]CodeUomVolumeType[SEP] \n", "140907 [CLS]FRDfix[SEP]EntityCollection[SEP] \n", "140908 [CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP] \n", "140909 [CLS]FRDfix[SEP]SurveillanceGroundStation[SEP] \n", "\n", " class_name_embedding_distance \n", "0 0.355953 \n", "1 0.237307 \n", "2 0.454936 \n", "3 0.51522 \n", "4 0.481079 \n", "... ... \n", "140905 0.126923 \n", "140906 0.171529 \n", "140907 -0.071783 \n", "140908 0.071912 \n", "140909 -0.020724 \n", "\n", "[140910 rows x 21 columns]" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "combined_dfs" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "combined_dfs.to_csv(dataset_name + '.csv', index=False, header=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### save the class profile in both source ontology and target ontology" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "import pickle\n", "\n", "pickle.dump(source_ontology, open(dataset_name + \"_source_ontology.pk\", \"wb\" ))\n", "pickle.dump(target_ontology, open(dataset_name + \"_target_ontology.pk\", \"wb\" ))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...LF_superclasses_overlapLF_properties_overlaplabelxyshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distance
306111-1-1-11111-1...-1-1133161600[CLS]WeatherCondition[SEP]WeatherCondition[SEP]1.0
5442-1-1-1-1-1-1-1-11-1...-1-11586791620[CLS]SurfaceWindCondition[SEP]Wind[SEP]0.7891
5491-1-1-11-1-1-1-1-1-1...-1-1161466[CLS]Airport[SEP]Aerodrome[SEP]0.508479
8752-1-1-1-1-1-1-1-11-1...-1-1195178814[CLS]PhysicalRunway[SEP]Runway[SEP]0.679397
10121-1-1-1-1-1-1-1-1-1-1...-1-11115691315[CLS]NavigationFix[SEP]SignificantPoint[SEP]0.319003
11271-1-1-1-1-1-1-1-11-1...-1-1112291171719[CLS]VisibilityCondition[SEP]HorizontalVisibil...0.810132
14389-1-1-1-1-1-1-1-11-1...-1-1115664241419[CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV...0.88163
15853-1-1-1-1-1-1-1-11-1...-1-111729812816[CLS]AircraftCapacity[SEP]Capacity[SEP]0.825409
23442-1-1-1-1-1-1-10-1-1...-1-112556792530[CLS]AircraftFlow[SEP]OccupancyTrafficMonitori...0.628079
36940-1-1-1-1-1-1-1-11-1...-1-1140340779[CLS]TimeInterval[SEP]TimeRangeType[SEP]0.760346
4496911-11-11111-1...-1-1149134800[CLS]Aircraft[SEP]Aircraft[SEP]1.0
4680311-1-1-11111-1...-1-11511381200[CLS]AircraftType[SEP]AircraftType[SEP]1.0
51377-1-1-1-1-1-1-1-11-1...-1-1156137131013[CLS]AircraftModel[SEP]AircraftMakeModelSeries...0.818485
53695-1-1-1-1-1-1-1-11-1...-1-115862513516[CLS]NavigationAid[SEP]RadioNavigationAid[SEP]0.899368
54340-1-1-1-1-1-1-1-11-1...-1-11593557814[CLS]AircraftEngine[SEP]Engine[SEP]0.898687
58021-1-1-1-1-1-1-1-11-1...-1-1163376101222[CLS]EngineType[SEP]CodeAircraftEngineType[SEP]0.880238
6073511-11-11111-1...-1-1166345700[CLS]Taxiway[SEP]Taxiway[SEP]1.0
63306-1-1-1-1-1-1-1-1-1-1...-1-116917191013[CLS]AirCarrier[SEP]AircraftOperator[SEP]0.614038
6868411-1-1-11111-1...-1-1175591300[CLS]AirspaceLayer[SEP]AirspaceLayer[SEP]1.0
70728-1-1-1-1-1-100-1-1...-1-117727322324[CLS]STAR[SEP]StandardInstrumentArrival[SEP]0.178293
75074-1-1-1-1-1-1-1-11-1...-1-118244141516[CLS]OperationalRunway[SEP]RunwayDirection[SEP]0.835488
86354-1-1-1-1-1-1-1-11-1...-1-1194344844[CLS]DeicingPad[SEP]DeicingArea[SEP]0.339917
8700411-11-11111-1...-1-119579800[CLS]Terminal[SEP]Terminal[SEP]1.0
94605-1-11-1-1-100-1-1...-1-1110336032426[CLS]SID[SEP]StandardInstrumentDeparture[SEP]-0.116515
9734611-11-11111-1...-1-11106356400[CLS]Gate[SEP]Gate[SEP]1.0
104410-1-1-1-1-1-1-1-1-1-1...-1-11114100789[CLS]AirportSpec[SEP]AerodromeSet[SEP]0.415925
10555911-1-1-11111-1...-1-111153341000[CLS]CrewMember[SEP]CrewMember[SEP]1.0
10614211-11-11111-1...-1-111162600[CLS]Flight[SEP]Flight[SEP]1.0
112057-1-1-1-1-1-1-1-11-1...-1-1112242762020[CLS]FlightSpec[SEP]FlightConditionCombination...0.760592
116390-1-1-1-1-1-1-1-11-1...-1-111271851399[CLS]AircraftTrackPoint[SEP]AircraftState[SEP]0.710066
128868-1-1-1-1-1-1-1-11-1...-1-11140768645[CLS]Taxipath[SEP]TaxiRoute[SEP]0.80481
131434-1-1-1-1-1-1-10-1-1...-1-1114358961213[CLS]RelativeFix[SEP]PointReference[SEP]0.449235
\n", "

32 rows × 21 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "3061 1 1 -1 \n", "5442 -1 -1 -1 \n", "5491 -1 -1 -1 \n", "8752 -1 -1 -1 \n", "10121 -1 -1 -1 \n", "11271 -1 -1 -1 \n", "14389 -1 -1 -1 \n", "15853 -1 -1 -1 \n", "23442 -1 -1 -1 \n", "36940 -1 -1 -1 \n", "44969 1 1 -1 \n", "46803 1 1 -1 \n", "51377 -1 -1 -1 \n", "53695 -1 -1 -1 \n", "54340 -1 -1 -1 \n", "58021 -1 -1 -1 \n", "60735 1 1 -1 \n", "63306 -1 -1 -1 \n", "68684 1 1 -1 \n", "70728 -1 -1 -1 \n", "75074 -1 -1 -1 \n", "86354 -1 -1 -1 \n", "87004 1 1 -1 \n", "94605 -1 -1 1 \n", "97346 1 1 -1 \n", "104410 -1 -1 -1 \n", "105559 1 1 -1 \n", "106142 1 1 -1 \n", "112057 -1 -1 -1 \n", "116390 -1 -1 -1 \n", "128868 -1 -1 -1 \n", "131434 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "3061 -1 -1 1 \n", "5442 -1 -1 -1 \n", "5491 1 -1 -1 \n", "8752 -1 -1 -1 \n", "10121 -1 -1 -1 \n", "11271 -1 -1 -1 \n", "14389 -1 -1 -1 \n", "15853 -1 -1 -1 \n", "23442 -1 -1 -1 \n", "36940 -1 -1 -1 \n", "44969 1 -1 1 \n", "46803 -1 -1 1 \n", "51377 -1 -1 -1 \n", "53695 -1 -1 -1 \n", "54340 -1 -1 -1 \n", "58021 -1 -1 -1 \n", "60735 1 -1 1 \n", "63306 -1 -1 -1 \n", "68684 -1 -1 1 \n", "70728 -1 -1 -1 \n", "75074 -1 -1 -1 \n", "86354 -1 -1 -1 \n", "87004 1 -1 1 \n", "94605 -1 -1 -1 \n", "97346 1 -1 1 \n", "104410 -1 -1 -1 \n", "105559 -1 -1 1 \n", "106142 1 -1 1 \n", "112057 -1 -1 -1 \n", "116390 -1 -1 -1 \n", "128868 -1 -1 -1 \n", "131434 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "3061 1 1 \n", "5442 -1 -1 \n", "5491 -1 -1 \n", "8752 -1 -1 \n", "10121 -1 -1 \n", "11271 -1 -1 \n", "14389 -1 -1 \n", "15853 -1 -1 \n", "23442 -1 0 \n", "36940 -1 -1 \n", "44969 1 1 \n", "46803 1 1 \n", "51377 -1 -1 \n", "53695 -1 -1 \n", "54340 -1 -1 \n", "58021 -1 -1 \n", "60735 1 1 \n", "63306 -1 -1 \n", "68684 1 1 \n", "70728 0 0 \n", "75074 -1 -1 \n", "86354 -1 -1 \n", "87004 1 1 \n", "94605 0 0 \n", "97346 1 1 \n", "104410 -1 -1 \n", "105559 1 1 \n", "106142 1 1 \n", "112057 -1 -1 \n", "116390 -1 -1 \n", "128868 -1 -1 \n", "131434 -1 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "3061 1 -1 ... \n", "5442 1 -1 ... \n", "5491 -1 -1 ... \n", "8752 1 -1 ... \n", "10121 -1 -1 ... \n", "11271 1 -1 ... \n", "14389 1 -1 ... \n", "15853 1 -1 ... \n", "23442 -1 -1 ... \n", "36940 1 -1 ... \n", "44969 1 -1 ... \n", "46803 1 -1 ... \n", "51377 1 -1 ... \n", "53695 1 -1 ... \n", "54340 1 -1 ... \n", "58021 1 -1 ... \n", "60735 1 -1 ... \n", "63306 -1 -1 ... \n", "68684 1 -1 ... \n", "70728 -1 -1 ... \n", "75074 1 -1 ... \n", "86354 1 -1 ... \n", "87004 1 -1 ... \n", "94605 -1 -1 ... \n", "97346 1 -1 ... \n", "104410 -1 -1 ... \n", "105559 1 -1 ... \n", "106142 1 -1 ... \n", "112057 1 -1 ... \n", "116390 1 -1 ... \n", "128868 1 -1 ... \n", "131434 -1 -1 ... \n", "\n", " LF_superclasses_overlap LF_properties_overlap label x y \\\n", "3061 -1 -1 1 3 316 \n", "5442 -1 -1 1 5 867 \n", "5491 -1 -1 1 6 1 \n", "8752 -1 -1 1 9 517 \n", "10121 -1 -1 1 11 56 \n", "11271 -1 -1 1 12 291 \n", "14389 -1 -1 1 15 664 \n", "15853 -1 -1 1 17 298 \n", "23442 -1 -1 1 25 567 \n", "36940 -1 -1 1 40 340 \n", "44969 -1 -1 1 49 134 \n", "46803 -1 -1 1 51 138 \n", "51377 -1 -1 1 56 137 \n", "53695 -1 -1 1 58 625 \n", "54340 -1 -1 1 59 355 \n", "58021 -1 -1 1 63 376 \n", "60735 -1 -1 1 66 345 \n", "63306 -1 -1 1 69 171 \n", "68684 -1 -1 1 75 59 \n", "70728 -1 -1 1 77 273 \n", "75074 -1 -1 1 82 44 \n", "86354 -1 -1 1 94 344 \n", "87004 -1 -1 1 95 79 \n", "94605 -1 -1 1 103 360 \n", "97346 -1 -1 1 106 356 \n", "104410 -1 -1 1 114 100 \n", "105559 -1 -1 1 115 334 \n", "106142 -1 -1 1 116 2 \n", "112057 -1 -1 1 122 427 \n", "116390 -1 -1 1 127 185 \n", "128868 -1 -1 1 140 768 \n", "131434 -1 -1 1 143 589 \n", "\n", " shared_word levenshtein_distance hamming_distance \\\n", "3061 16 0 0 \n", "5442 9 16 20 \n", "5491 4 6 6 \n", "8752 8 8 14 \n", "10121 9 13 15 \n", "11271 17 17 19 \n", "14389 24 14 19 \n", "15853 12 8 16 \n", "23442 9 25 30 \n", "36940 7 7 9 \n", "44969 8 0 0 \n", "46803 12 0 0 \n", "51377 13 10 13 \n", "53695 13 5 16 \n", "54340 7 8 14 \n", "58021 10 12 22 \n", "60735 7 0 0 \n", "63306 9 10 13 \n", "68684 13 0 0 \n", "70728 2 23 24 \n", "75074 14 15 16 \n", "86354 8 4 4 \n", "87004 8 0 0 \n", "94605 3 24 26 \n", "97346 4 0 0 \n", "104410 7 8 9 \n", "105559 10 0 0 \n", "106142 6 0 0 \n", "112057 6 20 20 \n", "116390 13 9 9 \n", "128868 6 4 5 \n", "131434 6 12 13 \n", "\n", " text_pair \\\n", "3061 [CLS]WeatherCondition[SEP]WeatherCondition[SEP] \n", "5442 [CLS]SurfaceWindCondition[SEP]Wind[SEP] \n", "5491 [CLS]Airport[SEP]Aerodrome[SEP] \n", "8752 [CLS]PhysicalRunway[SEP]Runway[SEP] \n", "10121 [CLS]NavigationFix[SEP]SignificantPoint[SEP] \n", "11271 [CLS]VisibilityCondition[SEP]HorizontalVisibil... \n", "14389 [CLS]RunwayVisibleRangeMeasurement[SEP]RunwayV... \n", "15853 [CLS]AircraftCapacity[SEP]Capacity[SEP] \n", "23442 [CLS]AircraftFlow[SEP]OccupancyTrafficMonitori... \n", "36940 [CLS]TimeInterval[SEP]TimeRangeType[SEP] \n", "44969 [CLS]Aircraft[SEP]Aircraft[SEP] \n", "46803 [CLS]AircraftType[SEP]AircraftType[SEP] \n", "51377 [CLS]AircraftModel[SEP]AircraftMakeModelSeries... \n", "53695 [CLS]NavigationAid[SEP]RadioNavigationAid[SEP] \n", "54340 [CLS]AircraftEngine[SEP]Engine[SEP] \n", "58021 [CLS]EngineType[SEP]CodeAircraftEngineType[SEP] \n", "60735 [CLS]Taxiway[SEP]Taxiway[SEP] \n", "63306 [CLS]AirCarrier[SEP]AircraftOperator[SEP] \n", "68684 [CLS]AirspaceLayer[SEP]AirspaceLayer[SEP] \n", "70728 [CLS]STAR[SEP]StandardInstrumentArrival[SEP] \n", "75074 [CLS]OperationalRunway[SEP]RunwayDirection[SEP] \n", "86354 [CLS]DeicingPad[SEP]DeicingArea[SEP] \n", "87004 [CLS]Terminal[SEP]Terminal[SEP] \n", "94605 [CLS]SID[SEP]StandardInstrumentDeparture[SEP] \n", "97346 [CLS]Gate[SEP]Gate[SEP] \n", "104410 [CLS]AirportSpec[SEP]AerodromeSet[SEP] \n", "105559 [CLS]CrewMember[SEP]CrewMember[SEP] \n", "106142 [CLS]Flight[SEP]Flight[SEP] \n", "112057 [CLS]FlightSpec[SEP]FlightConditionCombination... \n", "116390 [CLS]AircraftTrackPoint[SEP]AircraftState[SEP] \n", "128868 [CLS]Taxipath[SEP]TaxiRoute[SEP] \n", "131434 [CLS]RelativeFix[SEP]PointReference[SEP] \n", "\n", " class_name_embedding_distance \n", "3061 1.0 \n", "5442 0.7891 \n", "5491 0.508479 \n", "8752 0.679397 \n", "10121 0.319003 \n", "11271 0.810132 \n", "14389 0.88163 \n", "15853 0.825409 \n", "23442 0.628079 \n", "36940 0.760346 \n", "44969 1.0 \n", "46803 1.0 \n", "51377 0.818485 \n", "53695 0.899368 \n", "54340 0.898687 \n", "58021 0.880238 \n", "60735 1.0 \n", "63306 0.614038 \n", "68684 1.0 \n", "70728 0.178293 \n", "75074 0.835488 \n", "86354 0.339917 \n", "87004 1.0 \n", "94605 -0.116515 \n", "97346 1.0 \n", "104410 0.415925 \n", "105559 1.0 \n", "106142 1.0 \n", "112057 0.760592 \n", "116390 0.710066 \n", "128868 0.80481 \n", "131434 0.449235 \n", "\n", "[32 rows x 21 columns]" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "combined_dfs[combined_dfs.label==1]" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'uri': 'https://data.nasa.gov/ontologies/atmonto/data#AirportData',\n", " 'ontology_name': 'http://192.168.0.16:8080/rdf4j-server/repositories/atmonto',\n", " 'class_prefix': 'https://data.nasa.gov/ontologies/atmonto/data#',\n", " 'label': 'Airport Data',\n", " 'class_name': 'AirportData',\n", " 'class_name_root': 'AirportData',\n", " 'class_name_stemmed': 'airportdata',\n", " 'comment': \"Represents a collection of airport data for a given time period, as reported by the FAA's ASPM system.\",\n", " 'subclasses': [],\n", " 'superclasses': ['https://data.nasa.gov/ontologies/atmonto/data#IntervalData',\n", " 'genid-838b259312ed45b4be950d626e6a637b-A86',\n", " 'genid-838b259312ed45b4be950d626e6a637b-A104'],\n", " 'properties': ['etmsArrivals',\n", " 'edctDepartureHold',\n", " 'hasASPMmetCondition',\n", " 'scheduledDepartures',\n", " 'airportArrivalRate',\n", " 'etmsDepartures',\n", " 'scheduledArrivals',\n", " 'totalAirborneDelay',\n", " 'airportDepartureRate',\n", " 'aspmFlightRules',\n", " 'oagGateDepartureDelay',\n", " 'lowCeilingWITIdaily',\n", " 'oagArrivalDelay',\n", " 'lowCeilingWITIhourly',\n", " 'lowVisibilityWITIdaily',\n", " 'highWindWITIhourly',\n", " 'arrivalDemand',\n", " 'edctArrivalHold',\n", " 'lowVisibilityWITIhourly',\n", " 'highWindWITIdaily',\n", " 'departureDemand'],\n", " 'equivalent_classes': [],\n", " 'relevant_synonyms': [],\n", " 'label_words': ['airport', 'data'],\n", " 'label_long': 'airport data',\n", " 'class_name_words': ['airport', 'data'],\n", " 'class_long_name': 'airport data',\n", " 'class_name_embedding': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),\n", " 'class_name_words_embedding': array([ 3.8872697 , 1.10332 , 2.75805 , 3.2408 , 4.118155 ,\n", " 0.05914998, -2.0014 , 5.91364 , 0.43291998, 0.02850008,\n", " 6.964285 , 2.76508 , -5.6597 , 3.16825 , -3.3311498 ,\n", " -0.47694993, 3.2264004 , -1.7696351 , -7.64625 , -4.9125 ,\n", " 1.2530501 , -1.14476 , -7.6109 , -1.3828499 , -5.53065 ,\n", " -4.42645 , -1.8384999 , -3.7674499 , -1.6514001 , 2.547075 ,\n", " 2.86395 , 2.6335 , -4.73281 , 0.15149999, 5.01095 ,\n", " -4.092535 , -5.41155 , -3.604415 , 6.69285 , 3.73 ,\n", " 0.97705 , -2.149265 , 0.31780005, 5.1731 , -4.83665 ,\n", " -0.13426001, 5.3275 , -2.4619 , 1.7277501 , -2.04305 ,\n", " 2.32685 , -0.12824401, -1.6839 , -5.76685 , 0.5084499 ,\n", " -0.47375 , -3.429955 , -1.84322 , 3.85195 , -5.96418 ,\n", " 1.3923001 , 3.1525161 , -3.284985 , 0.28336 , 2.3954 ,\n", " 2.7264252 , -2.31651 , -1.27159 , -0.426045 , 0.751055 ,\n", " 0.06424999, 3.004425 , -2.552985 , 2.0137 , 0.09095001,\n", " 1.9951501 , -0.48444998, -1.434785 , -0.70615005, -3.19635 ,\n", " -1.6653349 , -0.34962 , 0.28144997, 1.275195 , 2.90605 ,\n", " -1.2043 , -1.09017 , -3.64415 , 3.5348 , -3.2254 ,\n", " -1.345 , 5.3385496 , 3.33195 , -0.887355 , -2.2745152 ,\n", " -1.0945649 , 1.6257 , -0.70520496, -0.669125 , 2.56215 ,\n", " 3.0923 , -1.196965 , -1.285355 , -0.50575006, -0.37950498,\n", " 3.391215 , -0.99118 , -1.52287 , 0.41139 , -1.31125 ,\n", " 3.7981 , 0.9962001 , 0.48896202, 0.04940003, 1.78175 ,\n", " 0.616885 , -3.9049702 , -1.47465 , -2.29255 , -5.5384502 ,\n", " -2.1420503 , -2.8541 , -0.48159 , 0.7339 , -1.71474 ,\n", " -3.14525 , 1.5982499 , 0.28479993, 2.2669501 , -3.25095 ,\n", " -0.38144 , 1.9542 , 0.06079996, -0.85580003, 1.0189351 ,\n", " 1.2187 , -2.85803 , 0.0941405 , 3.8074 , 0.18704998,\n", " -4.6354 , -2.9434 , 0.50527 , 1.180725 , 3.21525 ,\n", " 2.43875 , 1.0214901 , 1.4626001 , -3.1556 , 6.96836 ,\n", " -4.63407 , 1.5713 , 4.0911503 , 0.8388001 , 5.30435 ,\n", " -2.17955 , 3.42905 , 4.9851003 , -1.616075 , -2.98285 ,\n", " -1.22919 , -2.33533 , 0.5389205 , 5.731 , -1.82616 ,\n", " 0.79649997, -2.0757 , 2.8527498 , 0.73283994, -5.67275 ,\n", " -8.5309 , -0.5012499 , -1.658585 , 0.7421601 , 8.531724 ,\n", " 2.378785 , -0.94613004, -3.13805 , 1.68705 , -3.32451 ,\n", " 2.8302999 , 3.42525 , 0.17120004, -4.2854247 , 1.4248 ,\n", " 1.23384 , -3.28543 , -1.19362 , 2.3385 , -0.62919503,\n", " 1.3856999 , -1.858255 , -1.60795 , -3.7789001 , 0.42505002,\n", " 7.9492497 , -6.01398 , 0.6017501 , -1.001455 , 0.82453 ,\n", " -3.84375 , -2.7613 , 1.437095 , -2.5948 , 8.381835 ,\n", " 3.07295 , -2.6332002 , -4.89765 , 2.0208 , 0.304285 ,\n", " 0.31060502, -1.7304499 , 1.0133 , 3.9822 , 2.900128 ,\n", " 1.1159899 , 7.63245 , -5.00145 , -3.249 , -0.28030002,\n", " 0.650545 , 1.2922 , -3.45785 , -2.42065 , 3.45816 ,\n", " -6.5847 , 6.1506 , 2.174275 , -0.68729997, -3.74025 ,\n", " 3.4124498 , -5.85595 , -0.97310007, 4.7944 , 1.1766 ,\n", " 0.46455503, 4.9321003 , -0.8756 , 2.6435997 , 0.86618346,\n", " -5.0225 , -1.14609 , -2.2962499 , -1.03565 , -1.229915 ,\n", " 1.3095651 , -4.01855 , 2.6787 , -1.4907 , 1.566925 ,\n", " 2.14371 , -2.97195 , -6.828 , -7.1202 , -1.5709 ,\n", " -1.05247 , 3.1315498 , -3.7287002 , 2.8204498 , 1.6479 ,\n", " -0.8824 , 11.31065 , -2.32835 , -1.21285 , 3.6602 ,\n", " -0.55152 , 0.77307004, 5.60515 , -1.7354 , -6.29965 ,\n", " -2.34847 , 0.315395 , 0.33344996, 0.663005 , 0.37714994,\n", " -4.13295 , 0.90065 , -3.5833502 , -0.6922299 , 2.2624998 ,\n", " 4.19485 , 3.8320498 , 3.07025 , -2.9899 , 2.6443 ,\n", " -2.9198 , -0.26915002, -0.6220545 , -4.3361998 , -1.08202 ,\n", " 0.51065 , -1.3612299 , 5.0004997 , 2.311955 , -1.062025 ,\n", " 3.9514499 , 1.6149001 , -5.8729997 , 4.42145 , 6.59975 ],\n", " dtype=float32),\n", " 'class_long_name_embedding': array([ 3.8872697 , 1.10332 , 2.75805 , 3.2408 , 4.118155 ,\n", " 0.05914998, -2.0014 , 5.91364 , 0.43291998, 0.02850008,\n", " 6.964285 , 2.76508 , -5.6597 , 3.16825 , -3.3311498 ,\n", " -0.47694993, 3.2264004 , -1.7696351 , -7.64625 , -4.9125 ,\n", " 1.2530501 , -1.14476 , -7.6109 , -1.3828499 , -5.53065 ,\n", " -4.42645 , -1.8384999 , -3.7674499 , -1.6514001 , 2.547075 ,\n", " 2.86395 , 2.6335 , -4.73281 , 0.15149999, 5.01095 ,\n", " -4.092535 , -5.41155 , -3.604415 , 6.69285 , 3.73 ,\n", " 0.97705 , -2.149265 , 0.31780005, 5.1731 , -4.83665 ,\n", " -0.13426001, 5.3275 , -2.4619 , 1.7277501 , -2.04305 ,\n", " 2.32685 , -0.12824401, -1.6839 , -5.76685 , 0.5084499 ,\n", " -0.47375 , -3.429955 , -1.84322 , 3.85195 , -5.96418 ,\n", " 1.3923001 , 3.1525161 , -3.284985 , 0.28336 , 2.3954 ,\n", " 2.7264252 , -2.31651 , -1.27159 , -0.426045 , 0.751055 ,\n", " 0.06424999, 3.004425 , -2.552985 , 2.0137 , 0.09095001,\n", " 1.9951501 , -0.48444998, -1.434785 , -0.70615005, -3.19635 ,\n", " -1.6653349 , -0.34962 , 0.28144997, 1.275195 , 2.90605 ,\n", " -1.2043 , -1.09017 , -3.64415 , 3.5348 , -3.2254 ,\n", " -1.345 , 5.3385496 , 3.33195 , -0.887355 , -2.2745152 ,\n", " -1.0945649 , 1.6257 , -0.70520496, -0.669125 , 2.56215 ,\n", " 3.0923 , -1.196965 , -1.285355 , -0.50575006, -0.37950498,\n", " 3.391215 , -0.99118 , -1.52287 , 0.41139 , -1.31125 ,\n", " 3.7981 , 0.9962001 , 0.48896202, 0.04940003, 1.78175 ,\n", " 0.616885 , -3.9049702 , -1.47465 , -2.29255 , -5.5384502 ,\n", " -2.1420503 , -2.8541 , -0.48159 , 0.7339 , -1.71474 ,\n", " -3.14525 , 1.5982499 , 0.28479993, 2.2669501 , -3.25095 ,\n", " -0.38144 , 1.9542 , 0.06079996, -0.85580003, 1.0189351 ,\n", " 1.2187 , -2.85803 , 0.0941405 , 3.8074 , 0.18704998,\n", " -4.6354 , -2.9434 , 0.50527 , 1.180725 , 3.21525 ,\n", " 2.43875 , 1.0214901 , 1.4626001 , -3.1556 , 6.96836 ,\n", " -4.63407 , 1.5713 , 4.0911503 , 0.8388001 , 5.30435 ,\n", " -2.17955 , 3.42905 , 4.9851003 , -1.616075 , -2.98285 ,\n", " -1.22919 , -2.33533 , 0.5389205 , 5.731 , -1.82616 ,\n", " 0.79649997, -2.0757 , 2.8527498 , 0.73283994, -5.67275 ,\n", " -8.5309 , -0.5012499 , -1.658585 , 0.7421601 , 8.531724 ,\n", " 2.378785 , -0.94613004, -3.13805 , 1.68705 , -3.32451 ,\n", " 2.8302999 , 3.42525 , 0.17120004, -4.2854247 , 1.4248 ,\n", " 1.23384 , -3.28543 , -1.19362 , 2.3385 , -0.62919503,\n", " 1.3856999 , -1.858255 , -1.60795 , -3.7789001 , 0.42505002,\n", " 7.9492497 , -6.01398 , 0.6017501 , -1.001455 , 0.82453 ,\n", " -3.84375 , -2.7613 , 1.437095 , -2.5948 , 8.381835 ,\n", " 3.07295 , -2.6332002 , -4.89765 , 2.0208 , 0.304285 ,\n", " 0.31060502, -1.7304499 , 1.0133 , 3.9822 , 2.900128 ,\n", " 1.1159899 , 7.63245 , -5.00145 , -3.249 , -0.28030002,\n", " 0.650545 , 1.2922 , -3.45785 , -2.42065 , 3.45816 ,\n", " -6.5847 , 6.1506 , 2.174275 , -0.68729997, -3.74025 ,\n", " 3.4124498 , -5.85595 , -0.97310007, 4.7944 , 1.1766 ,\n", " 0.46455503, 4.9321003 , -0.8756 , 2.6435997 , 0.86618346,\n", " -5.0225 , -1.14609 , -2.2962499 , -1.03565 , -1.229915 ,\n", " 1.3095651 , -4.01855 , 2.6787 , -1.4907 , 1.566925 ,\n", " 2.14371 , -2.97195 , -6.828 , -7.1202 , -1.5709 ,\n", " -1.05247 , 3.1315498 , -3.7287002 , 2.8204498 , 1.6479 ,\n", " -0.8824 , 11.31065 , -2.32835 , -1.21285 , 3.6602 ,\n", " -0.55152 , 0.77307004, 5.60515 , -1.7354 , -6.29965 ,\n", " -2.34847 , 0.315395 , 0.33344996, 0.663005 , 0.37714994,\n", " -4.13295 , 0.90065 , -3.5833502 , -0.6922299 , 2.2624998 ,\n", " 4.19485 , 3.8320498 , 3.07025 , -2.9899 , 2.6443 ,\n", " -2.9198 , -0.26915002, -0.6220545 , -4.3361998 , -1.08202 ,\n", " 0.51065 , -1.3612299 , 5.0004997 , 2.311955 , -1.062025 ,\n", " 3.9514499 , 1.6149001 , -5.8729997 , 4.42145 , 6.59975 ],\n", " dtype=float32),\n", " 'class_name_sbert_embedding': array([ 0.63339657, -0.39302114, -0.29692358, -0.42592648, 0.61728376,\n", " 0.05914076, 0.08341481, 0.44862336, -0.03790497, 0.31142765,\n", " -0.269141 , -0.3433468 , 0.2642331 , 0.11647485, -0.99174184,\n", " -0.10261895, -0.05516205, -0.21566612, -0.2710152 , -0.4857328 ,\n", " 0.03511691, -0.2726552 , -0.5552488 , 0.02890387, 0.34782562,\n", " 0.46624103, 0.19567138, 0.84041715, 0.13409445, -0.65008414,\n", " 0.01648881, -0.1113698 , 0.6776639 , 0.6942818 , 0.5026263 ,\n", " -0.76597947, 0.41461557, -0.18007001, -0.46671253, 0.14250576,\n", " 0.14239779, -0.74605376, -0.05903619, 0.42762336, 0.25316805,\n", " 0.04766805, 0.2190044 , -0.15500143, -0.10371016, 0.46129015,\n", " 0.09021834, 0.15122923, 0.21357092, 0.53038883, -0.44574916,\n", " -0.31311578, 0.04459583, -0.22953464, -0.13122939, -0.24848488,\n", " -0.25816548, -0.4054286 , -0.12314361, 0.02773553, 0.0103608 ,\n", " 0.34545428, -0.03708348, -0.22789258, -0.10256238, -0.83873475,\n", " -0.619365 , -0.19247323, -0.57089585, 0.23552844, -0.12478512,\n", " 0.13951218, -0.08207695, 0.42320383, 0.17898422, 0.38654083,\n", " 0.27315754, 0.1958994 , -0.39281046, 0.09541574, 0.11850757,\n", " -0.65689003, 0.06506977, 0.05645774, -0.20784652, -0.36136895,\n", " 0.36697543, 0.34417626, 0.2366336 , 0.34813258, -0.17188996,\n", " 0.08397455, 0.07654463, -0.11087623, 0.85453427, 0.87202114,\n", " 0.32736343, 0.54915285, -0.5363212 , 0.3134281 , -0.7709645 ,\n", " -0.19237141, -0.05486321, -0.22105826, 0.55925155, -0.10616927,\n", " -0.7168907 , -0.13053545, -0.14996356, -0.23311797, -0.33923593,\n", " -0.15403211, -0.12197629, 0.9585854 , -0.5504753 , -0.5447017 ,\n", " -0.8742943 , -0.5547004 , 0.80631053, 0.01051934, -0.08632607,\n", " -0.5492959 , 0.32313836, -0.08981904, -0.39975262, -0.12273768,\n", " 0.33178332, -0.02572959, 0.084626 , -1.0392678 , -0.1886953 ,\n", " 0.4785083 , 0.42855337, 0.4577997 , -0.8291347 , 0.6059339 ,\n", " -0.12390832, -0.58641475, 0.6531682 , 0.38416684, 0.09582044,\n", " 0.600641 , -0.23851825, 0.5443171 , 0.34416476, -0.9863741 ,\n", " 0.9096383 , 0.32947496, 1.3437855 , 0.49365538, -0.977696 ,\n", " 0.32706434, 0.5307751 , 0.32284135, 0.20423734, 0.21784897,\n", " -0.37981552, -0.20923209, -0.2772786 , -0.00772125, 0.05943703,\n", " 0.09783422, -0.07353757, -0.16621079, -0.04832871, -0.06044794,\n", " 0.1448007 , -0.2492751 , -0.5701388 , 0.0319992 , -0.21889293,\n", " -0.28627884, 0.9231679 , 0.61940193, -0.30679864, -0.48967898,\n", " -0.91163284, -0.3459286 , 0.05820166, 0.4163258 , -0.07261014,\n", " -0.04837883, 0.2402266 , 0.07903974, -0.41161317, 0.17494854,\n", " 0.04178153, -0.30989295, 0.56634164, -0.29731098, 0.4860654 ,\n", " -0.18748593, 0.41616422, 0.12488022, -0.39766034, 0.46858734,\n", " 0.5078424 , 0.14220127, -0.04677373, -0.3367684 , -0.2007918 ,\n", " 0.22311848, -0.27145028, -0.06051612, -0.45116735, 0.41538554,\n", " 0.69062376, -0.0632478 , 0.34902117, -0.17573205, 0.7755381 ,\n", " 0.19892211, -0.708731 , -0.08299063, -0.6071787 , 0.19563927,\n", " -0.77925074, 0.18536189, 0.51981276, -0.48985705, -0.07168332,\n", " -0.28565466, -0.62296927, -0.3444481 , -0.07882509, 0.05675206,\n", " 0.6345393 , 0.14878003, -0.07368982, 0.3736307 , -0.21667928,\n", " 0.09958442, 0.08177236, 0.08884391, 0.11325274, -0.14085323,\n", " -0.07661641, -0.19607477, -0.34616673, 0.13566801, -0.05386356,\n", " 0.04308141, -0.28419146, -0.37218592, -0.3237766 , 0.3975941 ,\n", " -0.37981004, -0.47719866, -0.19470249, 0.48949358, -0.72257346,\n", " -0.3528576 , 0.56929636, 0.7765528 , -0.1694803 , 0.09053718,\n", " 0.7973823 , -0.27726328, 0.60133463, 0.28627834, 0.49431503,\n", " 0.6150052 , -0.00395475, -0.3001945 , 0.09684713, -0.16381556,\n", " -0.39931434, 0.09876732, 0.08852556, -0.48469466, 0.52771986,\n", " -0.16036694, -0.8477434 , 0.4638075 , 0.23140584, 0.13687187,\n", " -0.2526248 , -0.00748641, 0.36426085, -0.33601314, -0.13161689,\n", " -0.44217056, -0.7786683 , 0.25799894, -0.6875915 , 0.09065026,\n", " 0.20270473, -0.2211974 , -0.00717081, 0.45862004, 0.13451047,\n", " -0.4192033 , 0.41073728, 0.7297728 , -0.1404144 , -0.02578306,\n", " 0.11074343, 0.80340815, -0.14205675, 0.13733041, -0.40364847,\n", " 0.1875724 , 0.28392845, -0.1562859 , 0.90406775, 0.23885721,\n", " 0.2709257 , -0.6075147 , -0.10068362, -0.48757002, -0.15665205,\n", " 0.40234184, -0.23395497, 0.34376955, 0.48095143, -0.22524062,\n", " -0.29921326, 0.13134234, -0.01568425, 0.46884525, 0.33794135,\n", " -0.26347816, 0.03125902, 0.3482942 , -0.2832813 , -0.01969123,\n", " -0.02658848, 0.02980652, -0.5770907 , 0.54971015, 0.3361351 ,\n", " 0.03717348, 0.04646266, 0.5261377 , 0.21831053, 0.25524786,\n", " -0.18730597, 0.01722718, -0.25979906, 0.04332034, 0.8786683 ,\n", " 0.42404333, 0.10574551, 0.4654369 , 0.4963289 , -0.5753607 ,\n", " -0.05672239, -0.40130925, 0.01311014, -0.00654952, -0.2992847 ,\n", " 0.6338353 , 0.4375501 , -0.11424457, -0.52948225, -0.22566655,\n", " 0.4586533 , -0.47360685, -0.8971791 , -0.04664624, 0.69744086,\n", " 0.73476166, -0.4757257 , -0.21832305, -0.27924553, -0.4263652 ,\n", " -0.35620725, -0.36880982, 0.47264075, 0.27777928, 0.17868611,\n", " -0.5537318 , -0.05298463, -0.6159094 , 0.39988527, -0.00989682,\n", " -0.66358465, 0.77562886, -0.22076264, -0.25307304], dtype=float32),\n", " 'label_embedding': array([ 3.8872697 , 1.10332 , 2.75805 , 3.2408 , 4.118155 ,\n", " 0.05914998, -2.0014 , 5.91364 , 0.43291998, 0.02850008,\n", " 6.964285 , 2.76508 , -5.6597 , 3.16825 , -3.3311498 ,\n", " -0.47694993, 3.2264004 , -1.7696351 , -7.64625 , -4.9125 ,\n", " 1.2530501 , -1.14476 , -7.6109 , -1.3828499 , -5.53065 ,\n", " -4.42645 , -1.8384999 , -3.7674499 , -1.6514001 , 2.547075 ,\n", " 2.86395 , 2.6335 , -4.73281 , 0.15149999, 5.01095 ,\n", " -4.092535 , -5.41155 , -3.604415 , 6.69285 , 3.73 ,\n", " 0.97705 , -2.149265 , 0.31780005, 5.1731 , -4.83665 ,\n", " -0.13426001, 5.3275 , -2.4619 , 1.7277501 , -2.04305 ,\n", " 2.32685 , -0.12824401, -1.6839 , -5.76685 , 0.5084499 ,\n", " -0.47375 , -3.429955 , -1.84322 , 3.85195 , -5.96418 ,\n", " 1.3923001 , 3.1525161 , -3.284985 , 0.28336 , 2.3954 ,\n", " 2.7264252 , -2.31651 , -1.27159 , -0.426045 , 0.751055 ,\n", " 0.06424999, 3.004425 , -2.552985 , 2.0137 , 0.09095001,\n", " 1.9951501 , -0.48444998, -1.434785 , -0.70615005, -3.19635 ,\n", " -1.6653349 , -0.34962 , 0.28144997, 1.275195 , 2.90605 ,\n", " -1.2043 , -1.09017 , -3.64415 , 3.5348 , -3.2254 ,\n", " -1.345 , 5.3385496 , 3.33195 , -0.887355 , -2.2745152 ,\n", " -1.0945649 , 1.6257 , -0.70520496, -0.669125 , 2.56215 ,\n", " 3.0923 , -1.196965 , -1.285355 , -0.50575006, -0.37950498,\n", " 3.391215 , -0.99118 , -1.52287 , 0.41139 , -1.31125 ,\n", " 3.7981 , 0.9962001 , 0.48896202, 0.04940003, 1.78175 ,\n", " 0.616885 , -3.9049702 , -1.47465 , -2.29255 , -5.5384502 ,\n", " -2.1420503 , -2.8541 , -0.48159 , 0.7339 , -1.71474 ,\n", " -3.14525 , 1.5982499 , 0.28479993, 2.2669501 , -3.25095 ,\n", " -0.38144 , 1.9542 , 0.06079996, -0.85580003, 1.0189351 ,\n", " 1.2187 , -2.85803 , 0.0941405 , 3.8074 , 0.18704998,\n", " -4.6354 , -2.9434 , 0.50527 , 1.180725 , 3.21525 ,\n", " 2.43875 , 1.0214901 , 1.4626001 , -3.1556 , 6.96836 ,\n", " -4.63407 , 1.5713 , 4.0911503 , 0.8388001 , 5.30435 ,\n", " -2.17955 , 3.42905 , 4.9851003 , -1.616075 , -2.98285 ,\n", " -1.22919 , -2.33533 , 0.5389205 , 5.731 , -1.82616 ,\n", " 0.79649997, -2.0757 , 2.8527498 , 0.73283994, -5.67275 ,\n", " -8.5309 , -0.5012499 , -1.658585 , 0.7421601 , 8.531724 ,\n", " 2.378785 , -0.94613004, -3.13805 , 1.68705 , -3.32451 ,\n", " 2.8302999 , 3.42525 , 0.17120004, -4.2854247 , 1.4248 ,\n", " 1.23384 , -3.28543 , -1.19362 , 2.3385 , -0.62919503,\n", " 1.3856999 , -1.858255 , -1.60795 , -3.7789001 , 0.42505002,\n", " 7.9492497 , -6.01398 , 0.6017501 , -1.001455 , 0.82453 ,\n", " -3.84375 , -2.7613 , 1.437095 , -2.5948 , 8.381835 ,\n", " 3.07295 , -2.6332002 , -4.89765 , 2.0208 , 0.304285 ,\n", " 0.31060502, -1.7304499 , 1.0133 , 3.9822 , 2.900128 ,\n", " 1.1159899 , 7.63245 , -5.00145 , -3.249 , -0.28030002,\n", " 0.650545 , 1.2922 , -3.45785 , -2.42065 , 3.45816 ,\n", " -6.5847 , 6.1506 , 2.174275 , -0.68729997, -3.74025 ,\n", " 3.4124498 , -5.85595 , -0.97310007, 4.7944 , 1.1766 ,\n", " 0.46455503, 4.9321003 , -0.8756 , 2.6435997 , 0.86618346,\n", " -5.0225 , -1.14609 , -2.2962499 , -1.03565 , -1.229915 ,\n", " 1.3095651 , -4.01855 , 2.6787 , -1.4907 , 1.566925 ,\n", " 2.14371 , -2.97195 , -6.828 , -7.1202 , -1.5709 ,\n", " -1.05247 , 3.1315498 , -3.7287002 , 2.8204498 , 1.6479 ,\n", " -0.8824 , 11.31065 , -2.32835 , -1.21285 , 3.6602 ,\n", " -0.55152 , 0.77307004, 5.60515 , -1.7354 , -6.29965 ,\n", " -2.34847 , 0.315395 , 0.33344996, 0.663005 , 0.37714994,\n", " -4.13295 , 0.90065 , -3.5833502 , -0.6922299 , 2.2624998 ,\n", " 4.19485 , 3.8320498 , 3.07025 , -2.9899 , 2.6443 ,\n", " -2.9198 , -0.26915002, -0.6220545 , -4.3361998 , -1.08202 ,\n", " 0.51065 , -1.3612299 , 5.0004997 , 2.311955 , -1.062025 ,\n", " 3.9514499 , 1.6149001 , -5.8729997 , 4.42145 , 6.59975 ],\n", " dtype=float32),\n", " 'label_sbert_embedding': array([ 0.63339657, -0.39302114, -0.29692358, -0.42592648, 0.61728376,\n", " 0.05914076, 0.08341481, 0.44862336, -0.03790497, 0.31142765,\n", " -0.269141 , -0.3433468 , 0.2642331 , 0.11647485, -0.99174184,\n", " -0.10261895, -0.05516205, -0.21566612, -0.2710152 , -0.4857328 ,\n", " 0.03511691, -0.2726552 , -0.5552488 , 0.02890387, 0.34782562,\n", " 0.46624103, 0.19567138, 0.84041715, 0.13409445, -0.65008414,\n", " 0.01648881, -0.1113698 , 0.6776639 , 0.6942818 , 0.5026263 ,\n", " -0.76597947, 0.41461557, -0.18007001, -0.46671253, 0.14250576,\n", " 0.14239779, -0.74605376, -0.05903619, 0.42762336, 0.25316805,\n", " 0.04766805, 0.2190044 , -0.15500143, -0.10371016, 0.46129015,\n", " 0.09021834, 0.15122923, 0.21357092, 0.53038883, -0.44574916,\n", " -0.31311578, 0.04459583, -0.22953464, -0.13122939, -0.24848488,\n", " -0.25816548, -0.4054286 , -0.12314361, 0.02773553, 0.0103608 ,\n", " 0.34545428, -0.03708348, -0.22789258, -0.10256238, -0.83873475,\n", " -0.619365 , -0.19247323, -0.57089585, 0.23552844, -0.12478512,\n", " 0.13951218, -0.08207695, 0.42320383, 0.17898422, 0.38654083,\n", " 0.27315754, 0.1958994 , -0.39281046, 0.09541574, 0.11850757,\n", " -0.65689003, 0.06506977, 0.05645774, -0.20784652, -0.36136895,\n", " 0.36697543, 0.34417626, 0.2366336 , 0.34813258, -0.17188996,\n", " 0.08397455, 0.07654463, -0.11087623, 0.85453427, 0.87202114,\n", " 0.32736343, 0.54915285, -0.5363212 , 0.3134281 , -0.7709645 ,\n", " -0.19237141, -0.05486321, -0.22105826, 0.55925155, -0.10616927,\n", " -0.7168907 , -0.13053545, -0.14996356, -0.23311797, -0.33923593,\n", " -0.15403211, -0.12197629, 0.9585854 , -0.5504753 , -0.5447017 ,\n", " -0.8742943 , -0.5547004 , 0.80631053, 0.01051934, -0.08632607,\n", " -0.5492959 , 0.32313836, -0.08981904, -0.39975262, -0.12273768,\n", " 0.33178332, -0.02572959, 0.084626 , -1.0392678 , -0.1886953 ,\n", " 0.4785083 , 0.42855337, 0.4577997 , -0.8291347 , 0.6059339 ,\n", " -0.12390832, -0.58641475, 0.6531682 , 0.38416684, 0.09582044,\n", " 0.600641 , -0.23851825, 0.5443171 , 0.34416476, -0.9863741 ,\n", " 0.9096383 , 0.32947496, 1.3437855 , 0.49365538, -0.977696 ,\n", " 0.32706434, 0.5307751 , 0.32284135, 0.20423734, 0.21784897,\n", " -0.37981552, -0.20923209, -0.2772786 , -0.00772125, 0.05943703,\n", " 0.09783422, -0.07353757, -0.16621079, -0.04832871, -0.06044794,\n", " 0.1448007 , -0.2492751 , -0.5701388 , 0.0319992 , -0.21889293,\n", " -0.28627884, 0.9231679 , 0.61940193, -0.30679864, -0.48967898,\n", " -0.91163284, -0.3459286 , 0.05820166, 0.4163258 , -0.07261014,\n", " -0.04837883, 0.2402266 , 0.07903974, -0.41161317, 0.17494854,\n", " 0.04178153, -0.30989295, 0.56634164, -0.29731098, 0.4860654 ,\n", " -0.18748593, 0.41616422, 0.12488022, -0.39766034, 0.46858734,\n", " 0.5078424 , 0.14220127, -0.04677373, -0.3367684 , -0.2007918 ,\n", " 0.22311848, -0.27145028, -0.06051612, -0.45116735, 0.41538554,\n", " 0.69062376, -0.0632478 , 0.34902117, -0.17573205, 0.7755381 ,\n", " 0.19892211, -0.708731 , -0.08299063, -0.6071787 , 0.19563927,\n", " -0.77925074, 0.18536189, 0.51981276, -0.48985705, -0.07168332,\n", " -0.28565466, -0.62296927, -0.3444481 , -0.07882509, 0.05675206,\n", " 0.6345393 , 0.14878003, -0.07368982, 0.3736307 , -0.21667928,\n", " 0.09958442, 0.08177236, 0.08884391, 0.11325274, -0.14085323,\n", " -0.07661641, -0.19607477, -0.34616673, 0.13566801, -0.05386356,\n", " 0.04308141, -0.28419146, -0.37218592, -0.3237766 , 0.3975941 ,\n", " -0.37981004, -0.47719866, -0.19470249, 0.48949358, -0.72257346,\n", " -0.3528576 , 0.56929636, 0.7765528 , -0.1694803 , 0.09053718,\n", " 0.7973823 , -0.27726328, 0.60133463, 0.28627834, 0.49431503,\n", " 0.6150052 , -0.00395475, -0.3001945 , 0.09684713, -0.16381556,\n", " -0.39931434, 0.09876732, 0.08852556, -0.48469466, 0.52771986,\n", " -0.16036694, -0.8477434 , 0.4638075 , 0.23140584, 0.13687187,\n", " -0.2526248 , -0.00748641, 0.36426085, -0.33601314, -0.13161689,\n", " -0.44217056, -0.7786683 , 0.25799894, -0.6875915 , 0.09065026,\n", " 0.20270473, -0.2211974 , -0.00717081, 0.45862004, 0.13451047,\n", " -0.4192033 , 0.41073728, 0.7297728 , -0.1404144 , -0.02578306,\n", " 0.11074343, 0.80340815, -0.14205675, 0.13733041, -0.40364847,\n", " 0.1875724 , 0.28392845, -0.1562859 , 0.90406775, 0.23885721,\n", " 0.2709257 , -0.6075147 , -0.10068362, -0.48757002, -0.15665205,\n", " 0.40234184, -0.23395497, 0.34376955, 0.48095143, -0.22524062,\n", " -0.29921326, 0.13134234, -0.01568425, 0.46884525, 0.33794135,\n", " -0.26347816, 0.03125902, 0.3482942 , -0.2832813 , -0.01969123,\n", " -0.02658848, 0.02980652, -0.5770907 , 0.54971015, 0.3361351 ,\n", " 0.03717348, 0.04646266, 0.5261377 , 0.21831053, 0.25524786,\n", " -0.18730597, 0.01722718, -0.25979906, 0.04332034, 0.8786683 ,\n", " 0.42404333, 0.10574551, 0.4654369 , 0.4963289 , -0.5753607 ,\n", " -0.05672239, -0.40130925, 0.01311014, -0.00654952, -0.2992847 ,\n", " 0.6338353 , 0.4375501 , -0.11424457, -0.52948225, -0.22566655,\n", " 0.4586533 , -0.47360685, -0.8971791 , -0.04664624, 0.69744086,\n", " 0.73476166, -0.4757257 , -0.21832305, -0.27924553, -0.4263652 ,\n", " -0.35620725, -0.36880982, 0.47264075, 0.27777928, 0.17868611,\n", " -0.5537318 , -0.05298463, -0.6159094 , 0.39988527, -0.00989682,\n", " -0.66358465, 0.77562886, -0.22076264, -0.25307304], dtype=float32),\n", " 'comment_embedding': array([-3.1235311e+00, -6.2641823e-01, -9.1777903e-01, 6.6366911e-01,\n", " 6.7096014e+00, -9.8332578e-01, -7.6024109e-03, 4.3654003e+00,\n", " 1.8946286e+00, -2.1687596e+00, 7.6567507e+00, 1.3147062e+00,\n", " -4.0060573e+00, 7.8798968e-01, -7.0517176e-01, 3.6798685e+00,\n", " 2.3280156e+00, 1.6700238e+00, -1.6982446e+00, -1.4798963e+00,\n", " 1.4014966e+00, -1.7294805e+00, -3.4126122e+00, 5.9145963e-01,\n", " 1.9582546e-01, -2.3274577e+00, -2.3967662e+00, -7.2884953e-01,\n", " 1.3682428e-01, 1.7368242e-01, -3.3303335e-01, -1.4086194e-01,\n", " -1.0770761e-01, -1.3837900e+00, -2.6203225e+00, -8.2150048e-01,\n", " -7.9093951e-01, 6.7973709e-01, 1.7330537e+00, -5.0761324e-01,\n", " -4.2138812e-01, -1.2905772e+00, -1.5189879e+00, 7.0414078e-01,\n", " -2.1925912e+00, -1.6727471e-01, 1.2245990e+00, -1.4555693e+00,\n", " -4.8719287e-01, 3.5685089e-01, -1.5997418e+00, 3.0523982e+00,\n", " 8.0250204e-01, -4.6749802e+00, 3.7623438e-01, 3.0860764e-01,\n", " -8.7774539e-01, 7.4521285e-01, 1.0349742e+00, -2.6942067e+00,\n", " 4.8136574e-01, -1.0936507e+00, -2.3494098e-01, -1.2085283e+00,\n", " 2.2652385e+00, 1.7127023e+00, -1.3944852e+00, -4.1635880e+00,\n", " 7.5139707e-01, 2.0768631e+00, -2.6537827e-01, -3.3795282e-01,\n", " -3.2378166e+00, -7.2390974e-01, -1.7592523e-01, 2.0517840e+00,\n", " -2.4928710e+00, 2.4204123e+00, -3.2807667e+00, -9.3920720e-01,\n", " -5.1284003e+00, -2.5561273e-01, -5.9892666e-02, 5.8225703e-01,\n", " 2.8958714e+00, -2.9039320e-01, -3.5679510e+00, -4.0155945e+00,\n", " 2.1362498e+00, -1.2962682e+00, -1.4708369e+00, -1.8040888e+00,\n", " 2.7702494e+00, -3.3191402e+00, 5.5266875e-01, -3.1595848e+00,\n", " 9.9235046e-01, -6.9304234e-01, 5.9999987e-02, 2.5537038e+00,\n", " 2.4951112e+00, 1.1085443e+00, 2.2621260e+00, 4.8291726e+00,\n", " 6.2051600e-01, 5.2768731e+00, 2.2060280e+00, -2.7161098e+00,\n", " 1.1003834e+00, -3.1589570e+00, 1.6197687e+00, 2.8824306e+00,\n", " -2.4420626e+00, 1.3332965e+00, 6.8405050e-01, -6.9145864e-01,\n", " -1.4728889e+00, 1.4714416e+00, -4.3192420e-02, -1.5192963e+00,\n", " -2.2426042e-01, -3.8316269e+00, 1.7132162e+00, 2.9790461e+00,\n", " -1.5573308e+00, -5.1733799e+00, 3.9889771e-01, -2.2204823e+00,\n", " 4.2551069e+00, -2.5243032e+00, -2.9238727e+00, 1.8205049e+00,\n", " 3.7752185e+00, -2.2092104e+00, 4.9695423e-01, 1.5378567e+00,\n", " -4.1579479e-01, -1.7543180e+00, 2.9759145e+00, -2.4899774e+00,\n", " -2.4074807e+00, 6.3154292e-01, 8.9885241e-01, 2.2741601e+00,\n", " 1.8088964e+00, 1.6393734e+00, -2.4969418e+00, -1.1203551e+00,\n", " 2.1245930e+00, 4.4503517e+00, 5.3108656e-01, 1.9560076e+00,\n", " 6.9133103e-01, 3.0083995e+00, -2.7353424e-01, 1.1692781e+00,\n", " 2.6916230e+00, 8.1193042e-01, -2.2538271e+00, -5.3482229e-01,\n", " -7.8222901e-02, -2.2012942e+00, 1.6033155e+00, 3.0167358e+00,\n", " -2.0855687e+00, -1.4463428e+00, -4.8709998e+00, 3.3148110e+00,\n", " -4.7322524e-01, 7.0275104e-01, 1.1366441e+00, -1.2790984e+00,\n", " 2.0749874e+00, 7.1906328e-01, 2.1887670e+00, 9.2131233e-01,\n", " 8.9056671e-01, -1.3747220e+00, -2.9495003e+00, -1.4694353e+00,\n", " -1.9645313e+00, 8.9389199e-01, 1.2837757e+00, -6.7687935e-01,\n", " -5.9232950e-01, 1.0577616e-02, -1.9587312e+00, 8.5719436e-01,\n", " -4.8119617e-01, 2.2010546e+00, -4.8668915e-01, -2.0765519e+00,\n", " -1.0621316e+00, -1.7796953e+00, 2.0371218e+00, 1.4685282e+00,\n", " -2.9117608e+00, -2.0530579e+00, 9.6484321e-01, -8.5509501e-02,\n", " -7.8603625e-01, -2.7962337e+00, -2.5234485e-01, -3.0683110e+00,\n", " 5.1088767e+00, 2.2671714e+00, -4.3876204e+00, 2.3505979e+00,\n", " -6.8588561e-01, -2.0154607e-01, 8.8478249e-01, 1.5177824e+00,\n", " -1.9997584e+00, 2.6812947e+00, 1.5839319e+00, 1.5272602e+00,\n", " 1.6032099e+00, -3.2813082e+00, -1.3515104e+00, 1.5125763e-01,\n", " -1.6988553e+00, 1.6867394e+00, -1.2496831e+00, 9.4101286e-01,\n", " -1.7026720e+00, -1.7605956e+00, 1.9165274e+00, 2.1179852e+00,\n", " 1.2734061e+00, -9.5148247e-01, 4.3450890e+00, -5.1429911e+00,\n", " -1.4108771e+00, 4.6207056e+00, 2.2142348e+00, 1.5393038e+00,\n", " -1.1627586e+00, 8.4333855e-01, -1.6614653e+00, -1.2250688e+00,\n", " 3.8895345e-01, 1.1522247e+00, 1.4463414e+00, 7.6969528e-01,\n", " -1.6113783e+00, 1.8144712e+00, -4.7359762e+00, 1.2052358e+00,\n", " -5.8364660e-01, 3.2489507e+00, 2.5914860e+00, -4.0383244e+00,\n", " -5.6808457e+00, -1.5370265e+00, 8.4648587e-02, -1.8980671e+00,\n", " 3.2137423e+00, -2.9712394e-02, 1.4618889e+00, 4.7517046e-01,\n", " -1.0719572e+00, 5.6585183e+00, 3.3233330e+00, 2.4065857e+00,\n", " 8.8958614e-02, -2.6223817e-01, 9.3772423e-01, 1.8141975e+00,\n", " -1.8869452e+00, 3.0077338e-01, 7.4036193e-01, 1.2841139e+00,\n", " 6.2391192e-01, -1.8952520e+00, 1.5314109e+00, 6.6608989e-01,\n", " 2.5799692e+00, 6.3836679e-02, -9.0784490e-01, 4.6786621e-01,\n", " 2.6785402e+00, 1.0823452e+00, 7.9211581e-01, 1.3766676e+00,\n", " 2.4112036e+00, -1.5765696e+00, 5.9528720e-01, 1.4124506e+00,\n", " -2.2028403e+00, 1.3233331e+00, -1.3082130e-01, 4.1014358e-01,\n", " -1.7176570e-01, 1.8537472e+00, -1.1396093e+00, 3.2195110e+00,\n", " 1.3729550e-01, -2.7871208e+00, -2.3212919e-01, 1.9306859e+00],\n", " dtype=float32),\n", " 'comment_sbert_embedding': array([ 0.32550743, -0.19105907, -0.23646846, -0.23789929, 0.18831056,\n", " 0.20503958, -0.16481753, 0.10666469, -0.21791172, 0.16017315,\n", " -0.14445841, -0.1503527 , 0.11283708, 0.01801799, -0.5594964 ,\n", " 0.09627727, 0.3102129 , -0.45004857, 0.11560185, -0.2741976 ,\n", " 0.5536808 , 0.04205903, -0.5746004 , 0.06593198, 0.16042602,\n", " 0.36250144, -0.07206302, 0.13826355, 0.03135262, -0.18212718,\n", " -0.05877034, 0.13921337, 0.25195247, 0.26052013, -0.05298635,\n", " -0.37899056, 0.20674118, 0.0282355 , -0.21095791, 0.20348635,\n", " 0.06267516, -0.25651452, -0.20349687, 0.19172609, -0.15949786,\n", " -0.00801379, -0.03056532, -0.06362134, -0.4105994 , 0.36621678,\n", " -0.08171872, 0.08692051, 0.26958096, 0.19666286, -0.19175534,\n", " -0.0166069 , 0.11355569, -0.45373535, -0.19866692, -0.2716442 ,\n", " -0.14303498, -0.17037964, -0.11412176, 0.09751307, 0.2315443 ,\n", " 0.3091904 , -0.00362976, -0.17680359, 0.20185047, -0.02131914,\n", " -0.53860915, 0.14509243, -0.20896965, 0.18995872, -0.16954087,\n", " 0.26275107, 0.05736205, 0.14556536, 0.16605836, -0.04429451,\n", " 0.22263452, 0.360374 , -0.2629318 , 0.02098002, 0.19512631,\n", " -0.2123498 , -0.25814864, 0.17893772, -0.3794928 , 0.08013721,\n", " -0.07416105, 0.09055826, 0.19699818, -0.00893133, 0.0638694 ,\n", " 0.20512179, 0.2166729 , 0.01499518, 0.8002719 , -0.09425446,\n", " -0.17164859, 0.1888151 , -0.6081693 , -0.18922903, -0.24836995,\n", " -0.18319863, -0.23197453, -0.17944498, 0.27760705, 0.01367454,\n", " -0.46270362, 0.20753315, -0.15632202, -0.18507849, -0.08494152,\n", " -0.01718836, -0.2928301 , 0.44124904, -0.08860872, -0.05662508,\n", " -0.21991153, -0.07264362, 0.25344297, -0.02409109, 0.09397708,\n", " -0.01897902, 0.32616234, 0.03962844, -0.14276665, -0.3586229 ,\n", " 0.07273702, -0.20425697, 0.17648368, -0.6382814 , 0.2515223 ,\n", " 0.3218882 , 0.42047828, 0.08592252, -0.29981828, 0.26566836,\n", " 0.04094225, -0.12895645, 0.4147848 , 0.04148773, -0.07516924,\n", " 0.5592388 , 0.02929411, 0.1093744 , 0.07409364, -0.61231154,\n", " 0.36270785, 0.33691812, 0.4478113 , -0.05630169, -0.30789745,\n", " 0.23270221, 0.20380074, 0.01828348, 0.03933623, 0.26292613,\n", " -0.5551712 , 0.01407553, -0.16812992, 0.04447833, 0.15926751,\n", " 0.09445631, 0.25777987, -0.11816811, 0.06575045, 0.03275537,\n", " -0.0106766 , -0.2949281 , -0.25494096, 0.09501576, -0.04016197,\n", " -0.12420338, 0.2711013 , 0.28763154, 0.00523555, -0.05808796,\n", " -0.2950354 , -0.23314887, 0.04056858, 0.20205112, -0.09546056,\n", " -0.26615584, -0.00523098, 0.35008076, 0.32759872, -0.24325484,\n", " 0.43051994, -0.11585499, 0.15439485, -0.22843257, 0.33295095,\n", " -0.14714062, 0.47712868, 0.17352802, 0.00282932, 0.21467976,\n", " 0.06583311, 0.18242592, -0.40941992, 0.06839571, 0.06656072,\n", " 0.2980853 , -0.17632385, 0.19427213, -0.3954867 , 0.16800974,\n", " 0.32543412, 0.10566959, 0.22861966, -0.11589579, 0.38436198,\n", " 0.0905811 , -0.41171673, -0.17119522, -0.69786125, 0.01326227,\n", " -0.2113813 , -0.07150917, 0.11818007, -0.35347998, 0.05564011,\n", " -0.12656859, -0.10784784, -0.07840031, 0.01688592, -0.0459452 ,\n", " 0.19253351, 0.29338676, -0.04559649, 0.15496226, -0.20183453,\n", " 0.13792792, -0.2195652 , -0.14517736, -0.06194927, -0.17837203,\n", " -0.19346784, -0.28533223, -0.10037997, 0.03131716, 0.03350631,\n", " -0.0669862 , -0.03836739, -0.2928375 , -0.09523481, 0.34587753,\n", " -0.34545133, -0.37060204, -0.15343222, 0.06207243, -0.46817797,\n", " -0.4318202 , -0.15387617, 0.16843906, 0.07484666, -0.34462944,\n", " 0.38029778, -0.13241883, 0.40874738, 0.19930409, 0.39331597,\n", " 0.53935564, 0.04032796, -0.18001209, 0.19756097, -0.05717468,\n", " -0.46672285, 0.11357419, 0.16008145, -0.12339619, 0.23337385,\n", " -0.26803428, -0.19210549, 0.21170945, -0.01346762, -0.01718632,\n", " 0.1047542 , -0.25702745, 0.31302074, -0.20653866, -0.05209504,\n", " -0.18097042, -0.3405809 , 0.18618149, -0.08587341, 0.060531 ,\n", " 0.20678887, -0.31573895, -0.06899507, 0.29410753, 0.29175156,\n", " -0.18107477, -0.20722073, 0.5359958 , -0.17073412, -0.1289956 ,\n", " -0.07334364, 0.22835426, 0.05142631, 0.1351703 , -0.5252999 ,\n", " 0.09298443, 0.3261762 , 0.29387897, 0.09168253, 0.02212305,\n", " 0.36821774, -0.30793256, 0.02775574, -0.31078887, -0.05113143,\n", " 0.15165113, -0.46862754, 0.36753386, 0.28799912, -0.17484166,\n", " -0.26618913, -0.13855143, 0.17552446, 0.09318021, 0.36426023,\n", " -0.01749743, 0.10017347, 0.07606367, 0.25597668, -0.26279932,\n", " 0.26270592, -0.19487417, -0.07737736, -0.00209725, 0.25382972,\n", " 0.00351293, -0.31518945, 0.2254403 , -0.04965837, 0.11587984,\n", " -0.17025527, 0.16409867, -0.19047119, 0.2535907 , 0.34987944,\n", " 0.24991554, 0.20165932, 0.63635164, 0.27396402, -0.18605305,\n", " -0.01445364, 0.27673545, 0.05950854, -0.18255399, 0.00956589,\n", " -0.07515004, 0.2241886 , -0.25515217, -0.10669416, 0.143248 ,\n", " 0.1445124 , -0.18351448, -0.29054335, 0.07304002, 0.22713791,\n", " 0.06607918, -0.43033168, 0.2358682 , -0.01702075, -0.31435737,\n", " -0.13094059, -0.05917815, 0.20482849, 0.15127495, 0.13617414,\n", " -0.01283419, 0.02821784, -0.5914434 , 0.46288624, -0.27023545,\n", " -0.20863467, 0.44233108, -0.10389552, -0.01659838], dtype=float32),\n", " 'walk_embedding': None}" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "source_ontology[0].__dict__" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LF_class_name_equalLF_class_name_stemmed_equalLF_acronymsLF_class_name_synonymsLF_label_equalLF_root_nouns_equalLF_class_name_spacy_distanceLF_class_name_distanceLF_name_segment_overlapLF_label_words_overlap...LF_superclasses_overlapLF_properties_overlaplabelxyshared_wordlevenshtein_distancehamming_distancetext_pairclass_name_embedding_distance
0-1-1-1-1-1-1-1-1-1-1...-1-100091214[CLS]AirportData[SEP]ACDMIrregularity[SEP]0.355953
1-1-1-1-1-1-1-1-1-1-1...-1-1001488[CLS]AirportData[SEP]Aerodrome[SEP]0.237307
2-1-1-1-1-1-1-10-1-1...-1-100231011[CLS]AirportData[SEP]Flight[SEP]0.454936
3-1-1-1-1-1-1-10-1-1...-1-1003112024[CLS]AirportData[SEP]CodeACDMIrregularityType[...0.51522
4-1-1-1-1-1-1-1-1-1-1...-1-1004101315[CLS]AirportData[SEP]ADSBGroundStation[SEP]0.481079
..................................................................
140905-1-1-1-1-1-100-1-1...-1-1015391012930[CLS]FRDfix[SEP]CodeUomVolumeConcentrationType...0.126923
140906-1-1-1-1-1-100-1-1...-1-1015391101717[CLS]FRDfix[SEP]CodeUomVolumeType[SEP]0.171529
140907-1-1-1-1-1-100-1-1...-1-1015391211516[CLS]FRDfix[SEP]EntityCollection[SEP]-0.071783
140908-1-1-1-1-1-100-1-1...-1-1015391312727[CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP]0.071912
140909-1-1-1-1-1-100-1-1...-1-1015391412425[CLS]FRDfix[SEP]SurveillanceGroundStation[SEP]-0.020724
\n", "

140910 rows × 21 columns

\n", "
" ], "text/plain": [ " LF_class_name_equal LF_class_name_stemmed_equal LF_acronyms \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_synonyms LF_label_equal LF_root_nouns_equal \\\n", "0 -1 -1 -1 \n", "1 -1 -1 -1 \n", "2 -1 -1 -1 \n", "3 -1 -1 -1 \n", "4 -1 -1 -1 \n", "... ... ... ... \n", "140905 -1 -1 -1 \n", "140906 -1 -1 -1 \n", "140907 -1 -1 -1 \n", "140908 -1 -1 -1 \n", "140909 -1 -1 -1 \n", "\n", " LF_class_name_spacy_distance LF_class_name_distance \\\n", "0 -1 -1 \n", "1 -1 -1 \n", "2 -1 0 \n", "3 -1 0 \n", "4 -1 -1 \n", "... ... ... \n", "140905 0 0 \n", "140906 0 0 \n", "140907 0 0 \n", "140908 0 0 \n", "140909 0 0 \n", "\n", " LF_name_segment_overlap LF_label_words_overlap ... \\\n", "0 -1 -1 ... \n", "1 -1 -1 ... \n", "2 -1 -1 ... \n", "3 -1 -1 ... \n", "4 -1 -1 ... \n", "... ... ... ... \n", "140905 -1 -1 ... \n", "140906 -1 -1 ... \n", "140907 -1 -1 ... \n", "140908 -1 -1 ... \n", "140909 -1 -1 ... \n", "\n", " LF_superclasses_overlap LF_properties_overlap label x y \\\n", "0 -1 -1 0 0 0 \n", "1 -1 -1 0 0 1 \n", "2 -1 -1 0 0 2 \n", "3 -1 -1 0 0 3 \n", "4 -1 -1 0 0 4 \n", "... ... ... ... ... ... \n", "140905 -1 -1 0 153 910 \n", "140906 -1 -1 0 153 911 \n", "140907 -1 -1 0 153 912 \n", "140908 -1 -1 0 153 913 \n", "140909 -1 -1 0 153 914 \n", "\n", " shared_word levenshtein_distance hamming_distance \\\n", "0 9 12 14 \n", "1 4 8 8 \n", "2 3 10 11 \n", "3 11 20 24 \n", "4 10 13 15 \n", "... ... ... ... \n", "140905 1 29 30 \n", "140906 0 17 17 \n", "140907 1 15 16 \n", "140908 1 27 27 \n", "140909 1 24 25 \n", "\n", " text_pair \\\n", "0 [CLS]AirportData[SEP]ACDMIrregularity[SEP] \n", "1 [CLS]AirportData[SEP]Aerodrome[SEP] \n", "2 [CLS]AirportData[SEP]Flight[SEP] \n", "3 [CLS]AirportData[SEP]CodeACDMIrregularityType[... \n", "4 [CLS]AirportData[SEP]ADSBGroundStation[SEP] \n", "... ... \n", "140905 [CLS]FRDfix[SEP]CodeUomVolumeConcentrationType... \n", "140906 [CLS]FRDfix[SEP]CodeUomVolumeType[SEP] \n", "140907 [CLS]FRDfix[SEP]EntityCollection[SEP] \n", "140908 [CLS]FRDfix[SEP]LongitudinalSeparationMinima[SEP] \n", "140909 [CLS]FRDfix[SEP]SurveillanceGroundStation[SEP] \n", "\n", " class_name_embedding_distance \n", "0 0.355953 \n", "1 0.237307 \n", "2 0.454936 \n", "3 0.51522 \n", "4 0.481079 \n", "... ... \n", "140905 0.126923 \n", "140906 0.171529 \n", "140907 -0.071783 \n", "140908 0.071912 \n", "140909 -0.020724 \n", "\n", "[140910 rows x 21 columns]" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "combined_dfs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 4 }