# # dualloop # # File: data_preparation.py # # Authors: Deleted for purposes of anonymity # # Proprietor: Deleted for purposes of anonymity --- PROPRIETARY INFORMATION # # The software and its source code contain valuable trade secrets and shall be maintained in # confidence and treated as confidential information. The software may only be used for # evaluation and/or testing purposes, unless otherwise explicitly stated in the terms of a # license agreement or nondisclosure agreement with the proprietor of the software. # Any unauthorized publication, transfer to third parties, or duplication of the object or # source code---either totally or in part---is strictly prohibited. # # Copyright (c) 2019 Proprietor: Deleted for purposes of anonymity # All Rights Reserved. # # THE PROPRIETOR DISCLAIMS ALL WARRANTIES, EITHER EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS FOR A PARTICULAR PURPOSE AND THE WARRANTY AGAINST LATENT # DEFECTS, WITH RESPECT TO THE PROGRAM AND ANY ACCOMPANYING DOCUMENTATION. # # NO LIABILITY FOR CONSEQUENTIAL DAMAGES: # IN NO EVENT SHALL THE PROPRIETOR OR ANY OF ITS SUBSIDIARIES BE # LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES # FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF INFORMATION, OR # OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, INCIDENTAL, # ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF OR INABILITY # TO USE THIS PROGRAM, EVEN IF the proprietor HAS BEEN ADVISED OF # THE POSSIBILITY OF SUCH DAMAGES. # # For purposes of anonymity, the identity of the proprietor is not given herewith. # The identity of the proprietor will be given once the review of the # conference submission is completed. # # THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. # import json import os import pickle import sys import time import pandas as pd import preprocessing as preprocess from snorkel.labeling import labeling_function from snorkel.labeling import PandasLFApplier import nltk nltk.download('wordnet') from fuzzywuzzy import fuzz import re import wordsegment wordsegment.load() from scipy import spatial import numpy as np import Levenshtein import textdistance ABSTAIN = -1 UNMATCHED = 0 MATCHED = 1 def read_config(config_file): with open(config_file, 'r') as f: config = json.load(f) return config # global list and map for ontologies source_ontology = [] target_ontology = [] sourceMap = {} targetMap = {} def prepare_alignment_profile(rdf4j_server, dataset): rdf4j_server_url = rdf4j_server + "/rdf4j-server/repositories/" # add one ontology into the source_ontology repository_name = rdf4j_server_url + dataset['source'] alignment_profiles = preprocess.build_alignment_profile(repository_name, False) for profile in alignment_profiles: uri = profile.uri.lower() if uri not in sourceMap: sourceMap[uri] = profile source_ontology.append(profile) # add one ontology into the target_ontology repository_name = rdf4j_server_url + dataset['target'] alignment_profiles = preprocess.build_alignment_profile(repository_name, False) for profile in alignment_profiles: uri = profile.uri.lower() if uri not in targetMap: targetMap[uri] = profile target_ontology.append(profile) print("# of classess in the source ontology: ", len(source_ontology)) print("# of classess in the target ontology: ", len(target_ontology)) def init_dataframe_with_all_combinations(): items = [] begin_x = 0 begin_y = 0 for i in range(len(source_ontology)): for j in range(len(target_ontology)): x = begin_x + i y = begin_y + j items.append({'x': x, 'y': y}) cols = ['x', 'y'] return pd.DataFrame(items, columns=cols) def get_class_index(uri, clist): for idx in range(len(clist)): if clist[idx].uri.lower() == uri.lower(): return idx return -1 def load_ground_truth(df_all, datasets, dsname): # load the gold labels df_all['label'] = 0 ground_truth = [] fix_uri = dsname.startswith('conference') dataset = datasets[dsname] matches = preprocess.get_groundtruth(dataset['groundtruth'], fix_uri) print(len(matches)) ground_truth = ground_truth + matches for match in ground_truth: s_URI = match['source_class'] t_URI = match['target_class'] # find the location of class in the class list of source ontology and target ontology sIdx = get_class_index(s_URI, source_ontology) tIdx = get_class_index(t_URI, target_ontology) if (sIdx < 0): print("SOURCE_CLASS MISSING:" + s_URI) if (tIdx < 0): print("TARGET_CLASS MISSING:" + t_URI) # update the dataframe if (sIdx >= 0) and (tIdx >= 0): df_all.label[(df_all.x == sIdx) & (df_all.y == tIdx)] = 1 print("total number of matches in the groundtruth: ") print(len(df_all[df_all.label == 1])) def load_label_from_existing_methods(df_all, datasets, dsname): dataset = datasets[dsname] methods = dataset['existing_methods'] for m in methods: print(m) fix_uri = dsname.startswith('conference') predicted_matches = preprocess.get_groundtruth(methods[m], fix_uri) print(len(predicted_matches)) df_all[m] = 0 for match in predicted_matches: s_URI = match['source_class'] t_URI = match['target_class'] # find the location of class in the class list of source ontology and target ontology sIdx = get_class_index(s_URI, source_ontology) tIdx = get_class_index(t_URI, target_ontology) # update the dataframe if (sIdx >= 0) and (tIdx >= 0): df_all[m][(df_all.x == sIdx) & (df_all.y == tIdx)] = 1 continue else: if (sIdx < 0): print("Uknown source class: " + s_URI) if (tIdx < 0): print("Uknown target class: " + t_URI) def intersection(lst1, lst2): if lst1 == None or lst2 == None: return [] lst3 = [value for value in lst1 if value in lst2] return lst3 def calculate_cosine_similarity(s_embedding, t_embedding): if s_embedding is None or t_embedding is None: return None if len(s_embedding) == 0 or len(t_embedding) == 0: return None if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0: return None similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding) return similarity @labeling_function() def LF_aml(row): if 'aml' not in row: return ABSTAIN return row.aml @labeling_function() def LF_logmap(row): if 'logmap' not in row: return ABSTAIN return row.logmap @labeling_function() def LF_yam(row): if 'yam' not in row: return ABSTAIN return row.yam @labeling_function() def LF_class_name_equal(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.class_name.lower() == target_class.class_name.lower(): return MATCHED else: return ABSTAIN @labeling_function() def LF_class_name_stemmed_equal(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.class_name_stemmed.lower() == target_class.class_name_stemmed.lower(): return MATCHED else: return ABSTAIN @labeling_function() def LF_acronyms(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] name_x = source_class.class_name name_y = target_class.class_name acro_x = source_class.acronym acro_y = target_class.acronym # acro_x = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \1', re.sub('([A-Z]+)', r' \1', name_x)).split()]) # acro_y = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \1', re.sub('([A-Z]+)', r' \1', name_y)).split()]) if name_x.lower() == acro_y.lower() or name_y.lower() == acro_x.lower(): return MATCHED else: return ABSTAIN @labeling_function() def LF_class_name_synonyms(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] # from nltk.corpus import wordnet # a_class_synonyms = [] # for syn in wordnet.synsets(source_class.class_name): # for lm in syn.lemmas(): # a_class_synonyms.append(lm.name()) # b_class_synonyms = [] # for syn in wordnet.synsets(target_class.class_name): # for lm in syn.lemmas(): # b_class_synonyms.append(lm.name()) common = list(set(source_class.synonyms) & set(target_class.synonyms)) if len(common) > 0: return MATCHED return ABSTAIN @labeling_function() def LF_label_equal(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.label == None or target_class.label == None: return ABSTAIN if source_class.label == '' or target_class.label == '': return ABSTAIN if source_class.label == target_class.label: return MATCHED else: return ABSTAIN @labeling_function() def LF_root_nouns_equal(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.class_name_root == None or target_class.class_name_root == None: return ABSTAIN if source_class.class_name_root.lower() == target_class.class_name_root.lower() and fuzz.ratio( source_class.class_name_root, target_class.class_name_root) > 90: return MATCHED else: return ABSTAIN @labeling_function() def LF_class_name_spacy_distance(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] s = calculate_cosine_similarity(source_class.class_long_name_embedding, target_class.class_long_name_embedding) if s == None: return ABSTAIN elif s > 0.95: return MATCHED elif s < 0.20: return UNMATCHED return ABSTAIN @labeling_function() def LF_class_name_distance(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] r = fuzz.ratio(source_class.class_name, target_class.class_name) if r > 90: return MATCHED if r < 30: return UNMATCHED else: return ABSTAIN @labeling_function() def LF_name_segment_overlap(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.class_name_words == None or target_class.class_name_words == None: return ABSTAIN overlapset = intersection(source_class.class_name_words, target_class.class_name_words) if len(overlapset) >= 1: return MATCHED else: return ABSTAIN @labeling_function() def LF_label_words_overlap(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.label_words == None or target_class.label_words == None: return ABSTAIN overlapset = intersection(source_class.label_words, target_class.label_words) if len(overlapset) >= 1: return MATCHED else: return ABSTAIN @labeling_function() def LF_subclasses_overlap(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.subclasses == None or target_class.subclasses == None: return ABSTAIN overlapset = intersection(source_class.subclasses, target_class.subclasses) if len(overlapset) >= 1: return MATCHED else: return ABSTAIN @labeling_function() def LF_superclasses_overlap(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.superclasses == None or target_class.superclasses == None: return ABSTAIN overlapset = intersection(source_class.superclasses, target_class.superclasses) if len(overlapset) >= 1: return MATCHED else: return ABSTAIN @labeling_function() def LF_properties_overlap(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] if source_class.properties == None or target_class.properties == None: return ABSTAIN overlapset = intersection(source_class.properties, target_class.properties) if len(overlapset) >= 1: return MATCHED else: return ABSTAIN # calculate the distance between two strings def levenshtein_distance(string1, string2): return Levenshtein.distance(string1, string2) def hamming_distance(string1, string2): return textdistance.hamming.distance(string1, string2) def f_x1(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] source_class_name = source_class.class_name target_class_name = target_class.class_name return len(intersection(source_class_name, target_class_name)) def f_x2(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] source_class_name = source_class.class_name target_class_name = target_class.class_name return levenshtein_distance(source_class_name, target_class_name) def f_x3(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] source_class_name = source_class.class_name target_class_name = target_class.class_name return hamming_distance(source_class_name, target_class_name) def f_x4(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] source_class_name = source_class.class_name target_class_name = target_class.class_name sentence = '[CLS]' + source_class_name + '[SEP]' + target_class_name + '[SEP]' return sentence def f_x5(row): x = int(row['x']) y = int(row['y']) source_class = source_ontology[x] target_class = target_ontology[y] return calculate_cosine_similarity(source_class.class_long_name_embedding, target_class.class_long_name_embedding) def apply_ffs(df, ffs): F = [] for feature_function in ffs: F.append(df.apply(feature_function, axis=1)) return pd.concat(F, axis=1).values lfs = [ LF_aml, LF_logmap, LF_yam, LF_class_name_equal, LF_class_name_stemmed_equal, LF_acronyms, LF_class_name_synonyms, LF_label_equal, LF_root_nouns_equal, LF_class_name_spacy_distance, LF_class_name_distance, LF_name_segment_overlap, LF_label_words_overlap, LF_subclasses_overlap, LF_superclasses_overlap, LF_properties_overlap, ] if __name__ == '__main__': # Count the arguments arguments = len(sys.argv) - 1 if arguments < 1: print("please specify the data set [conference|ai4eu|nasa|anatomy]") exit() else: dataset_name = sys.argv[1] # load the configuration cfg = read_config('./config.json') # build the alignment profile for both source ontology and target ontology prepare_alignment_profile(cfg["rdf4j_server"], cfg["datasets"][dataset_name]) # initialize the dataframe for all combinations of source and target classes df_all = init_dataframe_with_all_combinations() # load the ground truth load_ground_truth(df_all, cfg["datasets"], dataset_name) # load the label given by existing methods load_label_from_existing_methods(df_all, cfg["datasets"], dataset_name) # # apply all labeling functions to get the label matrix # start = time.time() # applier = PandasLFApplier(lfs=lfs) # label_matrix = applier.apply(df=df_all) # end = time.time() # print("Labeling Time: ", end - start, "s") # # # convert label matrix to dataframe # label_columns = [lf.name for lf in lfs] # my_label_df = pd.DataFrame(data=label_matrix, columns=label_columns) # my_label_df['label'] = df_all.label.values # my_label_df['x'] = df_all.x.values # my_label_df['y'] = df_all.y.values # # # prepare the feature dataframe # feature_functions = [f_x1, f_x2, f_x3, f_x4, f_x5] # feature_columns = ['shared_word', 'levenshtein_distance', 'hamming_distance', 'text_pair', # 'class_name_embedding_distance'] # start = time.time() # feature_matrix = apply_ffs(df_all, feature_functions) # my_feature_df = pd.DataFrame(data=feature_matrix, columns=feature_columns) # end = time.time() # print("Feature Generation Time: ", end - start, "s") # # # merge the feature and label dataframe # dfs = [my_label_df, my_feature_df] # combined_dfs = pd.concat(dfs, axis=1) # # make sure the result folder is created if not os.path.exists('./results/' + dataset_name): os.makedirs('./results/' + dataset_name) # save the dataframe df_all.to_csv('./results/' + dataset_name + '/' + dataset_name + '.csv', index=False, header=True) # save the dataframe for both source and target ontologies pickle.dump(source_ontology, open('./results/' + dataset_name + '/' + dataset_name + "_source_ontology.pk", "wb")) pickle.dump(target_ontology, open('./results/' + dataset_name + '/' + dataset_name + "_target_ontology.pk", "wb"))