#
#     dualloop
#
#        File:  data_preparation.py
#
#     Authors: Deleted for purposes of anonymity
#
#     Proprietor: Deleted for purposes of anonymity --- PROPRIETARY INFORMATION
#
# The software and its source code contain valuable trade secrets and shall be maintained in
# confidence and treated as confidential information. The software may only be used for
# evaluation and/or testing purposes, unless otherwise explicitly stated in the terms of a
# license agreement or nondisclosure agreement with the proprietor of the software.
# Any unauthorized publication, transfer to third parties, or duplication of the object or
# source code---either totally or in part---is strictly prohibited.
#
#     Copyright (c) 2019 Proprietor: Deleted for purposes of anonymity
#     All Rights Reserved.
#
# THE PROPRIETOR DISCLAIMS ALL WARRANTIES, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS FOR A PARTICULAR PURPOSE AND THE WARRANTY AGAINST LATENT
# DEFECTS, WITH RESPECT TO THE PROGRAM AND ANY ACCOMPANYING DOCUMENTATION.
#
# NO LIABILITY FOR CONSEQUENTIAL DAMAGES:
# IN NO EVENT SHALL THE PROPRIETOR OR ANY OF ITS SUBSIDIARIES BE
# LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES
# FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF INFORMATION, OR
# OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, INCIDENTAL,
# ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF OR INABILITY
# TO USE THIS PROGRAM, EVEN IF the proprietor HAS BEEN ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGES.
#
# For purposes of anonymity, the identity of the proprietor is not given herewith.
# The identity of the proprietor will be given once the review of the
# conference submission is completed.
#
# THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY.
#

import json
import os
import pickle
import sys
import time
import pandas as pd
import preprocessing as preprocess

from snorkel.labeling import labeling_function
from snorkel.labeling import PandasLFApplier

import nltk

nltk.download('wordnet')

from fuzzywuzzy import fuzz

import re

import wordsegment

wordsegment.load()

from scipy import spatial
import numpy as np

import Levenshtein
import textdistance

ABSTAIN = -1
UNMATCHED = 0
MATCHED = 1

def read_config(config_file):
    with open(config_file, 'r') as f:
        config = json.load(f)
    return config


# global list and map for ontologies
source_ontology = []
target_ontology = []
sourceMap = {}
targetMap = {}


def prepare_alignment_profile(rdf4j_server, dataset):
    rdf4j_server_url = rdf4j_server + "/rdf4j-server/repositories/"

    # add one ontology into the source_ontology
    repository_name = rdf4j_server_url + dataset['source']
    alignment_profiles = preprocess.build_alignment_profile(repository_name, False)
    for profile in alignment_profiles:
        uri = profile.uri.lower()
        if uri not in sourceMap:
            sourceMap[uri] = profile
            source_ontology.append(profile)

    # add one ontology into the target_ontology
    repository_name = rdf4j_server_url + dataset['target']
    alignment_profiles = preprocess.build_alignment_profile(repository_name, False)
    for profile in alignment_profiles:
        uri = profile.uri.lower()
        if uri not in targetMap:
            targetMap[uri] = profile
            target_ontology.append(profile)

    print("# of classess in the source ontology: ", len(source_ontology))
    print("# of classess in the target ontology: ", len(target_ontology))


def init_dataframe_with_all_combinations():
    items = []
    begin_x = 0
    begin_y = 0
    for i in range(len(source_ontology)):
        for j in range(len(target_ontology)):
            x = begin_x + i
            y = begin_y + j
            items.append({'x': x, 'y': y})

    cols = ['x', 'y']
    return pd.DataFrame(items, columns=cols)


def get_class_index(uri, clist):
    for idx in range(len(clist)):
        if clist[idx].uri.lower() == uri.lower():
            return idx

    return -1


def load_ground_truth(df_all, datasets, dsname):
    # load the gold labels
    df_all['label'] = 0

    ground_truth = []
    fix_uri = dsname.startswith('conference')
    dataset = datasets[dsname]
    matches = preprocess.get_groundtruth(dataset['groundtruth'], fix_uri)

    print(len(matches))
    ground_truth = ground_truth + matches

    for match in ground_truth:
        s_URI = match['source_class']
        t_URI = match['target_class']

        # find the location of class in the class list of source ontology and target ontology
        sIdx = get_class_index(s_URI, source_ontology)
        tIdx = get_class_index(t_URI, target_ontology)

        if (sIdx < 0):
            print("SOURCE_CLASS MISSING:" + s_URI)

        if (tIdx < 0):
            print("TARGET_CLASS MISSING:" + t_URI)

        # update the dataframe
        if (sIdx >= 0) and (tIdx >= 0):
            df_all.label[(df_all.x == sIdx) & (df_all.y == tIdx)] = 1

    print("total number of matches in the groundtruth: ")
    print(len(df_all[df_all.label == 1]))


def load_label_from_existing_methods(df_all, datasets, dsname):
    dataset = datasets[dsname]
    methods = dataset['existing_methods']
    for m in methods:
        print(m)
        fix_uri = dsname.startswith('conference')
        predicted_matches = preprocess.get_groundtruth(methods[m], fix_uri)
        print(len(predicted_matches))

        df_all[m] = 0

        for match in predicted_matches:
            s_URI = match['source_class']
            t_URI = match['target_class']

            # find the location of class in the class list of source ontology and target ontology
            sIdx = get_class_index(s_URI, source_ontology)
            tIdx = get_class_index(t_URI, target_ontology)

            # update the dataframe
            if (sIdx >= 0) and (tIdx >= 0):
                df_all[m][(df_all.x == sIdx) & (df_all.y == tIdx)] = 1
                continue
            else:
                if (sIdx < 0):
                    print("Uknown source class: " + s_URI)

                if (tIdx < 0):
                    print("Uknown target class: " + t_URI)


def intersection(lst1, lst2):
    if lst1 == None or lst2 == None:
        return []

    lst3 = [value for value in lst1 if value in lst2]
    return lst3


def calculate_cosine_similarity(s_embedding, t_embedding):
    if s_embedding is None or t_embedding is None:
        return None

    if len(s_embedding) == 0 or len(t_embedding) == 0:
        return None

    if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:
        return None

    similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)
    return similarity


@labeling_function()
def LF_aml(row):
    if 'aml' not in row:
        return ABSTAIN
    return row.aml


@labeling_function()
def LF_logmap(row):
    if 'logmap' not in row:
        return ABSTAIN
    return row.logmap


@labeling_function()
def LF_yam(row):
    if 'yam' not in row:
        return ABSTAIN
    return row.yam


@labeling_function()
def LF_class_name_equal(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.class_name.lower() == target_class.class_name.lower():
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_class_name_stemmed_equal(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.class_name_stemmed.lower() == target_class.class_name_stemmed.lower():
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_acronyms(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    name_x = source_class.class_name
    name_y = target_class.class_name

    acro_x = source_class.acronym
    acro_y = target_class.acronym

    # acro_x = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \1', re.sub('([A-Z]+)', r' \1', name_x)).split()])
    # acro_y = ''.join([s[0] for s in re.sub('([A-Z][a-z]+)', r' \1', re.sub('([A-Z]+)', r' \1', name_y)).split()])
    if name_x.lower() == acro_y.lower() or name_y.lower() == acro_x.lower():
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_class_name_synonyms(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    # from nltk.corpus import wordnet
    # a_class_synonyms = []
    # for syn in wordnet.synsets(source_class.class_name):
    #     for lm in syn.lemmas():
    #         a_class_synonyms.append(lm.name())
    # b_class_synonyms = []
    # for syn in wordnet.synsets(target_class.class_name):
    #     for lm in syn.lemmas():
    #         b_class_synonyms.append(lm.name())

    common = list(set(source_class.synonyms) & set(target_class.synonyms))
    if len(common) > 0:
        return MATCHED

    return ABSTAIN


@labeling_function()
def LF_label_equal(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.label == None or target_class.label == None:
        return ABSTAIN

    if source_class.label == '' or target_class.label == '':
        return ABSTAIN

    if source_class.label == target_class.label:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_root_nouns_equal(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.class_name_root == None or target_class.class_name_root == None:
        return ABSTAIN

    if source_class.class_name_root.lower() == target_class.class_name_root.lower() and fuzz.ratio(
            source_class.class_name_root, target_class.class_name_root) > 90:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_class_name_spacy_distance(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    s = calculate_cosine_similarity(source_class.class_long_name_embedding, target_class.class_long_name_embedding)
    if s == None:
        return ABSTAIN
    elif s > 0.95:
        return MATCHED
    elif s < 0.20:
        return UNMATCHED
    return ABSTAIN


@labeling_function()
def LF_class_name_distance(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    r = fuzz.ratio(source_class.class_name, target_class.class_name)
    if r > 90:
        return MATCHED
    if r < 30:
        return UNMATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_name_segment_overlap(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.class_name_words == None or target_class.class_name_words == None:
        return ABSTAIN

    overlapset = intersection(source_class.class_name_words, target_class.class_name_words)
    if len(overlapset) >= 1:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_label_words_overlap(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.label_words == None or target_class.label_words == None:
        return ABSTAIN

    overlapset = intersection(source_class.label_words, target_class.label_words)
    if len(overlapset) >= 1:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_subclasses_overlap(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.subclasses == None or target_class.subclasses == None:
        return ABSTAIN

    overlapset = intersection(source_class.subclasses, target_class.subclasses)
    if len(overlapset) >= 1:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_superclasses_overlap(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.superclasses == None or target_class.superclasses == None:
        return ABSTAIN

    overlapset = intersection(source_class.superclasses, target_class.superclasses)
    if len(overlapset) >= 1:
        return MATCHED
    else:
        return ABSTAIN


@labeling_function()
def LF_properties_overlap(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if source_class.properties == None or target_class.properties == None:
        return ABSTAIN

    overlapset = intersection(source_class.properties, target_class.properties)
    if len(overlapset) >= 1:
        return MATCHED
    else:
        return ABSTAIN


# calculate the distance between two strings
def levenshtein_distance(string1, string2):
    return Levenshtein.distance(string1, string2)


def hamming_distance(string1, string2):
    return textdistance.hamming.distance(string1, string2)


def f_x1(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    source_class_name = source_class.class_name
    target_class_name = target_class.class_name

    return len(intersection(source_class_name, target_class_name))


def f_x2(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    source_class_name = source_class.class_name
    target_class_name = target_class.class_name

    return levenshtein_distance(source_class_name, target_class_name)


def f_x3(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    source_class_name = source_class.class_name
    target_class_name = target_class.class_name

    return hamming_distance(source_class_name, target_class_name)


def f_x4(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    source_class_name = source_class.class_name
    target_class_name = target_class.class_name

    sentence = '[CLS]' + source_class_name + '[SEP]' + target_class_name + '[SEP]'

    return sentence


def f_x5(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    return calculate_cosine_similarity(source_class.class_long_name_embedding, target_class.class_long_name_embedding)


def apply_ffs(df, ffs):
    F = []
    for feature_function in ffs:
        F.append(df.apply(feature_function, axis=1))
    return pd.concat(F, axis=1).values


lfs = [
    LF_aml,
    LF_logmap,
    LF_yam,

    LF_class_name_equal,
    LF_class_name_stemmed_equal,
    LF_acronyms,
    LF_class_name_synonyms,
    LF_label_equal,
    LF_root_nouns_equal,

    LF_class_name_spacy_distance,
    LF_class_name_distance,

    LF_name_segment_overlap,
    LF_label_words_overlap,
    LF_subclasses_overlap,
    LF_superclasses_overlap,
    LF_properties_overlap,
]

if __name__ == '__main__':
    # Count the arguments
    arguments = len(sys.argv) - 1
    if arguments < 1:
        print("please specify the data set [conference|ai4eu|nasa|anatomy]")
        exit()
    else:
        dataset_name = sys.argv[1]

    # load the configuration
    cfg = read_config('./config.json')

    # build the alignment profile for both source ontology and target ontology
    prepare_alignment_profile(cfg["rdf4j_server"], cfg["datasets"][dataset_name])

    # initialize the dataframe for all combinations of source and target classes
    df_all = init_dataframe_with_all_combinations()

    # load the ground truth
    load_ground_truth(df_all, cfg["datasets"], dataset_name)

    # load the label given by existing methods
    load_label_from_existing_methods(df_all, cfg["datasets"], dataset_name)



    # # apply all labeling functions to get the label matrix
    # start = time.time()
    # applier = PandasLFApplier(lfs=lfs)
    # label_matrix = applier.apply(df=df_all)
    # end = time.time()
    # print("Labeling Time: ", end - start, "s")
    #
    # # convert label matrix to dataframe
    # label_columns = [lf.name for lf in lfs]
    # my_label_df = pd.DataFrame(data=label_matrix, columns=label_columns)
    # my_label_df['label'] = df_all.label.values
    # my_label_df['x'] = df_all.x.values
    # my_label_df['y'] = df_all.y.values
    #
    # # prepare the feature dataframe
    # feature_functions = [f_x1, f_x2, f_x3, f_x4, f_x5]
    # feature_columns = ['shared_word', 'levenshtein_distance', 'hamming_distance', 'text_pair',
    #                    'class_name_embedding_distance']
    # start = time.time()
    # feature_matrix = apply_ffs(df_all, feature_functions)
    # my_feature_df = pd.DataFrame(data=feature_matrix, columns=feature_columns)
    # end = time.time()
    # print("Feature Generation Time: ", end - start, "s")
    #
    # # merge the feature and label dataframe
    # dfs = [my_label_df, my_feature_df]
    # combined_dfs = pd.concat(dfs, axis=1)
    #

    # make sure the result folder is created
    if not os.path.exists('./results/' + dataset_name):
        os.makedirs('./results/' + dataset_name)

    # save the dataframe
    df_all.to_csv('./results/' + dataset_name + '/' + dataset_name + '.csv', index=False, header=True)

    # save the dataframe for both source and target ontologies
    pickle.dump(source_ontology, open('./results/' + dataset_name + '/' + dataset_name + "_source_ontology.pk", "wb"))
    pickle.dump(target_ontology, open('./results/' + dataset_name + '/' + dataset_name + "_target_ontology.pk", "wb"))