#
#     dualloop
#
#        File:  blocking.py
#
#     Authors: Deleted for purposes of anonymity
#
#     Proprietor: Deleted for purposes of anonymity --- PROPRIETARY INFORMATION
#
# The software and its source code contain valuable trade secrets and shall be maintained in
# confidence and treated as confidential information. The software may only be used for
# evaluation and/or testing purposes, unless otherwise explicitly stated in the terms of a
# license agreement or nondisclosure agreement with the proprietor of the software.
# Any unauthorized publication, transfer to third parties, or duplication of the object or
# source code---either totally or in part---is strictly prohibited.
#
#     Copyright (c) 2019 Proprietor: Deleted for purposes of anonymity
#     All Rights Reserved.
#
# THE PROPRIETOR DISCLAIMS ALL WARRANTIES, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS FOR A PARTICULAR PURPOSE AND THE WARRANTY AGAINST LATENT
# DEFECTS, WITH RESPECT TO THE PROGRAM AND ANY ACCOMPANYING DOCUMENTATION.
#
# NO LIABILITY FOR CONSEQUENTIAL DAMAGES:
# IN NO EVENT SHALL THE PROPRIETOR OR ANY OF ITS SUBSIDIARIES BE
# LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES
# FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF INFORMATION, OR
# OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, INCIDENTAL,
# ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF OR INABILITY
# TO USE THIS PROGRAM, EVEN IF the proprietor HAS BEEN ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGES.
#
# For purposes of anonymity, the identity of the proprietor is not given herewith.
# The identity of the proprietor will be given once the review of the
# conference submission is completed.
#
# THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY.
#

import pandas as pd
import numpy as np
import pickle
import sys

from change_finder import ChangeFinder, SDAR
import matplotlib.pyplot as plt
import ruptures as rpt
import pickle
import seaborn as sns

# Count the arguments
arguments = len(sys.argv) - 1

if arguments < 1:
    print("please specify the data set [conference|ai4eu|nasa|anatomy]")
    exit()
else:
    dataset_name = sys.argv[1]

all_dataset_df = pd.read_csv('./results/' + dataset_name + '/' + dataset_name + '.csv')

source_ontology = pickle.load(open('./results/' + dataset_name + '/' + dataset_name + "_source_ontology.pk", "rb"))
target_ontology = pickle.load(open('./results/' + dataset_name + '/' + dataset_name + "_target_ontology.pk", "rb"))


def calculate_common_words(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    s1 = source_class.class_name_words
    s2 = target_class.class_name_words
    t = list(set(s1) & set(s2))

    return len(t)


all_dataset_df['num_common_words'] = all_dataset_df.apply(calculate_common_words, axis=1)

from scipy import spatial
import numpy as np


def calculate_cosine_similarity(s_embedding, t_embedding):
    if s_embedding is None or t_embedding is None:
        return None

    if len(s_embedding) == 0 or len(t_embedding) == 0:
        return None

    if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:
        return None

    similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)
    return similarity


# calculate the embedding for the selected feature
from sentence_transformers import SentenceTransformer

distilbert_a = SentenceTransformer('distilbert-base-nli-mean-tokens')
distilbert_b = SentenceTransformer('paraphrase-MiniLM-L6-v2')

features = ['class_name_words', 'label_words', 'comment']


def computer_embedding(ontology_list, feature, embedding_feature, model_type):
    for idx in range(len(ontology_list)):
        obj = ontology_list[idx]

        attr = getattr(obj, feature)
        if attr != None and attr != "":
            if model_type == 'a':
                em = distilbert_a.encode(getattr(obj, feature))
            elif model_type == 'b':
                em = distilbert_b.encode(getattr(obj, feature))

            setattr(obj, embedding_feature, em)


def calculate_similarity(row, selected_feature):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if hasattr(source_class, selected_feature) and hasattr(target_class, selected_feature):
        em1 = getattr(source_class, selected_feature)
        em2 = getattr(target_class, selected_feature)

        return calculate_cosine_similarity(em1, em2)
    else:
        return None


for feature in features:
    print(feature)

    # appply bert_a
    embedding_feature = feature + '_embedding_a'
    computer_embedding(target_ontology, feature, embedding_feature, 'a')
    computer_embedding(source_ontology, feature, embedding_feature, 'a')
    # print("compute feature embedding")
    similarity_feature = feature + '_similarity_a'
    all_dataset_df[similarity_feature] = all_dataset_df.apply(calculate_similarity,
                                                              selected_feature=embedding_feature, axis=1)
    print("compute embedding distance a")

    # appply bert_b
    embedding_feature = feature + '_embedding_b'
    computer_embedding(target_ontology, feature, embedding_feature, 'b')
    computer_embedding(source_ontology, feature, embedding_feature, 'b')
    # print("compute feature embedding")
    similarity_feature = feature + '_similarity_b'
    all_dataset_df[similarity_feature] = all_dataset_df.apply(calculate_similarity,
                                                              selected_feature=embedding_feature, axis=1)
    print("compute embedding distance b")

def print_true_match(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]
    print(source_class.class_name, ' => ', target_class.class_name, row['x'], row['y'])


def print_included_match(row, f):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]
    print(source_class.class_name, ' => ', target_class.class_name, row[f], row['x'], row['y'])

def apply_blocking_top_k(df, column, k):
    df[column + '_blocked'] = 1

    for x in range(len(source_ontology)):
        x_df = df.query('x==' + str(x) + ' &' + column + '.notnull()')[[column, 'x', 'y', column + '_blocked']]

        if len(x_df) == 0:
            # print(x, '0, ignored')
            continue

        x_df = x_df.sort_values(column, ascending=False)
        df.loc[x_df[:k].index, column + '_blocked'] = 0


columns = [
    'num_common_words',
    'class_name_words_similarity_a',
    'class_name_words_similarity_b',
    'label_words_similarity_a',
    'label_words_similarity_b',
    'comment_similarity_a',
    'comment_similarity_b',
]

# =============================== top-k =========================

MAX_NUM_CANDIDATES = 5
target_size = len(target_ontology)
max_k = target_size
if target_size > MAX_NUM_CANDIDATES:
    max_k = MAX_NUM_CANDIDATES

for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'
    apply_blocking_top_k(all_dataset_df, blocked_column, max_k)

# show the matches included by different blocking indexes
combination_statement = "";
missing_statement = ""
for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'

    all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis=1)

    total_num = len(all_dataset_df)
    num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))

    print(num_after_blocking, ' out of ', total_num, num_after_blocking * 100.0 / total_num)

    total_true_matches = len(all_dataset_df[all_dataset_df['label'] == 1])
    captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))

    print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches * 100.0 / total_true_matches)

    if combination_statement == "":
        combination_statement = " (" + blocked_key + '==0)'
        missing_statement = " (" + blocked_key + '==1)'
    else:
        combination_statement = combination_statement + " | (" + blocked_key + '==0)'
        missing_statement = missing_statement + " & (" + blocked_key + '==1)'

all_dataset_df['selected_after_blocking'] = 0

# calculate the union of all selected items by the committee of blockers
print("\r\n============= RESULT OF TOP-K blocking =====================")

# print(combination_statement)
selected_items = all_dataset_df.query(combination_statement)
all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1
total_selected_items = len(selected_items)

query = '(label==1) & (' + combination_statement + ')'
# print(query)
total_selected_matches = len(all_dataset_df.query(query))

query = 'label==1'
# print(query)
total_matches = len(all_dataset_df.query(query))

total_num = len(all_dataset_df)

print(total_selected_items, ' out of ', total_num, total_selected_items * 100.0 / total_num)
print(total_selected_matches, ' out of ', total_matches, total_selected_matches * 100.0 / total_matches)

top_k_recall = total_selected_matches * 100.0 / total_matches
top_k_reduction = 100 - total_selected_items * 100.0 / total_num

print("\r\n=======list of all true matches========")
true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis=1)

print("\r\n***** MISSING ****")

query = '(label==1) & (' + missing_statement + ')'
# print(query)
true_matches = all_dataset_df.query(query).apply(print_true_match, axis=1)

# save the blocked dataframe
query = 'selected_after_blocking==1'
all_dataset_df.query(query).to_csv('./results/' + dataset_name + '/' + dataset_name + '_blocked_top_k' + '.csv',
                                   index=False, header=True)