#
#     dualloop
#
#        File:  blocking.py
#
#     Authors: Deleted for purposes of anonymity
#
#     Proprietor: Deleted for purposes of anonymity --- PROPRIETARY INFORMATION
#
# The software and its source code contain valuable trade secrets and shall be maintained in
# confidence and treated as confidential information. The software may only be used for
# evaluation and/or testing purposes, unless otherwise explicitly stated in the terms of a
# license agreement or nondisclosure agreement with the proprietor of the software.
# Any unauthorized publication, transfer to third parties, or duplication of the object or
# source code---either totally or in part---is strictly prohibited.
#
#     Copyright (c) 2019 Proprietor: Deleted for purposes of anonymity
#     All Rights Reserved.
#
# THE PROPRIETOR DISCLAIMS ALL WARRANTIES, EITHER EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS FOR A PARTICULAR PURPOSE AND THE WARRANTY AGAINST LATENT
# DEFECTS, WITH RESPECT TO THE PROGRAM AND ANY ACCOMPANYING DOCUMENTATION.
#
# NO LIABILITY FOR CONSEQUENTIAL DAMAGES:
# IN NO EVENT SHALL THE PROPRIETOR OR ANY OF ITS SUBSIDIARIES BE
# LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES
# FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF INFORMATION, OR
# OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, INCIDENTAL,
# ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF OR INABILITY
# TO USE THIS PROGRAM, EVEN IF the proprietor HAS BEEN ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGES.
#
# For purposes of anonymity, the identity of the proprietor is not given herewith.
# The identity of the proprietor will be given once the review of the
# conference submission is completed.
#
# THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY.
#

import pandas as pd
import numpy as np
import pickle
import sys

from change_finder import ChangeFinder, SDAR
import matplotlib.pyplot as plt
import ruptures as rpt
import pickle
import seaborn as sns

# Count the arguments
arguments = len(sys.argv) - 1

if arguments < 1:
    print("please specify the data set [conference|ai4eu|nasa|anatomy]")
    exit()
else:
    dataset_name = sys.argv[1]

all_dataset_df = pd.read_csv('./results/' + dataset_name + '/' + dataset_name + '.csv')

source_ontology = pickle.load(open('./results/' + dataset_name + '/' + dataset_name + "_source_ontology.pk", "rb"))
target_ontology = pickle.load(open('./results/' + dataset_name + '/' + dataset_name + "_target_ontology.pk", "rb"))


def calculate_common_words(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    s1 = source_class.class_name_words
    s2 = target_class.class_name_words
    t = list(set(s1) & set(s2))

    return len(t)


all_dataset_df['num_common_words'] = all_dataset_df.apply(calculate_common_words, axis=1)

from scipy import spatial
import numpy as np


def calculate_cosine_similarity(s_embedding, t_embedding):
    if s_embedding is None or t_embedding is None:
        return None

    if len(s_embedding) == 0 or len(t_embedding) == 0:
        return None

    if np.count_nonzero(s_embedding) == 0 or np.count_nonzero(t_embedding) == 0:
        return None

    similarity = 1 - spatial.distance.cosine(s_embedding, t_embedding)
    return similarity


# calculate the embedding for the selected feature
from sentence_transformers import SentenceTransformer

distilbert_a = SentenceTransformer('distilbert-base-nli-mean-tokens')
distilbert_b = SentenceTransformer('paraphrase-MiniLM-L6-v2')

features = ['class_name_words', 'label_words', 'comment']


def computer_embedding(ontology_list, feature, embedding_feature, model_type):
    for idx in range(len(ontology_list)):
        obj = ontology_list[idx]

        attr = getattr(obj, feature)
        if attr != None and attr != "":
            if model_type == 'a':
                em = distilbert_a.encode(getattr(obj, feature))
            elif model_type == 'b':
                em = distilbert_b.encode(getattr(obj, feature))

            setattr(obj, embedding_feature, em)


def calculate_similarity(row, selected_feature):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]

    if hasattr(source_class, selected_feature) and hasattr(target_class, selected_feature):
        em1 = getattr(source_class, selected_feature)
        em2 = getattr(target_class, selected_feature)

        return calculate_cosine_similarity(em1, em2)
    else:
        return None


for feature in features:
    print(feature)

    # appply bert_a
    embedding_feature = feature + '_embedding_a'
    computer_embedding(target_ontology, feature, embedding_feature, 'a')
    computer_embedding(source_ontology, feature, embedding_feature, 'a')
    # print("compute feature embedding")
    similarity_feature = feature + '_similarity_a'
    all_dataset_df[similarity_feature] = all_dataset_df.apply(calculate_similarity,
                                                              selected_feature=embedding_feature, axis=1)
    print("compute embedding distance a")

    # appply bert_b
    embedding_feature = feature + '_embedding_b'
    computer_embedding(target_ontology, feature, embedding_feature, 'b')
    computer_embedding(source_ontology, feature, embedding_feature, 'b')
    # print("compute feature embedding")
    similarity_feature = feature + '_similarity_b'
    all_dataset_df[similarity_feature] = all_dataset_df.apply(calculate_similarity,
                                                              selected_feature=embedding_feature, axis=1)
    print("compute embedding distance b")

import changefinder


def findChangePoints(ts, r, order, smooth):
    '''
       r: Discounting rate
       order: AR model order
       smooth: smoothing window size T
    '''
    cf = changefinder.ChangeFinder(r=r, order=order, smooth=smooth)
    ts_score = [cf.update(p) for p in ts]
    return (ts_score)


def print_true_match(row):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]
    print(source_class.class_name, ' => ', target_class.class_name, row['x'], row['y'])


def print_included_match(row, f):
    x = int(row['x'])
    y = int(row['y'])
    source_class = source_ontology[x]
    target_class = target_ontology[y]
    print(source_class.class_name, ' => ', target_class.class_name, row[f], row['x'], row['y'])


def apply_blocking_top_n(df, column, n):
    df[column + '_blocked'] = 1

    x_df = df.sort_values(column, ascending=False)
    df.loc[x_df.head(n).index, column + '_blocked'] = 0


def apply_blocking_top_k(df, column, k):
    df[column + '_blocked'] = 1

    for x in range(len(source_ontology)):
        x_df = df.query('x==' + str(x) + ' &' + column + '.notnull()')[[column, 'x', 'y', column + '_blocked']]

        if len(x_df) == 0:
            # print(x, '0, ignored')
            continue

        x_df = x_df.sort_values(column, ascending=False)
        df.loc[x_df[:k].index, column + '_blocked'] = 0


def blocking_per_class_top_x(xdf, ranked_feature, check_point_detection, with_delta, debug_flag):
    # for ranked integer feature, the last increase
    if ranked_feature in ['num_common_words']:
        return len(xdf[xdf[ranked_feature] > 0])

    distances = xdf[ranked_feature].values

    num = len(distances)
    points = []

    if with_delta == True:
        for i in range(num - 1):
            cur_delta = abs(distances[i + 1] - distances[i])
            points.append(cur_delta)
    else:
        points = distances

    top_x = num
    scores = []
    if check_point_detection == "my_top_x":
        lst = np.gradient(np.gradient(points))
        top_x = len(lst) - np.argmax(lst[::-1])
    elif check_point_detection == "rbf":
        model = "rbf"
        algo = rpt.Dynp(model=model).fit(np.array(points))
        scores = algo.predict(n_bkps=1)
        top_x = scores[0] - 1
    elif check_point_detection == "l2":
        model = "l2"
        algo = rpt.Dynp(model=model).fit(np.array(points))
        scores = algo.predict(n_bkps=1)
        top_x = scores[0] - 1
    elif check_point_detection == "l1":
        model = "l1"
        algo = rpt.Dynp(model=model).fit(np.array(points))
        scores = algo.predict(n_bkps=1)
        top_x = scores[0] - 1
    elif check_point_detection == "online1":
        scores = findChangePoints(points, r=0.01, order=3, smooth=5)
        top_x = pd.Series(scores).nlargest(3).index[2]
    elif check_point_detection == "online2":
        cf = ChangeFinder()
        try:
            scores = [cf.update(p) for p in points]
            for idx in range(len(scores)):
                if scores[idx] < 2:
                    top_x = idx + 1
                    break
        except:
            top_x = 0

    if debug_flag == True:
        print(points)
        print(scores)
        print(top_x)

        xdf[:top_x].apply(print_included_match, f=ranked_feature, axis=1)

        plt.figure(figsize=(16, 4))
        plt.plot(distances, color='blue')

        plt.figure(figsize=(16, 4))
        plt.plot(points)

        plt.figure(figsize=(16, 4))
        plt.plot(scores, color='red')

        plt.show()

    return top_x


def apply_blocking_top_x(df, column, algm, max_k):
    df[column + '_blocked'] = 1

    for x in range(len(source_ontology)):
        x_df = df.query('x==' + str(x) + ' &' + column + '.notnull()')[[column, 'x', 'y', column + '_blocked']]

        if len(x_df) == 0:
            # print(x, '0, ignored')
            continue

        x_df = x_df.sort_values(column, ascending=False)
        top_x = blocking_per_class_top_x(x_df.head(max_k), column, algm, False, False)
        if top_x == 0:
            # print(x, '0, ignored')
            continue

        # update the block_flag
        # print(x, top_x)
        df.loc[x_df[:top_x].index, column + '_blocked'] = 0


columns = [
    'num_common_words',
    'class_name_words_similarity_a',
    'class_name_words_similarity_b',
    'label_words_similarity_a',
    'label_words_similarity_b',
    'comment_similarity_a',
    'comment_similarity_b',
]

# =============================== top-n =========================
top_n = 10000

for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'
    apply_blocking_top_n(all_dataset_df, blocked_column, top_n)

# show the matches included by different blocking indexes
combination_statement = "";
missing_statement = ""
for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'

    all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis=1)

    total_num = len(all_dataset_df)
    num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))

    print(num_after_blocking, ' out of ', total_num, num_after_blocking * 100.0 / total_num)

    total_true_matches = len(all_dataset_df[all_dataset_df['label'] == 1])
    captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))

    print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches * 100.0 / total_true_matches)

    if combination_statement == "":
        combination_statement = " (" + blocked_key + '==0)'
        missing_statement = " (" + blocked_key + '==1)'
    else:
        combination_statement = combination_statement + " | (" + blocked_key + '==0)'
        missing_statement = missing_statement + " & (" + blocked_key + '==1)'

all_dataset_df['selected_after_blocking'] = 0

# calculate the union of all selected items by the committee of blockers
print("\r\n============= RESULT OF TOP-N blocking =====================")
# print(combination_statement)
selected_items = all_dataset_df.query(combination_statement)
all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1
total_selected_items = len(selected_items)

query = '(label==1) & (' + combination_statement + ')'
# print(query)
total_selected_matches = len(all_dataset_df.query(query))

query = 'label==1'
# print(query)
total_matches = len(all_dataset_df.query(query))

total_num = len(all_dataset_df)

print(total_selected_items, ' out of ', total_num, total_selected_items * 100.0 / total_num)
print(total_selected_matches, ' out of ', total_matches, total_selected_matches * 100.0 / total_matches)

top_n_recall = total_selected_matches * 100.0 / total_matches
top_n_reduction = 100 - total_selected_items * 100.0 / total_num

print("\r\n=======list of all true matches========")
true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis=1)

print("\r\n***** MISSING ****")

query = '(label==1) & (' + missing_statement + ')'
# print(query)
true_matches = all_dataset_df.query(query).apply(print_true_match, axis=1)

# save the blocked dataframe
query = 'selected_after_blocking==1'
all_dataset_df.query(query).to_csv('./results/' + dataset_name + '/' + dataset_name + '_blocked_top_n' + '.csv',
                                   index=False, header=True)

# =============================== top-k =========================

MAX_NUM_CANDIDATES = 20
target_size = len(target_ontology)
max_k = target_size
if target_size > MAX_NUM_CANDIDATES:
    max_k = MAX_NUM_CANDIDATES

for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'
    apply_blocking_top_k(all_dataset_df, blocked_column, max_k)

# show the matches included by different blocking indexes
combination_statement = "";
missing_statement = ""
for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'

    all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis=1)

    total_num = len(all_dataset_df)
    num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))

    print(num_after_blocking, ' out of ', total_num, num_after_blocking * 100.0 / total_num)

    total_true_matches = len(all_dataset_df[all_dataset_df['label'] == 1])
    captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))

    print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches * 100.0 / total_true_matches)

    if combination_statement == "":
        combination_statement = " (" + blocked_key + '==0)'
        missing_statement = " (" + blocked_key + '==1)'
    else:
        combination_statement = combination_statement + " | (" + blocked_key + '==0)'
        missing_statement = missing_statement + " & (" + blocked_key + '==1)'

all_dataset_df['selected_after_blocking'] = 0

# calculate the union of all selected items by the committee of blockers
print("\r\n============= RESULT OF TOP-K blocking =====================")

# print(combination_statement)
selected_items = all_dataset_df.query(combination_statement)
all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1
total_selected_items = len(selected_items)

query = '(label==1) & (' + combination_statement + ')'
# print(query)
total_selected_matches = len(all_dataset_df.query(query))

query = 'label==1'
# print(query)
total_matches = len(all_dataset_df.query(query))

total_num = len(all_dataset_df)

print(total_selected_items, ' out of ', total_num, total_selected_items * 100.0 / total_num)
print(total_selected_matches, ' out of ', total_matches, total_selected_matches * 100.0 / total_matches)

top_k_recall = total_selected_matches * 100.0 / total_matches
top_k_reduction = 100 - total_selected_items * 100.0 / total_num

print("\r\n=======list of all true matches========")
true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis=1)

print("\r\n***** MISSING ****")

query = '(label==1) & (' + missing_statement + ')'
# print(query)
true_matches = all_dataset_df.query(query).apply(print_true_match, axis=1)

# save the blocked dataframe
query = 'selected_after_blocking==1'
all_dataset_df.query(query).to_csv('./results/' + dataset_name + '/' + dataset_name + '_blocked_top_k' + '.csv',
                                   index=False, header=True)

# =============================== top-x =========================

MAX_NUM_CANDIDATES = 200
target_size = len(target_ontology)
max_k = target_size
if target_size > MAX_NUM_CANDIDATES:
    max_k = MAX_NUM_CANDIDATES

changing_point_detection_algorithm = 'my_top_x'  # 'online2' 'rdf', 'l1', 'l2', 'online1', 'online2', 'my_top_x'
print_matches = True

for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'
    apply_blocking_top_x(all_dataset_df, blocked_column, changing_point_detection_algorithm, max_k)

# show the matches included by different blocking indexes
combination_statement = "";
missing_statement = ""
for blocked_column in columns:
    print("\r\n" + blocked_column)
    blocked_key = blocked_column + '_blocked'

    all_dataset_df.query(blocked_key + '==0 & label==1').apply(print_included_match, f=blocked_column, axis=1)

    total_num = len(all_dataset_df)
    num_after_blocking = len(all_dataset_df.query(blocked_key + '==0'))

    print(num_after_blocking, ' out of ', total_num, num_after_blocking * 100.0 / total_num)

    total_true_matches = len(all_dataset_df[all_dataset_df['label'] == 1])
    captured_true_matches = len(all_dataset_df.query(blocked_key + '==0 & label==1'))

    print(captured_true_matches, ' out of ', total_true_matches, captured_true_matches * 100.0 / total_true_matches)

    if combination_statement == "":
        combination_statement = " (" + blocked_key + '==0)'
        missing_statement = " (" + blocked_key + '==1)'
    else:
        combination_statement = combination_statement + " | (" + blocked_key + '==0)'
        missing_statement = missing_statement + " & (" + blocked_key + '==1)'

all_dataset_df['selected_after_blocking'] = 0

# calculate the union of all selected items by the committee of blockers
print("\r\n============= RESULT OF TOP-X blocking =====================")

# print(combination_statement)
selected_items = all_dataset_df.query(combination_statement)
all_dataset_df.loc[selected_items.index, 'selected_after_blocking'] = 1
total_selected_items = len(selected_items)

query = '(label==1) & (' + combination_statement + ')'
# print(query)
total_selected_matches = len(all_dataset_df.query(query))

query = 'label==1'
# print(query)
total_matches = len(all_dataset_df.query(query))

total_num = len(all_dataset_df)

print(total_selected_items, ' out of ', total_num, total_selected_items * 100.0 / total_num)
print(total_selected_matches, ' out of ', total_matches, total_selected_matches * 100.0 / total_matches)

top_x_recall = total_selected_matches * 100.0 / total_matches
top_x_reduction = 100 - total_selected_items * 100.0 / total_num

print("\r\n=======list of all true matches========")
true_matches = all_dataset_df.query('label==1').apply(print_true_match, axis=1)

print("\r\n***** MISSING ****")

query = '(label==1) & (' + missing_statement + ')'
# print(query)
true_matches = all_dataset_df.query(query).apply(print_true_match, axis=1)

# save the blocked dataframe
query = 'selected_after_blocking==1'
all_dataset_df.query(query).to_csv('./results/' + dataset_name + '/' + dataset_name + '_blocked_top_x' + '.csv',
                                   index=False, header=True)

# save the result into a figure
data = [
    ['top-n', 'recall', top_n_recall],
    ['top-n', 'reduction', top_n_reduction],
    ['top-k', 'recall', top_k_recall],
    ['top-k', 'reduction', top_k_reduction],
    ['top-x', 'recall', top_x_recall],
    ['top-x', 'reduction', top_x_reduction],
]

columns = ["blocking methods", "category", "result"]

result_df = pd.DataFrame.from_records(data, columns=columns)

# save the result into a file
result_df.to_csv('./results/' + dataset_name + '/' + dataset_name + '_blocking_result' + '.csv', index=False,
                 header=True)

# Plot
fig, ax1 = plt.subplots(figsize=(10, 6))
g = sns.barplot(x=columns[0], y="result", hue="category", \
                data=result_df, ax=ax1)

ax2 = ax1.twinx()

ax2.set_ylim(ax1.get_ylim())
ax2.set_yticklabels(np.round(ax1.get_yticks(), 1))

ax1.set_ylabel('recall(%)')
ax2.set_ylabel('reduction(%)')

plt.grid(False)
ax1.grid(False)
ax2.grid(False)

plt.savefig('./results/' + dataset_name + '/' + dataset_name + '_blocking_result.png', transparent=True)
plt.close(fig)