# # dualloop # # File: plot_results.py # # Authors: Deleted for purposes of anonymity # # Proprietor: Deleted for purposes of anonymity --- PROPRIETARY INFORMATION # # The software and its source code contain valuable trade secrets and shall be maintained in # confidence and treated as confidential information. The software may only be used for # evaluation and/or testing purposes, unless otherwise explicitly stated in the terms of a # license agreement or nondisclosure agreement with the proprietor of the software. # Any unauthorized publication, transfer to third parties, or duplication of the object or # source code---either totally or in part---is strictly prohibited. # # Copyright (c) 2019 Proprietor: Deleted for purposes of anonymity # All Rights Reserved. # # THE PROPRIETOR DISCLAIMS ALL WARRANTIES, EITHER EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS FOR A PARTICULAR PURPOSE AND THE WARRANTY AGAINST LATENT # DEFECTS, WITH RESPECT TO THE PROGRAM AND ANY ACCOMPANYING DOCUMENTATION. # # NO LIABILITY FOR CONSEQUENTIAL DAMAGES: # IN NO EVENT SHALL THE PROPRIETOR OR ANY OF ITS SUBSIDIARIES BE # LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES # FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF INFORMATION, OR # OTHER PECUNIARY LOSS AND INDIRECT, CONSEQUENTIAL, INCIDENTAL, # ECONOMIC OR PUNITIVE DAMAGES) ARISING OUT OF THE USE OF OR INABILITY # TO USE THIS PROGRAM, EVEN IF the proprietor HAS BEEN ADVISED OF # THE POSSIBILITY OF SUCH DAMAGES. # # For purposes of anonymity, the identity of the proprietor is not given herewith. # The identity of the proprietor will be given once the review of the # conference submission is completed. # # THIS HEADER MAY NOT BE EXTRACTED OR MODIFIED IN ANY WAY. # import pickle import sys import json import numpy as np import pandas as pd import seaborn as sns import matplotlib.pyplot as plt sns.set(font_scale = 3) sns.set_theme() sns.set_style("whitegrid") def plot_f1_result(results, dataset, filename): methods = ['WeSAL', 'AL-RF', 'DualLoop'] labels = ['WeSAL', 'AL-RF', 'DualLoop'] metric_name = 'f1' measurement_list = [] for exp in methods: experiment = results[exp] for iteration in experiment: r = experiment[iteration] measurement = { 'methods': exp, 'iteration': iteration, 'metric': r[metric_name] * 100.0 } measurement_list.append(measurement) exp_result_df = pd.DataFrame(measurement_list) fig, ax = plt.subplots(figsize=(6, 5)) plt.ylim(0, 100) p = sns.lineplot(x='iteration', y='metric', hue='methods', ax=ax, data=exp_result_df, style="methods", markers=True) p.set_xlabel("# of annotated data points") p.set_ylabel("F1 (%)") ax.title.set_text(dataset) plt.savefig(filename, transparent=False) plt.close(fig) def plot_cost_over_gain(results, dataset, filename): methods = ['WeSAL', 'AL-RF', 'DualLoop'] labels = ['WeSAL', 'AL-RF', 'DualLoop'] measurement_list = [] for exp in methods: costMap = {} experiment = results[exp] previous_gain = 0 for iteration in experiment: if iteration == 0: continue m = experiment[iteration] gain = m['num_true_matches'] cost = m['human_effort'] if gain == 0 and cost == 0: continue if gain not in costMap: for j in range(gain, previous_gain, -1): measurement = { 'methods': exp, 'gain': j, 'cost': cost } measurement_list.append(measurement) costMap[gain] = cost previous_gain = gain exp_result_df = pd.DataFrame(measurement_list) fig, ax = plt.subplots(figsize=(6, 5)) p = sns.lineplot(x='gain', y='cost', hue='methods', ax=ax, data=exp_result_df, style="methods", markers=True) p.set_xlabel("gain") p.set_ylabel("cost") ax.title.set_text(dataset) plt.savefig(filename, transparent=False) plt.close(fig) def plot_cost_over_recall(results, dataset, filename): methods = ['WeSAL', 'AL-RF', 'DualLoop'] labels = ['WeSAL', 'AL-RF', 'DualLoop'] measurement_list = [] for exp in methods: costMap = {} experiment = results[exp] previous_gain = 0 for iteration in experiment: if iteration == 0: continue m = experiment[iteration] gain = m['num_true_matches'] cost = m['human_effort'] if gain == 0 and cost == 0: continue if gain not in costMap: for j in range(gain, previous_gain, -1): measurement = { 'methods': exp, 'gain': j, 'cost': cost } measurement_list.append(measurement) costMap[gain] = cost previous_gain = gain exp_result_df = pd.DataFrame(measurement_list) fig, ax = plt.subplots(figsize=(6, 5)) p = sns.lineplot(x='gain', y='cost', hue='methods', ax=ax, data=exp_result_df, style="methods", markers=True) p.set_xlabel("gain") p.set_ylabel("cost") ax.title.set_text(dataset) plt.savefig(filename, transparent=False) plt.close(fig) def plot_cost_gap_with_bargroup(results, dataset, filename): methods = ['WeSAL', 'AL-RF', 'DualLoop'] labels = ['WeSAL', 'AL-RF', 'DualLoop'] measurement_list = [] for exp in methods: costMap = {} experiment = results[exp] previous_gain = 0 for iteration in experiment: if iteration == 0: continue m = experiment[iteration] gain = m['num_true_matches'] cost = m['human_effort'] if gain == 0 and cost == 0: continue if gain not in costMap: for j in range(gain, previous_gain, -1): measurement = { 'methods': exp, 'gain': j, 'cost': cost } measurement_list.append(measurement) costMap[gain] = cost previous_gain = gain exp_result_df = pd.DataFrame(measurement_list) fig, ax = plt.subplots(figsize=(6, 5)) p = sns.lineplot(x='gain', y='cost', hue='methods', ax=ax, data=exp_result_df, style="methods", markers=True) p.set_xlabel("gain") p.set_ylabel("cost") ax.title.set_text(dataset) plt.savefig(filename, transparent=False) plt.close(fig) def read_config(config_file): with open(config_file, 'r') as f: config = json.load(f) return config if __name__ == '__main__': # Count the arguments arguments = len(sys.argv) - 1 if arguments < 1: print("please specify the data set [conference|ai4eu|nasa|anatomy]") exit() else: dataset_name = sys.argv[1] # read the configuration file cfg = read_config('./config.json') # read the results results = {} for method_name in cfg['methods']: results[method_name] = pickle.load(open('./results/' + dataset_name + "/result_" + method_name + "_" + dataset_name + ".pk", 'rb')) plot_f1_result(results, dataset_name, './results/' + "result_" + dataset_name + '_f1.png') plot_cost_over_gain(results, dataset_name, './results/' + "result_" + dataset_name + '_cost_over_gain.png')