import numpy as np import tensorflow as tf import keras from keras.models import Model, load_model import keras.backend as K from model import get_sum_model import os, pickle from model import * from datetime import date l_exp = [0.00018, 0.00033, 0.00050, 0.00054, 0.00062, 0.00067, 0.00075, 0.00085, 0.00103, 0.00138, 0.00145, 0.00156, 0.00170, 0.00224, 0.00224, 0.00229, 0.00259, 0.00264, 0.00264, 0.00266, 0.00290, 0.00290, 0.00373, 0.00392, 0.00410, 0.00425, 0.00438, 0.00445, 0.00466, 0.00474, 0.00492, 0.00503, 0.00503, 0.00508, 0.00531, 0.00532, 0.00550, 0.00552, 0.00570, 0.00584, 0.00642, 0.00642, 0.00679, 0.00692, 0.00693, 0.00697, 0.00719, 0.00733, 0.00735, 0.00740, 0.00743, 0.00749, 0.00782, 0.00790, 0.00810, 0.00812, 0.00828, 0.00856, 0.00870, 0.00876, 0.00884, 0.00889, 0.00922, 0.00933, 0.00987, 0.00989, 0.00999, 0.01031, 0.01043, 0.01090, 0.01115, 0.01127, 0.01131, 0.01134, 0.01171, 0.01187, 0.01189, 0.01190, 0.01199, 0.01228, 0.01234, 0.01255, 0.01259, 0.01266, 0.01275, 0.01305, 0.01326, 0.01326, 0.01398, 0.01437, 0.01456, 0.01461, 0.01473, 0.01479, 0.01496, 0.01503, 0.01517, 0.01537, 0.01542, 0.01567, 0.01579, 0.01588, 0.01591, 0.01597, 0.01631, 0.01639, 0.01663, 0.01682, 0.01690, 0.01712, 0.01714, 0.01717, 0.01766, 0.01774, 0.01790, 0.01818, 0.01820, 0.01822, 0.01835, 0.01835, 0.01842, 0.01848, 0.01852, 0.01893, 0.01913, 0.01914, 0.01944, 0.01963, 0.01971, 0.02006, 0.02011, 0.02015, 0.02033, 0.02042, 0.02104, 0.02111, 0.02121, 0.02137, 0.02141, 0.02187, 0.02194, 0.02203, 0.02224, 0.02247, 0.02278, 0.02286, 0.02336, 0.02344, 0.02351, 0.02364, 0.02365, 0.02381, 0.02381, 0.02455, 0.02460, 0.02463, 0.02466, 0.02470, 0.02476, 0.02490, 0.02498, 0.02520, 0.02529, 0.02551, 0.02595, 0.02618, 0.02634, 0.02646, 0.02653, 0.02654, 0.02656, 0.02658, 0.02682, 0.02752, 0.02757, 0.02764, 0.02769, 0.02770, 0.02809, 0.02817, 0.02825, 0.02832, 0.02835, 0.02836, 0.02838, 0.02890, 0.02897, 0.02898, 0.02900, 0.02933, 0.02972, 0.02973, 0.02995, 0.02999, 0.03025, 0.03026, 0.03034, 0.03072, 0.03077, 0.03110, 0.03115, 0.03142, 0.03148, 0.03169, 0.03182, 0.03211, 0.03215, 0.03220, 0.03227, 0.03250, 0.03276, 0.03288, 0.03290, 0.03318, 0.03319, 0.03334, 0.03344, 0.03381, 0.03408, 0.03444, 0.03446, 0.03476, 0.03502, 0.03516, 0.03558, 0.03587, 0.03598, 0.03629, 0.03670, 0.03680, 0.03695, 0.03708, 0.03709, 0.03740, 0.03767, 0.03773, 0.03807, 0.03809, 0.03817, 0.03835, 0.03843, 0.03845, 0.03849, 0.03874, 0.03879, 0.03887, 0.03956, 0.03964, 0.03965, 0.04002, 0.04019, 0.04037, 0.04054, 0.04077, 0.04101, 0.04104, 0.04117, 0.04120, 0.04136, 0.04162, 0.04172, 0.04219, 0.04219, 0.04233, 0.04249, 0.04249, 0.04250, 0.04271, 0.04285, 0.04315, 0.04318, 0.04319, 0.04324, 0.04337, 0.04342, 0.04344, 0.04348, 0.04359, 0.04362, 0.04366, 0.04381, 0.04415, 0.04422, 0.04446, 0.04559, 0.04591, 0.04617, 0.04646, 0.04652, 0.04713, 0.04776, 0.04788, 0.04799, 0.04847, 0.04853, 0.04907, 0.05049, 0.05128, 0.05136, 0.05141, 0.05148, 0.05153, 0.05158, 0.05203, 0.05219, 0.05236, 0.05261, 0.05275, 0.05301, 0.05305, 0.05341, 0.05384, 0.05435, 0.05440, 0.05481, 0.05502, 0.05511, 0.05529, 0.05538, 0.05578, 0.05635, 0.05647, 0.05656, 0.05669, 0.05700, 0.05719, 0.05755, 0.05781, 0.05806, 0.05846, 0.05878, 0.05886, 0.05915, 0.05919, 0.05934, 0.05963, 0.05980, 0.06000, 0.06037, 0.06044, 0.06092, 0.06116, 0.06155, 0.06167, 0.06171, 0.06175, 0.06184, 0.06229, 0.06243, 0.06256, 0.06275, 0.06319, 0.06340, 0.06345, 0.06364, 0.06380, 0.06390, 0.06391, 0.06395, 0.06406, 0.06408, 0.06451, 0.06453, 0.06509, 0.06514, 0.06582, 0.06586, 0.06608, 0.06609, 0.06612, 0.06624, 0.06634, 0.06641, 0.06669, 0.06680, 0.06697, 0.06701, 0.06745, 0.06749, 0.06754, 0.06775, 0.06778, 0.06790, 0.06793, 0.06796, 0.06806, 0.06816, 0.06834, 0.06841, 0.06864, 0.06888, 0.06910, 0.06914, 0.06929, 0.06929, 0.06930, 0.06943, 0.06945, 0.06977, 0.07034, 0.07068, 0.07069, 0.07081, 0.07108, 0.07113, 0.07123, 0.07131, 0.07133, 0.07134, 0.07191, 0.07217, 0.07228, 0.07243, 0.07347, 0.07365, 0.07368, 0.07373, 0.07383, 0.07385, 0.07445, 0.07446, 0.07478, 0.07503, 0.07556, 0.07562, 0.07575, 0.07581, 0.07629, 0.07636, 0.07667, 0.07685, 0.07703, 0.07789, 0.07790, 0.07837, 0.07848, 0.07947, 0.07982, 0.07993, 0.08068, 0.08082, 0.08110, 0.08115, 0.08159, 0.08264, 0.08278, 0.08329, 0.08355, 0.08366, 0.08367, 0.08417, 0.08419, 0.08419, 0.08475, 0.08485, 0.08504, 0.08522, 0.08532, 0.08540, 0.08550, 0.08555, 0.08573, 0.08597, 0.08608, 0.08641, 0.08685, 0.08705, 0.08732, 0.08741, 0.08769, 0.08771, 0.08820, 0.08834, 0.08855, 0.08896, 0.08899, 0.08917, 0.08924, 0.08924, 0.08926, 0.08945, 0.08982, 0.09027, 0.09043, 0.09080, 0.09104, 0.09161, 0.09201, 0.09201, 0.09264, 0.09280, 0.09299, 0.09374, 0.09420, 0.09426, 0.09459, 0.09465, 0.09493, 0.09509, 0.09512, 0.09551, 0.09552, 0.09620, 0.09626, 0.09643, 0.09759, 0.09772, 0.09772, 0.09839, 0.09848, 0.09848, 0.09859, 0.09861, 0.09871, 0.09937, 0.09948, 0.10015, 0.10035, 0.10038, 0.10070, 0.10081, 0.10098, 0.10101, 0.10110, 0.10144, 0.10195, 0.10241, 0.10272, 0.10316, 0.10349, 0.10393, 0.10396, 0.10419, 0.10427, 0.10494, 0.10497, 0.10499, 0.10520, 0.10520, 0.10530, 0.10575, 0.10576, 0.10581, 0.10591, 0.10647, 0.10649, 0.10737, 0.10791, 0.10794, 0.10824, 0.10855, 0.10876, 0.10883, 0.10924, 0.10993, 0.10997, 0.11029, 0.11029, 0.11061, 0.11106, 0.11126, 0.11185, 0.11191, 0.11199, 0.11212, 0.11270, 0.11305, 0.11315, 0.11315, 0.11354, 0.11432, 0.11477, 0.11480, 0.11587, 0.11594, 0.11597, 0.11618, 0.11667, 0.11670, 0.11710, 0.11747, 0.11775, 0.11786, 0.11798, 0.11821, 0.11851, 0.11901, 0.11909, 0.11915, 0.11995, 0.12029, 0.12048, 0.12056, 0.12056, 0.12171, 0.12198, 0.12220, 0.12245, 0.12254, 0.12355, 0.12480, 0.12494, 0.12543, 0.12546, 0.12573, 0.12577, 0.12629, 0.12641, 0.12693, 0.12724, 0.12726, 0.12814, 0.12822, 0.12840, 0.12850, 0.12893, 0.12953, 0.13016, 0.13103, 0.13116, 0.13125, 0.13134, 0.13146, 0.13243, 0.13266, 0.13373, 0.13391, 0.13393, 0.13444, 0.13453, 0.13463, 0.13612, 0.13667, 0.13729, 0.13770, 0.13814, 0.13830, 0.13886, 0.13887, 0.14018, 0.14041, 0.14085, 0.14130, 0.14135, 0.14149, 0.14338, 0.14343, 0.14343, 0.14382, 0.14392, 0.14414, 0.14436, 0.14480, 0.14622, 0.14648, 0.14741, 0.14751, 0.14843, 0.14914, 0.14916, 0.15002, 0.15031, 0.15071, 0.15089, 0.15261, 0.15269, 0.15276, 0.15346, 0.15419, 0.15441, 0.15449, 0.15460, 0.15469, 0.15547, 0.15585, 0.15604, 0.15630, 0.15726, 0.15733, 0.15782, 0.15829, 0.15863, 0.15902, 0.16016, 0.16052, 0.16078, 0.16080, 0.16105, 0.16123, 0.16213, 0.16344, 0.16359, 0.16416, 0.16420, 0.16444, 0.16451, 0.16477, 0.16484, 0.16558, 0.16578, 0.16670, 0.16691, 0.16706, 0.16744, 0.16762, 0.16829, 0.16847, 0.16855, 0.16861, 0.16916, 0.16941, 0.17016, 0.17056, 0.17115, 0.17125, 0.17192, 0.17240, 0.17240, 0.17291, 0.17309, 0.17336, 0.17392, 0.17441, 0.17459, 0.17575, 0.17584, 0.17596, 0.17611, 0.17772, 0.17856, 0.17880, 0.17900, 0.17918, 0.18094, 0.18204, 0.18205, 0.18252, 0.18316, 0.18333, 0.18372, 0.18373, 0.18376, 0.18603, 0.18621, 0.18629, 0.18687, 0.18742, 0.18922, 0.18931, 0.19010, 0.19055, 0.19137, 0.19137, 0.19152, 0.19244, 0.19284, 0.19348, 0.19403, 0.19454, 0.19471, 0.19490, 0.19503, 0.19681, 0.19722, 0.19834, 0.19936, 0.20010, 0.20218, 0.20262, 0.20304, 0.20341, 0.20376, 0.20494, 0.20504, 0.20524, 0.20528, 0.20552, 0.20615, 0.20621, 0.20670, 0.20753, 0.20828, 0.21067, 0.21167, 0.21188, 0.21321, 0.21412, 0.21487, 0.21691, 0.21713, 0.21728, 0.21790, 0.21886, 0.21890, 0.21946, 0.21970, 0.22132, 0.22228, 0.22340, 0.22543, 0.22549, 0.22582, 0.22633, 0.22640, 0.22658, 0.22682, 0.22688, 0.22695, 0.22727, 0.22943, 0.23070, 0.23096, 0.23261, 0.23409, 0.23462, 0.23523, 0.23592, 0.23609, 0.23819, 0.23969, 0.24000, 0.24036, 0.24059, 0.24337, 0.24433, 0.24445, 0.24503, 0.24626, 0.24634, 0.24687, 0.24783, 0.24974, 0.25048, 0.25057, 0.25070, 0.25339, 0.25411, 0.25454, 0.25607, 0.25814, 0.25841, 0.25868, 0.25904, 0.25951, 0.26214, 0.26357, 0.26394, 0.26436, 0.26478, 0.26763, 0.26850, 0.26996, 0.27076, 0.27411, 0.27419, 0.27478, 0.27551, 0.27553, 0.27553, 0.27752, 0.27876, 0.27977, 0.28000, 0.28081, 0.28220, 0.28277, 0.28291, 0.28313, 0.28359, 0.28492, 0.28576, 0.28774, 0.28780, 0.28837, 0.28945, 0.28984, 0.29330, 0.29752, 0.29760, 0.29926, 0.30028, 0.30215, 0.30398, 0.30678, 0.30809, 0.30847, 0.30918, 0.30951, 0.31014, 0.31042, 0.31134, 0.31136, 0.31214, 0.31524, 0.31588, 0.31632, 0.31666, 0.32025, 0.32260, 0.32451, 0.32487, 0.32531, 0.32670, 0.32697, 0.32800, 0.32843, 0.33150, 0.33363, 0.33445, 0.33569, 0.33674, 0.33830, 0.33872, 0.33933, 0.34315, 0.34444, 0.34686, 0.34787, 0.35183, 0.35225, 0.35287, 0.35857, 0.35875, 0.36265, 0.36269, 0.36482, 0.37436, 0.37446, 0.37729, 0.37846, 0.37895, 0.38758, 0.39093, 0.39110, 0.39129, 0.39369, 0.39817, 0.39861, 0.40254, 0.40260, 0.40305, 0.40424, 0.40445, 0.40554, 0.41336, 0.41625, 0.41912, 0.42133, 0.42574, 0.42887, 0.42893, 0.43552, 0.43912, 0.44271, 0.45022, 0.45049, 0.45158, 0.45470, 0.45552, 0.45860, 0.46377, 0.46500, 0.46722, 0.47138, 0.47613, 0.48200, 0.49551, 0.50962, 0.52365, 0.52400, 0.53008, 0.53431, 0.53801, 0.53885, 0.53996, 0.54616, 0.55390, 0.55697, 0.56252, 0.56381, 0.57993, 0.58727, 0.60287, 0.61361, 0.64448, 0.65605, 0.66612, 0.66887, 0.68660, 0.68949, 0.69482, 0.70933, 0.75816, 0.78771, 0.80919, 0.84408, 0.86884, 0.96250, 0.97324, 1.00000] primes = [11, 9433, 42533, 102931] def prod(nss, nd): return np.prod(nss[:nd]) def shrink(X, c, l): cm = (int)(l*(1-c)) idremove = [] lenlost = 0.0 i = 0 while(True): x = list(np.where(X[:,0]==i)[0]) if len(idremove) + len(x) < cm: idremove += x lenlost += len(x) * i i+=1 else: lenlost += (cm-len(idremove)) * i idremove += x[:cm-len(idremove)] break; X = np.delete(X, idremove, 0) del idremove[:] return X def rdn(date): y = date[0] m = date[1] d = date[2] if m < 3: y -= 1 m += 12 return 365 * y + y / 4 - y / 100 + y / 400 + (153 * m - 457) / 5 + d - 306 def parse_date(s): return list(map(int, s.split('-'))) def date_to_string(s): return str(s.year) + '-' + str(s.month) + '-' + str(s.day) def dif_str(str1, str2, type): if type == "R": vd = float(str1) - float(str2) elif type == "D": date1 = parse_date(str1) date2 = parse_date(str2) vd = (date(date1[0], date1[1], date1[2]) - date(date2[0], date2[1], date2[2])).days else: vd = 1 return vd def parse_keys(keys, type): if type == "R": return np.array([float(key) for key in keys]) elif type == "D": keys = [parse_date(key) for key in keys] return np.array([(date(key[0], key[1], key[2]) - date(1900, 1, 1)).days for key in keys]) else: return np.array(keys) def extract_emb(iris_model, model_fnm, neb=128, ncd=0, nd=2): vf = iris_model.get_layer('lambda_7').output model_vf = Model(iris_model.layers[0].input, vf) emb = {} xs = np.zeros((1, 2*(ncd+2), nd + 1)) for id in range(neb * neb): xs.fill(0) xs[0, 0, 0] = 1 xs[0,-2:,0] = 1 for d in range(nd): xs[0, 0, d + 1] = id % neb id = int(id / neb) xs[0, -2, :] = xs[0, 0, :] xs[0, :, 1:] = np.maximum(xs[0, :, 1:], -1) + 1 v = model_vf.predict(xs)[0] emb[','.join([str(int(s)) for s in xs[0,0,1:]])] = v pickle.dump(emb, open('tmp/emb-' + os.path.splitext(os.path.basename(model_fnm))[0] + '.pkl', 'wb')) return emb def my_loss_fn(y_true, y_pred): y_true = K.cast(y_true, 'float32') y_pred = K.cast(y_pred, 'float32') return K.mean(K.max(y_true/y_pred, y_true/y_pred), axis=-1) def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))