Skip to content
Snippets Groups Projects
Commit 538a1a8f authored by Maximilian Kurthen's avatar Maximilian Kurthen
Browse files

added plots from defense and benchmark scripts

parent bd6ade64
No related branches found
No related tags found
No related merge requests found
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
...@@ -15,6 +15,7 @@ args = parser.parse_args() ...@@ -15,6 +15,7 @@ args = parser.parse_args()
NAME = args.name NAME = args.name
FIRST_ID = args.first_id FIRST_ID = args.first_id
LAST_ID = args.last_id LAST_ID = args.last_id
SEED = args.seed
N_BINS = args.nbins N_BINS = args.nbins
NOISE_VAR = args.noise_var NOISE_VAR = args.noise_var
...@@ -24,9 +25,11 @@ VERBOSITY = args.verbosity ...@@ -24,9 +25,11 @@ VERBOSITY = args.verbosity
POWER_SPECTRUM_BETA_STR = args.power_spectrum_beta POWER_SPECTRUM_BETA_STR = args.power_spectrum_beta
POWER_SPECTRUM_F_STR = args.power_spectrum_f POWER_SPECTRUM_F_STR = args.power_spectrum_f
RHO = args.rho RHO = args.rho
REPETITIONS = args.repetitions
SCALE_MAX = args.scale_max SCALE_MAX = args.scale_max
SUBSAMPLE = args.subsample SUBSAMPLE = args.subsample
CONFIG = args.config CONFIG = args.config
if CONFIG is not None: if CONFIG is not None:
with open('./model_configurations.txt') as f: with open('./model_configurations.txt') as f:
...@@ -59,7 +62,6 @@ print( ...@@ -59,7 +62,6 @@ print(
SCALE_MAX, SCALE_MAX,
NAME)) NAME))
np.random.seed(1)
POWER_SPECTRUM_BETA = lambda q: eval(POWER_SPECTRUM_BETA_STR) POWER_SPECTRUM_BETA = lambda q: eval(POWER_SPECTRUM_BETA_STR)
POWER_SPECTRUM_F = lambda q: eval(POWER_SPECTRUM_F_STR) POWER_SPECTRUM_F = lambda q: eval(POWER_SPECTRUM_F_STR)
scale = (0, SCALE_MAX) scale = (0, SCALE_MAX)
...@@ -72,59 +74,61 @@ if os.path.isfile(prediction_file): ...@@ -72,59 +74,61 @@ if os.path.isfile(prediction_file):
prediction_file = './benchmark_predictions/{}_{}_{}.txt'.format( prediction_file = './benchmark_predictions/{}_{}_{}.txt'.format(
BENCHMARK, NAME, c) BENCHMARK, NAME, c)
np.random.seed(1)
sum_of_weights = 0 sum_of_weights = 0
weighted_correct = 0 weighted_correct = 0
for i in range(FIRST_ID-1, LAST_ID): for rep in range(REPETITIONS):
(x, y), true_direction, weight = get_pair( np.random.seed(rep)
i, BENCHMARK, subsample_size=SUBSAMPLE)
if true_direction == 0: for i in range(FIRST_ID-1, LAST_ID):
continue (x, y), true_direction, weight = get_pair(
i, BENCHMARK, subsample_size=SUBSAMPLE)
scaler = MinMaxScaler(scale) if true_direction == 0:
x, y = scaler.fit_transform(np.array((x, y)).T).T continue
bcm = bayesian_causal_model.cause_model_shallow.CausalModelShallow( scaler = MinMaxScaler(scale)
N_bins=N_BINS, x, y = scaler.fit_transform(np.array((x, y)).T).T
noise_var=NOISE_VAR,
rho=RHO, bcm = bayesian_causal_model.cause_model_shallow.CausalModelShallow(
power_spectrum_beta=POWER_SPECTRUM_BETA, N_bins=N_BINS,
power_spectrum_f=POWER_SPECTRUM_F, noise_var=NOISE_VAR,
) rho=RHO,
power_spectrum_beta=POWER_SPECTRUM_BETA,
bcm.set_data(x, y) power_spectrum_f=POWER_SPECTRUM_F,
)
H1 = bcm.get_evidence(direction=1, verbosity=VERBOSITY - 1)
H2 = bcm.get_evidence(direction=-1, verbosity=VERBOSITY - 1) bcm.set_data(x, y)
predicted_direction = 1 if int(H1 < H2) else -1
H1 = bcm.get_evidence(direction=1, verbosity=VERBOSITY - 1)
if predicted_direction == true_direction: H2 = bcm.get_evidence(direction=-1, verbosity=VERBOSITY - 1)
fore = colorama.Fore.GREEN predicted_direction = 1 if int(H1 < H2) else -1
weighted_correct += weight
else: if predicted_direction == true_direction:
fore = colorama.Fore.RED fore = colorama.Fore.GREEN
sum_of_weights += weight weighted_correct += weight
accuracy = weighted_correct / sum_of_weights else:
fore = colorama.Fore.RED
if VERBOSITY > 0: sum_of_weights += weight
print( accuracy = weighted_correct / sum_of_weights
'dataset {}, {} true direction: {}, predicted direction {}\n'
'H1: {:.2e},\n H2: {:.2e},\n{}' if VERBOSITY > 0:
'accuracy so far: {:.2f}'.format( print(
i, 'dataset {}, {} true direction: {}, predicted direction {}\n'
fore, 'H1: {:.2e},\n H2: {:.2e},\n{}'
true_direction, 'accuracy so far: {:.2f}'.format(
predicted_direction, i,
H1, fore,
H2, true_direction,
colorama.Style.RESET_ALL, predicted_direction,
accuracy)) H1,
H2,
with open(prediction_file, 'a') as f: colorama.Style.RESET_ALL,
f.write('{} {} {} {}\n'.format(i+1, predicted_direction, H1, H2)) accuracy))
with open(prediction_file, 'a') as f:
f.write('{} {} {} {}\n'.format(i+1, predicted_direction, H1, H2))
print('accuracy: {:.2f}'.format(accuracy)) print('accuracy: {:.2f}'.format(accuracy))
......
...@@ -22,6 +22,12 @@ LAST_ID = args.last_id ...@@ -22,6 +22,12 @@ LAST_ID = args.last_id
VERBOSITY = args.verbosity VERBOSITY = args.verbosity
SUBSAMPLE_SIZE = args.subsample SUBSAMPLE_SIZE = args.subsample
REPETITIONS = args.repetitions REPETITIONS = args.repetitions
BCM = args.do_bcm
ANM_HSIC = args.do_anm_hsic
ANM_MML = args.do_anm_mml
IGCI = args.do_igci
CGNN = args.do_cgnn
if LAST_ID is None: if LAST_ID is None:
LAST_ID = get_benchmark_default_length(BENCHMARK) LAST_ID = get_benchmark_default_length(BENCHMARK)
...@@ -79,12 +85,6 @@ weighted_correct_anm_mml = 0 ...@@ -79,12 +85,6 @@ weighted_correct_anm_mml = 0
weighted_correct_igci = 0 weighted_correct_igci = 0
weighted_correct_cgnn = 0 weighted_correct_cgnn = 0
BCM = 1
ANM_HSIC = 1
ANM_MML = 1
IGCI = 1
CGNN = 1
save_stdout = sys.stdout save_stdout = sys.stdout
if VERBOSITY == 0: if VERBOSITY == 0:
# silence output # silence output
...@@ -98,7 +98,7 @@ for rep in range(REPETITIONS): ...@@ -98,7 +98,7 @@ for rep in range(REPETITIONS):
# re-silence output # re-silence output
if VERBOSITY == 0: if VERBOSITY == 0:
sys.stdout = open('trash_stdout', 'w') sys.stdout = open('trash_stdout', 'w')
np.random.seed(rep*10) np.random.seed(rep)
# encountered some memory leakage problems, see if this can be avoided by # encountered some memory leakage problems, see if this can be avoided by
# restarting the matlab engine # restarting the matlab engine
del eng del eng
...@@ -128,7 +128,11 @@ for rep in range(REPETITIONS): ...@@ -128,7 +128,11 @@ for rep in range(REPETITIONS):
scaler = MinMaxScaler(feature_range=(0, 1)) scaler = MinMaxScaler(feature_range=(0, 1))
x_scaled, y_scaled = scaler.fit_transform(np.array((x, y)).T).T x_scaled, y_scaled = scaler.fit_transform(np.array((x, y)).T).T
pair_predictions = {'true_direction': true_direction} pair_predictions = {
'true_direction': true_direction,
'number': i,
'weight': weight
}
# test bcm # test bcm
if BCM: if BCM:
bcm = bayesian_causal_model.cause_model_shallow.CausalModelShallow( bcm = bayesian_causal_model.cause_model_shallow.CausalModelShallow(
...@@ -256,7 +260,7 @@ for rep in range(REPETITIONS): ...@@ -256,7 +260,7 @@ for rep in range(REPETITIONS):
sys.stdout = open('trash_stdout', 'w') sys.stdout = open('trash_stdout', 'w')
with open('predictions_{}.txt'.format(BENCHMARK), 'a') as f: with open('predictions_{}.txt'.format(BENCHMARK), 'a') as f:
f.write(str({i: pair_predictions}) + '\n') f.write(str(pair_predictions) + '\n')
# reactivate output # reactivate output
sys.stdout = save_stdout sys.stdout = save_stdout
print( print(
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment