Commit 14c672d7 authored by Matteo.Guardiani's avatar Matteo.Guardiani
Browse files

cleanup: put MaternKernelModel into specific class. Separated component...

cleanup: put MaternKernelModel into specific class. Separated component creation into individual methods.
parent 19e8a379
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <>.
# Copyright(C) 2013-2022 Max-Planck-Society
# Author: Matteo Guardiani
# NIFTy is being developed at the Max-Planck-Institut fuer Astrophysik.
import argparse
import json
import os
import sys
import nifty7 as ift
import numpy as np
from mpi4py import MPI
n_task = comm.Get_size()
rank = comm.Get_rank()
except ImportError:
comm = None
n_task = 1
rank = 0
master = (rank == 0)
from covid_matern_model import covid_matern_model_maker
from data_utilities import save_KL_sample, save_KL_position
from utilities import get_op_post_mean
from const import npix_age, npix_ll
from data import Data
from covid_matern_model2 import MaternCausalModel
# from evidence_g import get_evidence
import matplotlib.colors as colors
# Parser Setup
parser = argparse.ArgumentParser()
parser.add_argument('--json_file', type=str, required=True) # FIXME: Add help --help
parser.add_argument('--csv_file', type=str, required=True)
parser.add_argument('--reshuffle_parameter', type=int, required=True)
args = parser.parse_args()
json_file = args.json_file
csv_file = args.csv_file
reshuffle_iterator = args.reshuffle_parameter
if __name__ == '__main__':
# Read in the configuration file
current_path = os.path.abspath('.')
inversion_parameter = False
if 'inv' in json_file:
inversion_parameter = True
file_setup = open(json_file, "r")
setup = json.load(file_setup)
# Preparing the filename string and plots folder to store live results
if not os.path.exists('./plots'):
filename = "plots/covid_combined_matern_{}.png"
# Results Output Folders
path_j = os.path.basename(json_file)
path_c = os.path.basename(csv_file)
results_path = os.path.join('./Automized_Results_Matern', os.path.splitext(path_j)[0], os.path.splitext(path_c)[0],
results_path = os.path.normpath(results_path)
os.makedirs(results_path, exist_ok=True)
# Load the model
data = Data(npix_age, npix_ll, json_file, reshuffle_iterator, inversion_parameter, csv_file)
model = MaternCausalModel(data, False)
# Setup the response & define the amplitudes
R = ift.GeometryRemover(
R_lamb = R(model.lambda_combined)
A1 = model.amplitudes[0]
A2 = model.amplitudes[1]
# Specify data space
data_space =
# Generate mock signal and data
seed = setup['seed']
if setup['mock']:
# data
mock_position = ift.from_random(model.lambda_combined.domain, 'normal')
data = R_lamb(mock_position)
data = ift.random.current_rng().poisson(data.val.astype(np.float64))
indep_tag = '_indep'
if not setup['same data'] and indep_tag in json_file:
print("\nUsing syinthetic data generated from joint model on independent model")
joint_json_file = json_file.replace(indep_tag, '')
file_setup = open(joint_json_file, "r")
joint_setup = json.load(file_setup)
joint_model = MaternCausalModel(data, False)
joint_model = covid_matern_model_maker(npix_age, npix_ll, joint_setup, csv_file, reshuffle_iterator, False,
joint_lamb_comb = joint_model['combined lambda']
mock_position = ift.from_random(joint_lamb_comb.domain, 'normal')
data = R_lamb(mock_position)
data = ift.random.current_rng().poisson(data.val.astype(np.float64))
if not setup['same data'] and not indep_tag in json_file:
print("\nUsing syinthetic data generated from independent model on joint model")
indep_json_file = os.path.splitext(json_file)[0] + '_indep' + os.path.splitext(json_file)[1]
file_setup = open(indep_json_file, "r")
indep_setup = json.load(file_setup)
indep_model = covid_matern_model_maker(npix_age, npix_ll, indep_setup, csv_file, reshuffle_iterator, False,
indep_lamb_comb = indep_model['combined lambda']
mock_position = ift.from_random(indep_lamb_comb.domain, 'normal')
data = R_lamb(mock_position)
data = ift.random.current_rng().poisson(data.val.astype(np.float64))
data = ift.makeField(data_space, data)
if setup['mock']:
plot = ift.Plot()
plot.add(lamb_comb(mock_position), title='Full Field')
plot.add(R.adjoint(data), title='Data')
plot.add([A1.force(mock_position)], title='Power Spectrum 1')
plot.add([A2.force(mock_position)], title='Power Spectrum 2')
plot.output(ny=3, nx=2, xsize=10, ysize=10, name=filename.format("setup"))
# Minimization parameters
ic_sampling = ift.AbsDeltaEnergyController(deltaE=1e-5, iteration_limit=250, convergence_level=250)
ic_newton = ift.AbsDeltaEnergyController(deltaE=1e-5, iteration_limit=5, name='newton', convergence_level=3)
minimizer = ift.NewtonCG(ic_newton, enable_logging=True)
# Set up likelihood and information Hamiltonian
likelihood = ift.PoissonianEnergy(data) @ R_lamb
H = ift.StandardHamiltonian(likelihood, ic_sampling)
# Begin minimization
initial_mean = ift.from_random(H.domain, 'normal') * 0.1
mean = initial_mean
N_steps = 35 # 34
for i in range(N_steps):
if i < 27:
ic_newton = ift.AbsDeltaEnergyController(deltaE=1e-5, iteration_limit=10, name='newton',
ic_newton = ift.AbsDeltaEnergyController(deltaE=1e-5, iteration_limit=20, name='newton',
minimizer = ift.NewtonCG(ic_newton, enable_logging=True)
if i < 30:
N_samples = 5
elif i < 33:
N_samples = 20
N_samples = 500
# Draw new samples and minimize KL
KL = ift.MetricGaussianKL(mean, H, N_samples, comm=comm, mirror_samples=True, nanisinf=True)
KL, convergence = minimizer(KL)
samples = tuple(KL.samples)
mean = KL.position
if master:
it = 0
pos_path = os.path.join(results_path, "KL_position")
save_KL_position(mean, pos_path)
print("KL position saved", file=sys.stderr)
sam_path = os.path.join(results_path, "samples")
os.makedirs(sam_path, exist_ok=True)
for sample in samples:
save_KL_sample(sample, os.path.join(sam_path, "KL_sample_{}".format(it)))
it += 1
print("KL samples saved", file=sys.stderr)
# Minisanity check
ift.extra.minisanity(data, lambda x: ift.makeOp(R_lamb(x).ptw('reciprocal')), R_lamb, mean,
samples) # Fix Me: Check noise implementation in minisanity
# Plot current reconstruction
plot = ift.Plot()
if setup['mock']:
plot.add([lamb_comb(mock_position)], title="ground truth")
plot.add(R.adjoint(data), title='Data')
plot.add([lamb_comb(mean)], title="reconstruction")
plot.add([lamb_joint.force(mean)], title="Joint component")
plot.add([A1.force(mean), A1.force(mock_position)], title="power1")
plot.add([A2.force(mean), A2.force(mock_position)], title="power2")
plot.add([ic_newton.history, ic_sampling.history, minimizer.inversion_history],
label=['KL', 'Sampling', 'Newton inversion'], title='Cumulative energies', s=[None, None, 1],
alpha=[None, 0.2, None])
plot.add([lamb_comb(mean)], title="Reconstruction", norm=colors.SymLogNorm(linthresh=10e-1),
extent=boundaries, aspect="auto")
plot.add([lamb_full.force(mean)], title="Joint Component Reconstruction",
norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries, aspect="auto")
plot.add([cond_prob.force(mean)], title="Conditional Probability Reconstruction",
norm=colors.SymLogNorm(linthresh=6 * 10e-4), extent=boundaries, aspect="auto")
# plot.add([Aj.force(mean)], title="power1 joint") # FIX ME: MAYBE ACCOUNT FOR THE MARGINALIZATION ??
plot.add([A1.force(mean)], title="power1 independent")
plot.add([A2.force(mean)], title="power2 independent")
plot.add(lamb_ag_full.force(mean), title="Age Reconstruction (full)", aspect="auto")
plot.add(lamb_ll_full.force(mean), title="Log load Reconstruction (full)", aspect="auto")
plot.output(nx=3, ny=3, ysize=10, xsize=15, name=filename.format("loop_{:02d}".format(i)))
print('Lamb combined check:', lamb_comb(KL.position).val.sum(), '\n', file=sys.stderr)
print('Zm Xi:', zm.force(KL.position).val, '\n', file=sys.stderr)
if master:
lamb_comb_mean, lamb_comb_var = get_op_post_mean(lamb_comb, mean, samples)
cond_prob_mean, cond_prob_var = get_op_post_mean(cond_prob, mean, samples)
lamb_full_mean, lamb_full_var = get_op_post_mean(lamb_full.exp(), mean, samples)
powers1 = []
powers2 = []
for sample in samples:
p1 = A1.force(sample + mean)
p2 = A2.force(sample + mean)
# Final Plots
filename_res = "Results.png"
filename_res = os.path.join(results_path, filename_res)
plot = ift.Plot()
plot.add(lamb_comb_mean, title="Posterior Mean", norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries,
plot.add(lamb_comb_var.sqrt(), title="Posterior Standard Deviation", norm=colors.SymLogNorm(linthresh=10e-1),
extent=boundaries, aspect="auto")
plot.add([cond_prob.force(mean)], title="Conditional Probability Reconstruction",
norm=colors.SymLogNorm(linthresh=6 * 10e-4), extent=boundaries, aspect="auto")
plot.add([A1.force(mean)], title="Age Independent Power Spectrum (log[S(k^2)])")
plot.add([A2.force(mean)], title="Log load Independent Power Spectrum (log[S(k^2)])")
plot.add(lamb_ag_full.force(mean), title="Age Reconstruction (full)", norm=colors.SymLogNorm(linthresh=10e-1),
plot.add(lamb_ll_full.force(mean), title="Log load Reconstruction (full)",
norm=colors.SymLogNorm(linthresh=10e-1), aspect="auto")
plot.add([lamb_full.exp().force(mean)], title="Joint Component Reconstruction (full)",
norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries, aspect="auto")
plot.output(ny=3, nx=3, xsize=20, ysize=15, name=filename_res)
print("Saved results as", filename_res, file=sys.stderr)
# Error Plots
filename_ers = "Errors.png"
filename_ers = os.path.join(results_path, filename_ers)
plot = ift.Plot()
plot.add(lamb_comb_mean, title="Posterior Mean", norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries,
plot.add(lamb_comb_var.sqrt() * lamb_comb_mean.ptw('reciprocal'), title="Relative Uncertainty",
norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries, aspect="auto")
plot.add(cond_prob_mean, title="Conditional Probability Reconstruction Mean",
norm=colors.SymLogNorm(linthresh=6 * 10e-4), extent=boundaries, aspect="auto")
plot.add(cond_prob_var.sqrt() * cond_prob_mean.ptw('reciprocal'),
title="Relative Uncertainty on Conditional Probability Reconstruction",
norm=colors.SymLogNorm(linthresh=10e-1), extent=boundaries, aspect="auto")
plot.add(lamb_full_mean, title="Joint Component Reconstruction Mean", norm=colors.SymLogNorm(linthresh=10e-2),
plot.add(lamb_full_var.sqrt() * lamb_full_mean.ptw('reciprocal'),
title="Relative Uncertainty on Joint Component Reconstruction", norm=colors.SymLogNorm(linthresh=10e-2),
plot.output(ny=3, nx=2, xsize=15, ysize=15, name=filename_ers)
print("Saved results as", filename_ers, file=sys.stderr)
# Get the evidence # Uncomment the following to compute the evidence directly from this script. # evidence =
# get_evidence(KL, data=data) # print("EVIDENCE", file=sys.stderr) # print(evidence, file=sys.stderr) # #
# print('\n', file=sys.stderr)
# # if dataset_file: # # shutil.copy(dataset_file, results_path)
# if json_file: # shutil.copy(json_file, results_path)
# with open(os.path.join(results_path,'evidence.txt'), 'wb') as file: # # file.write('EVIDENCE CALCULATION
# \n\n') # pickle.dump(evidence, file)
# with open('./Automized_Results_Matern/Automized_Evidences.txt', 'a') as file: # evidences = # #
# os.path.basename(json_file) + ', ' + os.path.basename(csv_file) + '_' + str(resh_it) + ', Evidence Mean: '
# + str(evidence['estimate']) + '\n\n' # file.write(evidences)
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment