Commit 70eaf835 authored by Carl Poelking's avatar Carl Poelking
Browse files

Identifying memory issues: subgraphs.

parent 236376dc
......@@ -44,6 +44,7 @@ def flg_compute_k_flg(S1, S2):
return abs(det_12)**0.5/abs(det_1_2)**0.25, warn
def adjust_regularization(graphs, options):
print "Adjust eta, gamma based on %d graphs." % len(graphs)
# Adjust L-regularization
traces = []
for g in graphs:
......@@ -56,6 +57,7 @@ def adjust_regularization(graphs, options):
# Adjust S-regularization
traces = []
for g in graphs:
print g.label
if options['laplacian.hierarchical']:
k, travg_S1, travg_S2 = compare_graphs_hierarchical(g, g, options, return_traces=True)
else:
......@@ -87,7 +89,7 @@ def compare_graphs_kernelized(L1, L2, K12, options, zero_eps=1e-10, verbose=Fals
if eigvals[0] < -zero_eps:
e_min = np.min(eigvals)
e_max = np.max(eigvals)
if abs(e_min/e_max) > zero_eps**0.5:
if abs(e_min/e_max) > 0.01: #zero_eps**0.5:
#print K12
print "WARNING Eigvals < 0 (%+1.7e) (%+1.7e) (%+1.7e)" % (e_min, e_max, e_min/e_max)
......@@ -307,9 +309,9 @@ class ParticleGraph(object):
subgraphs.append([])
for i in range(self.D.shape[0]):
idcs_sub = np.where(self.D[i] < r_cut_l)[0]
D_sub = self.D[idcs_sub][:,idcs_sub]
L_sub = self.L[idcs_sub][:,idcs_sub]
P_sub = self.P[idcs_sub]
D_sub = self.D[idcs_sub][:,idcs_sub] # TODO Copies array => too memory-intensive
L_sub = self.L[idcs_sub][:,idcs_sub] # TODO Same here
P_sub = self.P[idcs_sub] # TODO Same here
subgraph = ParticleSubgraph(self, self.centers[i], r_cut_l, i, idcs_sub, P_sub, L_sub, D_sub)
subgraphs[-1].append(subgraph)
#print i
......@@ -388,7 +390,9 @@ class ParticleGraph(object):
spectrum.computePowerGradients()
spectrum.computeGlobal()
# Adapt spectrum
adaptor = kern.KernelAdaptorFactory[options_soap.get('kernel.adaptor')](options_soap)
adaptor = kern.KernelAdaptorFactory[options_soap.get('kernel.adaptor')](
options_soap,
types_global=options_descriptor['type_list'])
ix = adaptor.adapt(spectrum)
dim = ix.shape[1]
assert ix.shape[0] == n_atoms
......
import os
import numpy as np
import multiprocessing as mp
import functools as fct
import json
import datetime
import resource
def mp_compute_column_block(gi, gj_list, kfct):
"""
......@@ -20,7 +23,7 @@ def mp_compute_column_block(gi, gj_list, kfct):
krow.append(k)
return krow
def mp_compute_upper_triangle(kfct, g_list, n_procs, n_blocks, log=None, **kwargs):
def mp_compute_upper_triangle(kfct, g_list, n_procs, n_blocks, log=None, tstart_twall=(None,None), **kwargs):
"""
Compute kernel matrix computed from pairs of objects in object list
......@@ -32,6 +35,8 @@ def mp_compute_upper_triangle(kfct, g_list, n_procs, n_blocks, log=None, **kwarg
n_blocks: number of column blocks onto which computation is split
kwargs: keyword arguments supplied to kfct
"""
t_start = tstart_twall[0]
t_wall = tstart_twall[1]
dim = len(g_list)
kmat = np.zeros((dim,dim))
# Embed mp index in g-list objects
......@@ -39,12 +44,14 @@ def mp_compute_upper_triangle(kfct, g_list, n_procs, n_blocks, log=None, **kwarg
# Divide onto column blocks
col_idcs = np.arange(len(g_list))
col_div_list = np.array_split(col_idcs, n_blocks)
for col_div in col_div_list:
for col_div_idx, col_div in enumerate(col_div_list):
t_in = datetime.datetime.now()
# Column start, column end
c0 = col_div[0]
c1 = col_div[-1]+1
if log: log << "Column block [%d:%d]" % (c0, c1) << log.endl
if log: log << "Column block i[%d:%d] j[%d:%d]" % (0, c1, c0, c1) << log.endl
gj_list = g_list[c0:c1]
gi_list = g_list[0:c1]
# Prime kernel function
kfct_primed = fct.partial(
kfct,
......@@ -56,11 +63,29 @@ def mp_compute_upper_triangle(kfct, g_list, n_procs, n_blocks, log=None, **kwarg
gj_list=gj_list,
kfct=kfct_primed)
# Map & close
kmat_column_block = pool.map(mp_compute_column_block_primed, g_list)
kmat_column_block = np.array(kmat_column_block)
kmat[:,c0:c1] = kmat_column_block
npyfile = 'out.block_i_%d_%d_j_%d_%d.npy' % (0, c1, c0, c1)
# ... but first check for previous calculations of same slice
if npyfile in os.listdir('./'):
if log: log << "Load block from '%s'" % npyfile << log.endl
kmat_column_block = np.load(npyfile)
else:
kmat_column_block = pool.map(mp_compute_column_block_primed, gi_list)
kmat_column_block = np.array(kmat_column_block)
np.save(npyfile, kmat_column_block)
# Update kernel matrix
kmat[0:c1,c0:c1] = kmat_column_block
pool.close()
pool.join()
# Check time
t_out = datetime.datetime.now()
dt_block = t_out-t_in
if t_start and t_wall:
t_elapsed = t_out-t_start
if log: log << "Time elapsed =" << t_elapsed << " (wall time = %s) (maxmem = %d)" % (t_wall, resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) << log.endl
if col_div_idx+1 != len(col_div_list) and t_elapsed+dt_block > t_wall-dt_block:
log << "Wall time hit expected for next iteration, break ..." << log.endl
break
else: pass
return kmat
def json_load_utf8(file_handle):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment