Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
MPIBP-Hummer
BioEM
Commits
a53afc0b
Commit
a53afc0b
authored
Jul 19, 2017
by
Luka Stanisic
Browse files
additional statistical summary of the execution together with the improved overall output
parent
bad8337a
Changes
4
Hide whitespace changes
Inline
Side-by-side
bioem.cpp
View file @
a53afc0b
...
...
@@ -103,7 +103,7 @@ ostream& operator<<(ostream& os, const vector<T>& v)
bioem
::
bioem
()
{
FFTAlgo
=
getenv
(
"FFTALGO"
)
==
NULL
?
1
:
atoi
(
getenv
(
"FFTALGO"
));
DebugOutput
=
getenv
(
"BIOEM_DEBUG_OUTPUT"
)
==
NULL
?
2
:
atoi
(
getenv
(
"BIOEM_DEBUG_OUTPUT"
));
DebugOutput
=
getenv
(
"BIOEM_DEBUG_OUTPUT"
)
==
NULL
?
0
:
atoi
(
getenv
(
"BIOEM_DEBUG_OUTPUT"
));
nProjectionsAtOnce
=
getenv
(
"BIOEM_PROJECTIONS_AT_ONCE"
)
==
NULL
?
1
:
atoi
(
getenv
(
"BIOEM_PROJECTIONS_AT_ONCE"
));
Autotuning
=
false
;
}
...
...
@@ -562,6 +562,9 @@ int bioem::run()
int
iOrientEnd
=
(
int
)
((
long
long
int
)
(
mpi_rank
+
1
)
*
param
.
nTotGridAngles
/
mpi_size
);
if
(
iOrientEnd
>
param
.
nTotGridAngles
)
iOrientEnd
=
param
.
nTotGridAngles
;
/* Vectors for computing statistic on different parts of the code */
TimeStat
ts
((
iOrientEnd
-
iOrientStart
),
param
.
nTotCTFs
);
if
(
DebugOutput
>=
1
)
ts
.
InitTimeStat
(
4
);
// **************************Loop Over orientations***************************************
...
...
@@ -569,8 +572,11 @@ int bioem::run()
{
// ***************************************************************************************
// ***** Creating Projection for given orientation and transforming to Fourier space *****
if
(
DebugOutput
>=
1
)
timer2
.
ResetStart
();
if
(
DebugOutput
>=
2
)
timer
.
ResetStart
();
if
(
DebugOutput
>=
1
)
{
timer2
.
ResetStart
();
timer
.
ResetStart
();
}
int
iTmpEnd
=
std
::
min
(
iOrientEnd
,
iOrientAtOnce
+
nProjectionsAtOnce
);
// **************************Parallel orientations for projections at once***************
...
...
@@ -580,8 +586,12 @@ int bioem::run()
{
createProjection
(
iOrient
,
&
proj_mapsFFT
[(
iOrient
-
iOrientAtOnce
)
*
ProjMapSize
]);
}
if
(
DebugOutput
>=
2
)
printf
(
"
\t
Time Projection %d: %f (rank %d)
\n
"
,
iOrientAtOnce
,
timer
.
GetCurrentElapsedTime
(),
mpi_rank
);
if
(
DebugOutput
>=
1
)
{
ts
.
time
=
timer
.
GetCurrentElapsedTime
();
ts
.
Add
(
TS_PROJECTION
);
if
(
DebugOutput
>=
2
)
printf
(
"
\t
Time Projection %d: %f (rank %d)
\n
"
,
iOrientAtOnce
,
ts
.
time
,
mpi_rank
);
}
/* Recalibrate if needed */
if
(
Autotuning
&&
((
iOrientAtOnce
-
iOrientStart
)
%
RECALIB_FACTOR
==
0
)
&&
((
iOrientEnd
-
iOrientAtOnce
)
>
RECALIB_FACTOR
)
&&
(
iOrientAtOnce
!=
iOrientStart
))
{
...
...
@@ -599,52 +609,69 @@ int bioem::run()
for
(
int
iConv
=
0
;
iConv
<
param
.
nTotCTFs
;
iConv
++
)
{
// *** Calculating convolutions of projection map and crosscorrelations ***
if
(
DebugOutput
>=
2
)
timer
.
ResetStart
();
if
(
DebugOutput
>=
1
)
timer
.
ResetStart
();
createConvolutedProjectionMap
(
iOrient
,
iConv
,
proj_mapFFT
,
conv_map
,
conv_mapFFT
,
sumCONV
,
sumsquareCONV
);
if
(
DebugOutput
>=
2
)
printf
(
"
\t\t
Time Convolution %d %d: %f (rank %d)
\n
"
,
iOrient
,
iConv
,
timer
.
GetCurrentElapsedTime
(),
mpi_rank
);
if
(
DebugOutput
>=
1
)
{
ts
.
time
=
timer
.
GetCurrentElapsedTime
();
ts
.
Add
(
TS_CONVOLUTION
);
if
(
DebugOutput
>=
2
)
printf
(
"
\t\t
Time Convolution %d %d: %f (rank %d)
\n
"
,
iOrient
,
iConv
,
ts
.
time
,
mpi_rank
);
}
if
((
DebugOutput
>=
2
)
||
(
Autotuning
&&
aut
.
Needed
(
iConv
)))
timer
.
ResetStart
();
myfloat_t
amp
,
pha
,
env
;
if
((
DebugOutput
>=
1
)
||
(
Autotuning
&&
aut
.
Needed
(
iConv
)))
timer
.
ResetStart
();
myfloat_t
amp
,
pha
,
env
;
amp
=
param
.
CtfParam
[
iConv
].
pos
[
0
];
pha
=
param
.
CtfParam
[
iConv
].
pos
[
1
];
env
=
param
.
CtfParam
[
iConv
].
pos
[
2
];
amp
=
param
.
CtfParam
[
iConv
].
pos
[
0
];
pha
=
param
.
CtfParam
[
iConv
].
pos
[
1
];
env
=
param
.
CtfParam
[
iConv
].
pos
[
2
];
// ******************Internal loop over Reference images CUDA or OpenMP******************
// *** Comparing each calculated convoluted map with all experimental maps ***
compareRefMaps
(
iOrient
,
iConv
,
amp
,
pha
,
env
,
conv_map
,
conv_mapFFT
,
sumCONV
,
sumsquareCONV
);
double
compTime
=
0.
;
ts
.
time
=
0.
;
if
(
DebugOutput
>=
1
)
{
ts
.
time
=
timer
.
GetCurrentElapsedTime
();
ts
.
Add
(
TS_COMPARISON
);
}
if
(
DebugOutput
>=
2
)
{
compTime
=
timer
.
GetCurrentElapsedTime
();
const
int
nShifts
=
2
*
param
.
param_device
.
maxDisplaceCenter
/
param
.
param_device
.
GridSpaceCenter
+
1
;
const
double
nFlops
=
(
double
)
RefMap
.
ntotRefMap
*
(
double
)
nShifts
*
(
double
)
nShifts
*
(((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
5.
+
25.
)
/
compT
ime
;
(((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
5.
+
25.
)
/
ts
.
t
ime
;
const
double
nGBs
=
(
double
)
RefMap
.
ntotRefMap
*
(
double
)
nShifts
*
(
double
)
nShifts
*
(((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
2.
+
8.
)
*
(
double
)
sizeof
(
myfloat_t
)
/
compT
ime
;
const
double
nGBs2
=
(
double
)
RefMap
.
ntotRefMap
*
((
double
)
param
.
param_device
.
NumberPixels
*
(
double
)
param
.
param_device
.
NumberPixels
+
8.
)
*
(
double
)
sizeof
(
myfloat_t
)
/
compT
ime
;
(((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
((
double
)
param
.
param_device
.
NumberPixels
-
(
double
)
param
.
param_device
.
maxDisplaceCenter
/
2.
)
*
2.
+
8.
)
*
(
double
)
sizeof
(
myfloat_t
)
/
ts
.
t
ime
;
const
double
nGBs2
=
(
double
)
RefMap
.
ntotRefMap
*
((
double
)
param
.
param_device
.
NumberPixels
*
(
double
)
param
.
param_device
.
NumberPixels
+
8.
)
*
(
double
)
sizeof
(
myfloat_t
)
/
ts
.
t
ime
;
if
(
Autotuning
)
printf
(
"
\t\t
Time Comparison %d %d: %f sec (%f GFlops, %f GB/s (cached), %f GB/s, with GPU workload %d%%) (rank %d)
\n
"
,
iOrient
,
iConv
,
compT
ime
,
nFlops
/
1000000000.
,
nGBs
/
1000000000.
,
nGBs2
/
1000000000.
,
aut
.
Workload
(),
mpi_rank
);
else
printf
(
"
\t\t
Time Comparison %d %d: %f sec (%f GFlops, %f GB/s (cached), %f GB/s) (rank %d)
\n
"
,
iOrient
,
iConv
,
compT
ime
,
nFlops
/
1000000000.
,
nGBs
/
1000000000.
,
nGBs2
/
1000000000.
,
mpi_rank
);
if
(
Autotuning
)
printf
(
"
\t\t
Time Comparison %d %d: %f sec (%f GFlops, %f GB/s (cached), %f GB/s, with GPU workload %d%%) (rank %d)
\n
"
,
iOrient
,
iConv
,
ts
.
t
ime
,
nFlops
/
1000000000.
,
nGBs
/
1000000000.
,
nGBs2
/
1000000000.
,
aut
.
Workload
(),
mpi_rank
);
else
printf
(
"
\t\t
Time Comparison %d %d: %f sec (%f GFlops, %f GB/s (cached), %f GB/s) (rank %d)
\n
"
,
iOrient
,
iConv
,
ts
.
t
ime
,
nFlops
/
1000000000.
,
nGBs
/
1000000000.
,
nGBs2
/
1000000000.
,
mpi_rank
);
}
if
(
Autotuning
&&
aut
.
Needed
(
iConv
))
{
if
(
compT
ime
==
0.
)
compT
ime
=
timer
.
GetCurrentElapsedTime
();
aut
.
Tune
(
compT
ime
);
if
(
aut
.
Finished
()
&&
DebugOutput
>=
2
)
printf
(
"
\
t\
t
Optimal GPU workload %d%% (rank %d)
\n
"
,
aut
.
Workload
(),
mpi_rank
);
if
(
ts
.
t
ime
==
0.
)
ts
.
t
ime
=
timer
.
GetCurrentElapsedTime
();
aut
.
Tune
(
ts
.
t
ime
);
if
(
aut
.
Finished
()
&&
DebugOutput
>=
1
)
printf
(
"
\t
Optimal GPU workload %d%% (rank %d)
\n
"
,
aut
.
Workload
(),
mpi_rank
);
rebalanceWrapper
(
aut
.
Workload
());
}
}
if
(
DebugOutput
>=
1
)
{
printf
(
"
\t
Total time for projection %d: %f (rank %d)
\n
"
,
iOrient
,
timer2
.
GetCurrentElapsedTime
(),
mpi_rank
);
ts
.
time
=
timer2
.
GetCurrentElapsedTime
();
ts
.
Add
(
TS_TPROJECTION
);
printf
(
"
\t
Total time for projection %d: %f (rank %d)
\n
"
,
iOrient
,
ts
.
time
,
mpi_rank
);
timer2
.
ResetStart
();
}
}
}
/* Statistical summary on different parts of the code */
if
(
DebugOutput
>=
1
)
{
ts
.
PrintTimeStat
(
mpi_rank
);
ts
.
EmptyTimeStat
();
}
//deallocating fftw_complex vector
myfftw_free
(
proj_mapsFFT
);
myfftw_free
(
conv_mapFFT
);
...
...
include/timer.h
View file @
a53afc0b
...
...
@@ -11,6 +11,15 @@
#ifndef TIMER_H
#define TIMER_H
#include
<stdio.h>
#include
<string>
#include
<numeric>
#include
<vector>
#include
<algorithm>
#include
<cmath>
using
namespace
std
;
class
HighResTimer
{
public:
...
...
@@ -30,6 +39,40 @@ private:
double
ElapsedTime
;
double
StartTime
;
int
running
;
};
};
/* Structure for saving a vector of timings */
typedef
struct
_TimeLog
{
vector
<
double
>
vec
;
double
sum
;
double
stdev
;
string
name
;
}
TimeLog
;
enum
TS_NAMES
{
TS_TPROJECTION
,
TS_PROJECTION
,
TS_CONVOLUTION
,
TS_COMPARISON
};
/* Structure for saving timings of different parts of code and doing basic statistics on them */
class
TimeStat
{
public:
TimeStat
(
int
Angles
,
int
CTFs
)
:
time
(
0
),
tl
(
NULL
)
{
angles
=
Angles
;
ctfs
=
CTFs
;};
~
TimeStat
()
{
EmptyTimeStat
();};
void
InitTimeLog
(
int
log
,
int
size
,
string
s
);
void
InitTimeStat
(
int
nlogs
);
void
EmptyTimeStat
();
void
inline
Add
(
int
log
)
{
tl
[
log
].
vec
.
push_back
(
time
);};
void
ComputeTimeStat
();
void
PrintTimeStat
(
int
mpi_rank
);
/* Variable for storing times during the execution */
double
time
;
private:
TimeLog
*
tl
;
int
total_logs
;
int
angles
;
int
ctfs
;
};
#endif
map.cpp
View file @
a53afc0b
...
...
@@ -352,7 +352,7 @@ void bioem_Probability::init(size_t maps, size_t angles, size_t cc, bioem& bio)
nAngles
=
angles
;
nCC
=
cc
;
ptr
=
bio
.
malloc_device_host
(
get_size
(
maps
,
angles
,
cc
,
bio
.
param
.
param_device
.
writeAngles
,
bio
.
param
.
param_device
.
writeCC
));
cout
<<
"Allocation #Maps "
<<
maps
<<
" #Angles "
<<
angles
<<
" #cross.cor "
<<
cc
<<
"
\n
"
;
if
(
bio
.
DebugOutput
>=
1
)
cout
<<
"Allocation #Maps "
<<
maps
<<
" #Angles "
<<
angles
<<
" #cross.cor "
<<
cc
<<
"
\n
"
;
//<< " == " << get_size(maps, angles, cc, bio.param.param_device.writeAngles, bio.param.param_device.writeCC)<< "\n";
set_pointers
();
}
...
...
timer.cpp
View file @
a53afc0b
...
...
@@ -90,4 +90,59 @@ double HighResTimer::GetFrequency()
#endif
}
double
HighResTimer
::
Frequency
=
HighResTimer
::
GetFrequency
();
\ No newline at end of file
double
HighResTimer
::
Frequency
=
HighResTimer
::
GetFrequency
();
void
TimeStat
::
InitTimeLog
(
int
log
,
int
size
,
string
s
)
{
tl
[
log
].
vec
.
reserve
(
size
);
tl
[
log
].
name
=
s
;
tl
[
log
].
sum
=
0.
;
tl
[
log
].
stdev
=
0.
;
}
void
TimeStat
::
InitTimeStat
(
int
nlogs
)
{
total_logs
=
nlogs
;
tl
=
new
TimeLog
[
total_logs
];
InitTimeLog
(
TS_TPROJECTION
,
angles
,
"Total time of projection"
);
InitTimeLog
(
TS_PROJECTION
,
angles
,
"Projection"
);
InitTimeLog
(
TS_CONVOLUTION
,
angles
*
ctfs
,
"Convolution"
);
InitTimeLog
(
TS_COMPARISON
,
angles
*
ctfs
,
"Comparison"
);
}
void
TimeStat
::
EmptyTimeStat
()
{
if
(
tl
==
NULL
)
return
;
delete
[
]
tl
;
tl
=
NULL
;
time
=
0.
;
}
void
TimeStat
::
ComputeTimeStat
()
{
double
mean
,
sq_sum
;
vector
<
double
>
diff
;
for
(
int
i
=
0
;
i
<
total_logs
;
i
++
)
{
tl
[
i
].
sum
=
std
::
accumulate
(
tl
[
i
].
vec
.
begin
(),
tl
[
i
].
vec
.
end
(),
0.0
);
mean
=
tl
[
i
].
sum
/
tl
[
i
].
vec
.
size
();
diff
.
resize
(
tl
[
i
].
vec
.
size
());
std
::
transform
(
tl
[
i
].
vec
.
begin
(),
tl
[
i
].
vec
.
end
(),
diff
.
begin
(),
std
::
bind2nd
(
std
::
minus
<
double
>
(),
mean
));
sq_sum
=
std
::
inner_product
(
diff
.
begin
(),
diff
.
end
(),
diff
.
begin
(),
0.0
);
tl
[
i
].
stdev
=
std
::
sqrt
(
sq_sum
/
tl
[
i
].
vec
.
size
());
}
}
void
TimeStat
::
PrintTimeStat
(
int
mpi_rank
)
{
ComputeTimeStat
();
for
(
int
i
=
0
;
i
<
total_logs
;
i
++
)
{
printf
(
"SUMMARY -> %s: Total %f sec; Mean %f sec; Std.Dev. %f (rank %d)
\n
"
,
tl
[
i
].
name
.
c_str
(),
tl
[
i
].
sum
,
tl
[
i
].
sum
/
tl
[
i
].
vec
.
size
(),
tl
[
i
].
stdev
,
mpi_rank
);
}
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment