From 09f6f6e6af14b055c3b0837b5df983c1e2d88aa5 Mon Sep 17 00:00:00 2001
From: Thomas <purcell@fhi-berlin.mpg.de>
Date: Thu, 3 Jun 2021 13:51:01 +0200
Subject: [PATCH] Test slots now seperated
Only primary features are stored in memory to get around the hard copy issue
Should not affect perfromance that much since test_values are rarely used
---
.../value_storage/nodes_value_containers.cpp | 25 ++++++-------------
.../value_storage/nodes_value_containers.hpp | 24 +++++++++++++++---
2 files changed, 27 insertions(+), 22 deletions(-)
diff --git a/src/feature_creation/node/value_storage/nodes_value_containers.cpp b/src/feature_creation/node/value_storage/nodes_value_containers.cpp
index 6155fad8..64bbc875 100644
--- a/src/feature_creation/node/value_storage/nodes_value_containers.cpp
+++ b/src/feature_creation/node/value_storage/nodes_value_containers.cpp
@@ -3,10 +3,12 @@
int node_value_arrs::N_SELECTED = 0;
int node_value_arrs::N_SAMPLES = 0;
int node_value_arrs::N_STORE_FEATURES = 0;
+int node_value_arrs::N_PRIMARY_FEATURES = 0;
int node_value_arrs::N_RUNGS_STORED = 0;
int node_value_arrs::N_SAMPLES_TEST = 0;
int node_value_arrs::MAX_N_THREADS = omp_get_max_threads();
int node_value_arrs::N_OP_SLOTS = 0;
+int node_value_arrs::N_OP_TEST_SLOTS = 0;
int node_value_arrs::N_PARAM_OP_SLOTS = 0;
int node_value_arrs::MAX_RUNG = 0;
@@ -57,8 +59,10 @@ void node_value_arrs::initialize_values_arr(
N_SAMPLES_TEST = n_samples_test;
N_RUNGS_STORED = 0;
N_STORE_FEATURES = n_primary_feat;
+ N_PRIMARY_FEATURES = n_primary_feat;
MAX_RUNG = max_rung;
N_OP_SLOTS = 2 * (static_cast<int>(std::pow(2, max_rung)) - 1);
+ N_OP_TEST_SLOTS = 2 * (static_cast<int>(std::pow(2, max_rung)) - 1);
VALUES_ARR = std::vector<double>(N_STORE_FEATURES * N_SAMPLES);
TEST_VALUES_ARR = std::vector<double>(N_STORE_FEATURES * N_SAMPLES_TEST);
@@ -139,9 +143,6 @@ void node_value_arrs::resize_values_arr(const int n_dims, const int n_feat, cons
VALUES_ARR.resize(N_STORE_FEATURES * N_SAMPLES);
VALUES_ARR.shrink_to_fit();
- TEST_VALUES_ARR.resize(N_STORE_FEATURES * N_SAMPLES_TEST);
- TEST_VALUES_ARR.shrink_to_fit();
-
if(use_temp)
{
N_OP_SLOTS = 2 * (static_cast<int>(std::pow(2, MAX_RUNG - N_RUNGS_STORED)) - 1);
@@ -151,20 +152,12 @@ void node_value_arrs::resize_values_arr(const int n_dims, const int n_feat, cons
TEMP_STORAGE_REG.resize(MAX_N_THREADS * (N_OP_SLOTS * N_STORE_FEATURES + 1), - 1);
TEMP_STORAGE_REG.shrink_to_fit();
-
- TEMP_STORAGE_TEST_ARR.resize(MAX_N_THREADS * (N_OP_SLOTS * N_STORE_FEATURES + 1) * N_SAMPLES_TEST);
- TEMP_STORAGE_TEST_ARR.shrink_to_fit();
-
- TEMP_STORAGE_TEST_REG.resize(MAX_N_THREADS * (N_OP_SLOTS * N_STORE_FEATURES + 1), - 1);
- TEMP_STORAGE_TEST_REG.shrink_to_fit();
}
else
{
N_OP_SLOTS = 0;
TEMP_STORAGE_ARR = {};
TEMP_STORAGE_REG = {};
- TEMP_STORAGE_TEST_ARR = {};
- TEMP_STORAGE_TEST_REG = {};
}
}
@@ -200,19 +193,15 @@ double* node_value_arrs::get_test_value_ptr(
const bool for_comp
)
{
- if(rung <= N_RUNGS_STORED)
+ if(rung == 0)
{
- if(arr_ind > N_STORE_FEATURES)
- {
- throw std::logic_error("Requested arr_ind (" + std::to_string(arr_ind) + ") is too high (max " + std::to_string(N_STORE_FEATURES) +")");
- }
return access_test_value_arr(arr_ind);
}
- int op_slot = get_op_slot(rung, offset, for_comp);
+ int op_slot = get_op_test_slot(rung, offset, for_comp);
temp_storage_test_reg(arr_ind, op_slot) = feat_ind;
return access_temp_storage_test(
- (arr_ind % N_STORE_FEATURES) + (op_slot % N_OP_SLOTS) * N_STORE_FEATURES + omp_get_thread_num() * (N_STORE_FEATURES * N_OP_SLOTS + 1)
+ (arr_ind % N_PRIMARY_FEATURES) + (op_slot % N_OP_TEST_SLOTS) * N_PRIMARY_FEATURES + omp_get_thread_num() * (N_PRIMARY_FEATURES * N_OP_TEST_SLOTS + 1)
);
}
diff --git a/src/feature_creation/node/value_storage/nodes_value_containers.hpp b/src/feature_creation/node/value_storage/nodes_value_containers.hpp
index 61f9748b..94e63791 100644
--- a/src/feature_creation/node/value_storage/nodes_value_containers.hpp
+++ b/src/feature_creation/node/value_storage/nodes_value_containers.hpp
@@ -39,10 +39,12 @@ namespace node_value_arrs
extern int N_SELECTED; //!< Number of features selected
extern int N_SAMPLES; //!< Number of training samples for each feature
extern int N_SAMPLES_TEST; //!< Number of test samples for each feature
+ extern int N_PRIMARY_FEATURES; //!< Number of primary features with stored values
extern int N_STORE_FEATURES; //!< Number of features with stored values
extern int N_RUNGS_STORED; //!< Number of rungs with values stored
extern int MAX_N_THREADS; //!< Get the maximum number of threads possible
extern int N_OP_SLOTS; //!< The number of possible operator slots
+ extern int N_OP_TEST_SLOTS; //!< The number of possible operator slots for the test set
extern int N_PARAM_OP_SLOTS; //!< The maximum number of possible operator slots
extern int MAX_RUNG; //!< The maximum rung for all features
@@ -162,6 +164,20 @@ namespace node_value_arrs
return std::abs(N_OP_SLOTS / (1 + !for_comp) - static_cast<int>(std::pow(2, MAX_RUNG - rung)) - offset);
}
+ /**
+ * @brief Get the operator slot associated with a given rung/offset for the test data
+ *
+ * @param rung Rung of the feature
+ * @param offset Offset used to prevent overwrites
+ * @param for_comp If true get a slot dedicated to comparing features
+ *
+ * @return The operator slot to use
+ */
+ inline int get_op_test_slot(const int rung, const int offset, const bool for_comp)
+ {
+ return std::abs(N_OP_TEST_SLOTS / (1 + !for_comp) - static_cast<int>(std::pow(2, MAX_RUNG - rung)) - offset);
+ }
+
/**
* @brief Get the parameter operator slot associated with a given rung/offset
*
@@ -202,7 +218,7 @@ namespace node_value_arrs
inline int& temp_storage_test_reg(const unsigned long int ind, const int op_slot=0)
{
return TEMP_STORAGE_TEST_REG[
- (ind % N_STORE_FEATURES) + (op_slot % N_OP_SLOTS) * N_STORE_FEATURES + omp_get_thread_num() * (N_STORE_FEATURES * N_OP_SLOTS + 1)
+ (ind % N_PRIMARY_FEATURES) + (op_slot % N_OP_TEST_SLOTS) * N_PRIMARY_FEATURES + omp_get_thread_num() * (N_PRIMARY_FEATURES * N_OP_TEST_SLOTS + 1)
];
}
@@ -238,9 +254,9 @@ namespace node_value_arrs
inline int& temp_storage_test_reg(const unsigned long int ind, const int rung, const int offset, const bool for_comp)
{
return TEMP_STORAGE_TEST_REG[
- (ind % N_STORE_FEATURES) +
- (get_op_slot(rung, offset, for_comp) % N_OP_SLOTS) * N_STORE_FEATURES +
- omp_get_thread_num() * (N_STORE_FEATURES * N_OP_SLOTS + 1)
+ (ind % N_PRIMARY_FEATURES) +
+ (get_op_slot(rung, offset, for_comp) % N_OP_TEST_SLOTS) * N_PRIMARY_FEATURES +
+ omp_get_thread_num() * (N_PRIMARY_FEATURES * N_OP_TEST_SLOTS + 1)
];
}
--
GitLab