From 4a6952f03793de29eb2ee95beb9aaaee2cde4ac3 Mon Sep 17 00:00:00 2001
From: Thomas <purcell@fhi-berlin.mpg.de>
Date: Tue, 1 Jun 2021 22:14:24 +0200
Subject: [PATCH] Add hard_copy to the nodes and Residual reparameterization

hard_copy to ensure reparamterized nodes don't affect those already selected
---
 .../feature_space/FeatureSpace.cpp            | 25 +++++++++++++++++++
 src/feature_creation/node/FeatureNode.hpp     | 15 +++++++++++
 src/feature_creation/node/ModelNode.hpp       |  6 +++++
 src/feature_creation/node/Node.hpp            | 18 +++++++++++++
 .../node/operator_nodes/OperatorNode.hpp      |  6 +++++
 .../abs/absolute_value.cpp                    |  8 ++++++
 .../abs/absolute_value.hpp                    |  6 +++++
 .../abs/parameterized_absolute_value.cpp      |  9 +++++++
 .../abs/parameterized_absolute_value.hpp      |  6 +++++
 .../abs_diff/absolute_difference.cpp          |  8 ++++++
 .../abs_diff/absolute_difference.hpp          |  7 ++++++
 .../parameterized_absolute_difference.cpp     |  9 +++++++
 .../parameterized_absolute_difference.hpp     |  6 +++++
 .../allowed_operator_nodes/add/add.cpp        |  8 ++++++
 .../allowed_operator_nodes/add/add.hpp        |  6 +++++
 .../add/parameterized_add.cpp                 |  9 +++++++
 .../add/parameterized_add.hpp                 |  6 +++++
 .../allowed_operator_nodes/cb/cube.cpp        |  8 ++++++
 .../allowed_operator_nodes/cb/cube.hpp        |  6 +++++
 .../cb/parameterized_cube.cpp                 |  9 +++++++
 .../cb/parameterized_cube.hpp                 |  6 +++++
 .../allowed_operator_nodes/cbrt/cube_root.cpp |  8 ++++++
 .../allowed_operator_nodes/cbrt/cube_root.hpp |  6 +++++
 .../cbrt/parameterized_cube_root.cpp          |  9 +++++++
 .../cbrt/parameterized_cube_root.hpp          |  6 +++++
 .../allowed_operator_nodes/cos/cos.cpp        |  8 ++++++
 .../allowed_operator_nodes/cos/cos.hpp        |  6 +++++
 .../cos/parameterized_cos.cpp                 |  9 +++++++
 .../cos/parameterized_cos.hpp                 |  6 +++++
 .../allowed_operator_nodes/div/divide.cpp     |  8 ++++++
 .../allowed_operator_nodes/div/divide.hpp     |  6 +++++
 .../div/parameterized_divide.cpp              |  9 +++++++
 .../div/parameterized_divide.hpp              |  6 +++++
 .../exp/exponential.cpp                       |  8 ++++++
 .../exp/exponential.hpp                       |  6 +++++
 .../exp/parameterized_exponential.cpp         |  9 +++++++
 .../exp/parameterized_exponential.hpp         |  6 +++++
 .../allowed_operator_nodes/inv/inverse.cpp    |  8 ++++++
 .../allowed_operator_nodes/inv/inverse.hpp    |  6 +++++
 .../inv/parameterized_inverse.cpp             |  9 +++++++
 .../inv/parameterized_inverse.hpp             |  6 +++++
 .../allowed_operator_nodes/log/log.cpp        |  8 ++++++
 .../allowed_operator_nodes/log/log.hpp        |  6 +++++
 .../log/parameterized_log.cpp                 |  9 +++++++
 .../log/parameterized_log.hpp                 |  6 +++++
 .../allowed_operator_nodes/mult/multiply.cpp  |  8 ++++++
 .../allowed_operator_nodes/mult/multiply.hpp  |  6 +++++
 .../mult/parameterized_multiply.cpp           |  9 +++++++
 .../mult/parameterized_multiply.hpp           |  6 +++++
 .../neg_exp/negative_exponential.cpp          |  8 ++++++
 .../neg_exp/negative_exponential.hpp          |  6 +++++
 .../parameterized_negative_exponential.cpp    |  9 +++++++
 .../parameterized_negative_exponential.hpp    |  6 +++++
 .../sin/parameterized_sin.cpp                 |  9 +++++++
 .../sin/parameterized_sin.hpp                 |  6 +++++
 .../allowed_operator_nodes/sin/sin.cpp        |  8 ++++++
 .../allowed_operator_nodes/sin/sin.hpp        |  6 +++++
 .../six_pow/parameterized_sixth_power.cpp     |  9 +++++++
 .../six_pow/parameterized_sixth_power.hpp     |  6 +++++
 .../six_pow/sixth_power.cpp                   |  8 ++++++
 .../six_pow/sixth_power.hpp                   |  6 +++++
 .../sq/parameterized_square.cpp               |  9 +++++++
 .../sq/parameterized_square.hpp               |  6 +++++
 .../allowed_operator_nodes/sq/square.cpp      |  8 ++++++
 .../allowed_operator_nodes/sq/square.hpp      |  6 +++++
 .../sqrt/parameterized_square_root.cpp        |  9 +++++++
 .../sqrt/parameterized_square_root.hpp        |  6 +++++
 .../sqrt/square_root.cpp                      |  8 ++++++
 .../sqrt/square_root.hpp                      |  6 +++++
 .../sub/parameterized_subtract.cpp            |  9 +++++++
 .../sub/parameterized_subtract.hpp            |  6 +++++
 .../allowed_operator_nodes/sub/subtract.cpp   |  8 ++++++
 .../allowed_operator_nodes/sub/subtract.hpp   |  6 +++++
 src/python/bindings_docstring_keyed.hpp       |  5 ++++
 74 files changed, 569 insertions(+)

diff --git a/src/feature_creation/feature_space/FeatureSpace.cpp b/src/feature_creation/feature_space/FeatureSpace.cpp
index 98d1b362..66dee6c8 100644
--- a/src/feature_creation/feature_space/FeatureSpace.cpp
+++ b/src/feature_creation/feature_space/FeatureSpace.cpp
@@ -812,6 +812,31 @@ void FeatureSpace::project_generated(const double* prop, const int size, std::ve
 
 void FeatureSpace::sis(const std::vector<double>& prop)
 {
+    // Reparameterize for the residuals
+#ifdef PARAMETERIZE
+    if(_phi_selected.size() > 0)
+    {
+        // Make a hard copy of the previously selected features
+        for(int ff = _phi_selected.size() - _n_sis_select; ff < _phi_selected.size(); ++ff)
+        {
+            _phi_selected[ff] = _phi_selected[ff]->hard_copy();
+        }
+
+        // Reparameterize based on residuals
+        #pragma omp parallel
+        {
+            std::shared_ptr<NLOptimizer> optimizer = nlopt_wrapper::get_optimizer(_project_type, _task_sizes, prop, _max_phi, _max_param_depth);
+            #pragma omp for schedule(dynamic)
+            for(int ff = _start_gen[1]; ff < _phi.size(); ++ff)
+            {
+                if(_phi[ff]->n_params() > 0)
+                {
+                    _phi[ff]->get_parameters(optimizer);
+                }
+            }
+        }
+    }
+#endif
     // Create output directories if needed
     boost::filesystem::path p(_feature_space_file.c_str());
     boost::filesystem::create_directories(p.remove_filename());
diff --git a/src/feature_creation/node/FeatureNode.hpp b/src/feature_creation/node/FeatureNode.hpp
index 85d06b16..aa0c9d76 100644
--- a/src/feature_creation/node/FeatureNode.hpp
+++ b/src/feature_creation/node/FeatureNode.hpp
@@ -134,6 +134,12 @@ public:
      */
     ~FeatureNode();
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual inline node_ptr hard_copy()const {return std::make_shared<FeatureNode>(*this);}
+
     /**
      * @brief Get the list of feature expressions
      * @return vector storing the expressions for all primary features that show up in feature in the order they appear in the postfix notation
@@ -366,6 +372,15 @@ public:
      */
     inline void set_parameters(const double* params){};
 
+    //DocString: feat_node_get_params
+    /**
+     * @brief Solve the non-linear optimization to set the parameters
+     * @details Fits the data points from _feats->value_ptr and prop to get the parameters for the feature
+     *
+     * @param optimizer The optimizer used to get the paremeters
+     */
+    void get_parameters(std::shared_ptr<NLOptimizer> optimizer){};
+
     /**
      * @brief returns the number of parameters for this feature
      *
diff --git a/src/feature_creation/node/ModelNode.hpp b/src/feature_creation/node/ModelNode.hpp
index 0c19dbe1..4f6cbbbc 100644
--- a/src/feature_creation/node/ModelNode.hpp
+++ b/src/feature_creation/node/ModelNode.hpp
@@ -134,6 +134,12 @@ public:
      */
     ~ModelNode();
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    inline node_ptr hard_copy()const {return std::make_shared<ModelNode>(*this);}
+
     /**
      * @brief Generate the list of functions used to evaluate the value of this feature for a new data point from the postfix expression
      */
diff --git a/src/feature_creation/node/Node.hpp b/src/feature_creation/node/Node.hpp
index cbcd40dd..2ed4be8b 100644
--- a/src/feature_creation/node/Node.hpp
+++ b/src/feature_creation/node/Node.hpp
@@ -30,6 +30,9 @@ namespace py = boost::python;
 namespace np = boost::python::numpy;
 #endif
 
+// Forward Declaration of NLOptimizer
+class NLOptimizer;
+
 // DocString: cls_node
 /**
  * @brief Base class for a Node
@@ -113,6 +116,12 @@ public:
      */
     virtual ~Node();
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual std::shared_ptr<Node> hard_copy()const = 0;
+
     /**
      * @brief Get the list of feature expressions
      * @return vector storing the expressions for all primary features that show up in feature in the order they appear in the postfix notation
@@ -379,6 +388,15 @@ public:
      */
     virtual void set_parameters(const double* params) = 0;
 
+    //DocString: node_get_params
+    /**
+     * @brief Solve the non-linear optimization to set the parameters
+     * @details Fits the data points from _feats->value_ptr and prop to get the parameters for the feature
+     *
+     * @param optimizer The optimizer used to get the paremeters
+     */
+    virtual void get_parameters(std::shared_ptr<NLOptimizer> optimizer) = 0;
+
     /**
      * @brief returns the number of parameters for this feature
      *
diff --git a/src/feature_creation/node/operator_nodes/OperatorNode.hpp b/src/feature_creation/node/operator_nodes/OperatorNode.hpp
index 38e23fc2..db985149 100644
--- a/src/feature_creation/node/operator_nodes/OperatorNode.hpp
+++ b/src/feature_creation/node/operator_nodes/OperatorNode.hpp
@@ -112,6 +112,12 @@ public:
     ~OperatorNode()
     {}
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const = 0;
+
     /**
      * @brief Get the list of feature expressions
      * @return vector storing the expressions for all primary features that show up in feature in the order they appear in the postfix notation
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.cpp
index a152c40e..48cc8992 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.cpp
@@ -87,6 +87,14 @@ AbsNode::AbsNode(const node_ptr feat, const unsigned long int feat_ind, const do
     }
 }
 
+node_ptr AbsNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AbsNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void AbsNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.hpp
index da033d87..f37662c9 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/absolute_value.hpp
@@ -58,6 +58,12 @@ public:
      */
     AbsNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: abs_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.cpp
index dd374d96..2c166a95 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.cpp
@@ -72,6 +72,15 @@ AbsParamNode::AbsParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr AbsParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AbsParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void AbsParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     // Change the sign of alpha as a control on linear dependency without forcing one sign or another
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.hpp
index 6cbe1933..3be3d22f 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs/parameterized_absolute_value.hpp
@@ -90,6 +90,12 @@ public:
      */
     AbsParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: abs_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.cpp
index 4afc56de..b92ed490 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.cpp
@@ -124,6 +124,14 @@ AbsDiffNode::AbsDiffNode(const node_ptr feat_1, const node_ptr feat_2, const uns
     }
 }
 
+node_ptr AbsDiffNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AbsDiffNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void AbsDiffNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.hpp
index 5cdf23e0..6e17c5ad 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/absolute_difference.hpp
@@ -63,6 +63,13 @@ public:
      */
     AbsDiffNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
+
     // DocString: abs_diff_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.cpp
index f16c83ea..970bdddb 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.cpp
@@ -116,6 +116,15 @@ AbsDiffParamNode::AbsDiffParamNode(const node_ptr feat_1, const node_ptr feat_2,
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr AbsDiffParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AbsDiffParamNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void AbsDiffParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.hpp
index 4d1becc6..a1830dcb 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/abs_diff/parameterized_absolute_difference.hpp
@@ -92,6 +92,12 @@ public:
      */
     AbsDiffParamNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: abs_diff_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.cpp
index f6fd65ce..6d832484 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.cpp
@@ -118,6 +118,14 @@ AddNode::AddNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned lo
     }
 }
 
+node_ptr AddNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AddNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void AddNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     _feats[0]->update_add_sub_leaves(add_sub_leaves, pl_mn, expected_abs_tot);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.hpp
index e13dc19d..a532154d 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/add.hpp
@@ -60,6 +60,12 @@ public:
      */
     AddNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: add_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.cpp
index ff048f83..f33046b9 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.cpp
@@ -79,6 +79,15 @@ AddParamNode::AddParamNode(const node_ptr feat_1, const node_ptr feat_2, const u
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr AddParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<AddParamNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void AddParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.hpp
index 46125a7c..6071fedb 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/add/parameterized_add.hpp
@@ -93,6 +93,12 @@ public:
      */
     AddParamNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: add_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.cpp
index bc5b5ffe..394c6e0c 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.cpp
@@ -82,6 +82,14 @@ CbNode::CbNode(const node_ptr feat, const unsigned long int feat_ind, const doub
 
 }
 
+node_ptr CbNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CbNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void CbNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.hpp
index 59d1ca58..8412c6c2 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/cube.hpp
@@ -58,6 +58,12 @@ public:
      */
     CbNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: cb_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.cpp
index 7405f7a4..2aeb18d0 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.cpp
@@ -122,6 +122,15 @@ void CbParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
     }
 }
 
+node_ptr CbParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CbParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void CbNode::set_value(const double* params, int offset, const bool for_comp, const int depth) const
 {
     bool is_root = (offset == -1);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.hpp
index 245ac8fc..c7470e67 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cb/parameterized_cube.hpp
@@ -90,6 +90,12 @@ public:
      */
     CbParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: cb_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.cpp
index f2f8bf96..420dc6c8 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.cpp
@@ -87,6 +87,14 @@ CbrtNode::CbrtNode(const node_ptr feat, const unsigned long int feat_ind, const
     }
 }
 
+node_ptr CbrtNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CbrtNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void CbrtNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.hpp
index fdbb5811..94abff84 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/cube_root.hpp
@@ -58,6 +58,12 @@ public:
      */
     CbrtNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: cbrt_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.cpp
index 8004b651..e4762873 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.cpp
@@ -100,6 +100,15 @@ CbrtParamNode::CbrtParamNode(const node_ptr feat, const unsigned long int feat_i
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr CbrtParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CbrtParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void CbrtParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.hpp
index 9b1ef4e7..f2c47e1d 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cbrt/parameterized_cube_root.hpp
@@ -89,6 +89,12 @@ public:
      */
     CbrtParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: cbrt_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.cpp
index d68a5ed5..a7a9d70e 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.cpp
@@ -72,6 +72,14 @@ CosNode::CosNode(const node_ptr feat, const unsigned long int feat_ind, const do
     }
 }
 
+node_ptr CosNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CosNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void CosNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.hpp
index 981707b5..a159789b 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/cos.hpp
@@ -58,6 +58,12 @@ public:
      */
     CosNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: cos_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.cpp
index 1e021e0e..57b4d8ae 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.cpp
@@ -99,6 +99,15 @@ CosParamNode::CosParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr CosParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<CosParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void CosParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.hpp
index c1faab75..1b437577 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/cos/parameterized_cos.hpp
@@ -90,6 +90,12 @@ public:
      */
     CosParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: cos_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.cpp
index 75b9e0c3..5fd36ead 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.cpp
@@ -128,6 +128,14 @@ DivNode::DivNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned lo
 
 }
 
+node_ptr DivNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<DivNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void DivNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.hpp
index 3451ef8c..ca40eb12 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/divide.hpp
@@ -60,6 +60,12 @@ public:
      */
     DivNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: div_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.cpp
index 5cbf5402..2d6d45fe 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.cpp
@@ -106,6 +106,15 @@ DivParamNode::DivParamNode(const node_ptr feat_1, const node_ptr feat_2, const u
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr DivParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<DivParamNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void DivParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.hpp
index 66d8f8f7..b958eef9 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/div/parameterized_divide.hpp
@@ -93,6 +93,12 @@ public:
      */
     DivParamNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: div_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.cpp
index 4eb2a0a2..7a071ebc 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.cpp
@@ -84,6 +84,14 @@ ExpNode::ExpNode(const node_ptr feat, const unsigned long int feat_ind, const do
     set_test_value();
 }
 
+node_ptr ExpNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<ExpNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void ExpNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.hpp
index 8e24fa3d..e71f9906 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/exponential.hpp
@@ -58,6 +58,12 @@ public:
      */
     ExpNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: exp_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.cpp
index 406f9a1f..bda77447 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.cpp
@@ -111,6 +111,15 @@ ExpParamNode::ExpParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr ExpParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<ExpParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void ExpParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.hpp
index 985fed88..43c2845a 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/exp/parameterized_exponential.hpp
@@ -90,6 +90,12 @@ public:
      */
     ExpParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: exp_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.cpp
index 3e2d06ac..7d52017f 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.cpp
@@ -77,6 +77,14 @@ InvNode::InvNode(const node_ptr feat, const unsigned long int feat_ind, const do
 
 }
 
+node_ptr InvNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<InvNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void InvNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.hpp
index 16de6ce7..12d95a9a 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/inverse.hpp
@@ -54,6 +54,12 @@ public:
      */
     InvNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: inv_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.cpp
index fc14efc3..9126df9a 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.cpp
@@ -107,6 +107,15 @@ InvParamNode::InvParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr InvParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<InvParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void InvParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.hpp
index 3b46926a..c43f0187 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/inv/parameterized_inverse.hpp
@@ -90,6 +90,12 @@ public:
      */
     InvParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: inv_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.cpp
index 3ae33732..f11e1c7e 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.cpp
@@ -109,6 +109,14 @@ LogNode::LogNode(const node_ptr feat, const unsigned long int feat_ind, const do
     set_test_value();
 }
 
+node_ptr LogNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<LogNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void LogNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.hpp
index 5596bcff..503e524a 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/log.hpp
@@ -58,6 +58,12 @@ public:
      */
     LogNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: log_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.cpp
index 55f10032..eed292ff 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.cpp
@@ -135,6 +135,15 @@ LogParamNode::LogParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr LogParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<LogParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void LogParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.hpp
index 202dda03..1d139f89 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/log/parameterized_log.hpp
@@ -90,6 +90,12 @@ public:
      */
     LogParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: log_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.cpp
index 7f4744d2..a3433c36 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.cpp
@@ -127,6 +127,14 @@ MultNode::MultNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned
     }
 }
 
+node_ptr MultNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<MultNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void MultNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.hpp
index d6724144..6d146221 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/multiply.hpp
@@ -61,6 +61,12 @@ public:
      */
     MultNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: mult_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.cpp
index b54761ad..a3c83818 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.cpp
@@ -78,6 +78,15 @@ MultParamNode::MultParamNode(const node_ptr feat_1, const node_ptr feat_2, const
     get_parameters(optimizer);
 }
 
+node_ptr MultParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<MultParamNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void MultParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.hpp
index fb339e3f..0ab306c0 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/mult/parameterized_multiply.hpp
@@ -91,6 +91,12 @@ public:
      */
     MultParamNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: mult_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.cpp
index 2c03958f..d704e9ca 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.cpp
@@ -83,6 +83,14 @@ NegExpNode::NegExpNode(const node_ptr feat, const unsigned long int feat_ind, co
     }
 }
 
+node_ptr NegExpNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<NegExpNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void NegExpNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.hpp
index 87da93e7..850ae2c6 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/negative_exponential.hpp
@@ -59,6 +59,12 @@ public:
      */
     NegExpNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: neg_exp_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.cpp
index 7fd8a2da..039b3e90 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.cpp
@@ -111,6 +111,15 @@ NegExpParamNode::NegExpParamNode(const node_ptr feat, const unsigned long int fe
     _params.resize(n_params(),  0.0);
 }
 
+node_ptr NegExpParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<NegExpParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void NegExpParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.hpp
index 9d0ad8b3..5e40984d 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/neg_exp/parameterized_negative_exponential.hpp
@@ -90,6 +90,12 @@ public:
      */
     NegExpParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: neg_exp_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.cpp
index 20da40a7..0d480eaf 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.cpp
@@ -99,6 +99,15 @@ SinParamNode::SinParamNode(const node_ptr feat, const unsigned long int feat_ind
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr SinParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SinParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void SinParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.hpp
index cb713756..a57e8b05 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/parameterized_sin.hpp
@@ -90,6 +90,12 @@ public:
      */
     SinParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: sin_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.cpp
index 2e6064c9..8d99ebc5 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.cpp
@@ -61,6 +61,14 @@ SinNode::SinNode(const node_ptr feat, const unsigned long int feat_ind, const do
     }
 }
 
+node_ptr SinNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SinNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void SinNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.hpp
index 9ce624bd..0e641d28 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sin/sin.hpp
@@ -59,6 +59,12 @@ public:
      */
     SinNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: sin_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.cpp
index 7a47dd57..54fa1ce3 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.cpp
@@ -114,6 +114,15 @@ SixPowParamNode::SixPowParamNode(const node_ptr feat, const unsigned long int fe
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr SixPowParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SixPowParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void SixPowParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.hpp
index 262e17ae..0798bead 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/parameterized_sixth_power.hpp
@@ -90,6 +90,12 @@ public:
      */
     SixPowParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: six_pow_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.cpp
index 2ebf1eac..b6f0999e 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.cpp
@@ -90,6 +90,14 @@ SixPowNode::SixPowNode(const node_ptr feat, const unsigned long int feat_ind, co
     }
 }
 
+node_ptr SixPowNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SixPowNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void SixPowNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.hpp
index 1f55baf1..1dbf6138 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/six_pow/sixth_power.hpp
@@ -59,6 +59,12 @@ public:
      */
     SixPowNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: six_pow_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.cpp
index 3458c2da..a967cd98 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.cpp
@@ -105,6 +105,15 @@ SqParamNode::SqParamNode(const node_ptr feat, const unsigned long int feat_ind,
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr SqParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SqParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void SqParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.hpp
index 0f6cf20e..282d5027 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/parameterized_square.hpp
@@ -90,6 +90,12 @@ public:
      */
     SqParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: sq_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.cpp
index 7cf62256..a4f8a045 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.cpp
@@ -80,6 +80,14 @@ SqNode::SqNode(const node_ptr feat, const unsigned long int feat_ind, const doub
 
 }
 
+node_ptr SqNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SqNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void SqNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.hpp
index e9ddf6bc..ffffc816 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sq/square.hpp
@@ -58,6 +58,12 @@ public:
      */
     SqNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: sq_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.cpp
index 7d1d548f..3b2098c7 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.cpp
@@ -120,6 +120,15 @@ SqrtParamNode::SqrtParamNode(const node_ptr feat, const unsigned long int feat_i
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr SqrtParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SqrtParamNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void SqrtParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     // Change the sign of alpha as a control on linear dependency without forcing one sign or another
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.hpp
index f67b7fd8..3b44b479 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/parameterized_square_root.hpp
@@ -92,6 +92,12 @@ public:
      */
     SqrtParamNode(const node_ptr feat, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: sqrt_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.cpp
index 04ecd4b2..ddfbc9ee 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.cpp
@@ -89,6 +89,14 @@ SqrtNode::SqrtNode(const node_ptr feat, const unsigned long int feat_ind, const
 
 }
 
+node_ptr SqrtNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SqrtNode>(_feats[0]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void SqrtNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     std::string key = expr();
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.hpp
index 3206b869..c061e5e6 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sqrt/square_root.hpp
@@ -59,6 +59,12 @@ public:
      */
     SqrtNode(const node_ptr feat, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: sqrt_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.cpp
index af894157..acdc1f4d 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.cpp
@@ -79,6 +79,15 @@ SubParamNode::SubParamNode(const node_ptr feat_1, const node_ptr feat_2, const u
     _params.resize(n_params(), 0.0);
 }
 
+node_ptr SubParamNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SubParamNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    cp->set_parameters(_params.data());
+    return cp;
+}
+
 void SubParamNode::get_parameters(std::shared_ptr<NLOptimizer> optimizer)
 {
     double min_res = optimizer->optimize_feature_params(this);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.hpp
index f18c5821..d3a8e481 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/parameterized_subtract.hpp
@@ -93,6 +93,12 @@ public:
      */
     SubParamNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, double const l_bound=1e-50, const double u_bound=1e50);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    node_ptr hard_copy()const;
+
     // DocString: sub_param_node_set_value
     /**
      * @brief Set the values of the training data for the feature inside of the value storage arrays
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.cpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.cpp
index 8ee3d232..4fe55753 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.cpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.cpp
@@ -118,6 +118,14 @@ SubNode::SubNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned lo
     }
 }
 
+node_ptr SubNode::hard_copy()const
+{
+    node_ptr cp = std::make_shared<SubNode>(_feats[0]->hard_copy(), _feats[1]->hard_copy(), _feat_ind);
+    cp->set_selected(_selected);
+    cp->set_d_mat_ind(_d_mat_ind);
+    return cp;
+}
+
 void SubNode::update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const
 {
     _feats[0]->update_add_sub_leaves(add_sub_leaves, pl_mn, expected_abs_tot);
diff --git a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.hpp b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.hpp
index 6bcf0a90..308d1be2 100644
--- a/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.hpp
+++ b/src/feature_creation/node/operator_nodes/allowed_operator_nodes/sub/subtract.hpp
@@ -62,6 +62,12 @@ public:
      */
     SubNode(const node_ptr feat_1, const node_ptr feat_2, const unsigned long int feat_ind, const double l_bound, const double u_bound);
 
+    /**
+     * @brief Makes a hard copy node (all items in _feats are a new shared_ptr)
+     * @return A shared_ptr of a copy of this node with all members hard copied
+     */
+    virtual node_ptr hard_copy()const;
+
     // DocString: sub_node_unit
     /**
      * @brief Get the unit of the feature (combine the units of _feats)
diff --git a/src/python/bindings_docstring_keyed.hpp b/src/python/bindings_docstring_keyed.hpp
index 9c6f4f7c..304ffd95 100644
--- a/src/python/bindings_docstring_keyed.hpp
+++ b/src/python/bindings_docstring_keyed.hpp
@@ -59,6 +59,7 @@ namespace sisso
             struct NodeWrap :  Node, py::wrapper<Node>
             {
             public:
+                inline node_ptr hard_copy() const {return this->get_override("hard_copy")();}
                 inline std::string expr() const {return this->get_override("expr")();}
                 inline std::string expr(double*, int depth=1) const {return this->get_override("expr")();}
                 inline std::string get_latex_expr() const {return this->get_override("latex_expr")();}
@@ -87,6 +88,7 @@ namespace sisso
                 inline std::string get_postfix_term() const {return this->get_override("get_postfix_term")();}
                 inline void update_add_sub_leaves(std::map<std::string, int>& add_sub_leaves, const int pl_mn, int& expected_abs_tot) const {this->get_override("update_add_sub_leaves")();}
                 inline void update_div_mult_leaves(std::map<std::string, double>& div_mult_leaves, const double fact, double& expected_abs_tot) const {this->get_override("update_div_mult_leaves")();}
+                inline void get_parameters(std::shared_ptr<NLOptimizer> optimizer){this->get_override("get_parameters")();}
                 inline std::vector<double> parameters() const {return this->get_override("parameters")();}
                 inline void set_parameters(std::vector<double>, bool check_sz=true){this->get_override("set_parameters")();}
                 inline void set_parameters(const double*){this->get_override("set_parameters")();}
@@ -107,6 +109,7 @@ namespace sisso
             template<int N>
             struct OperatorNodeWrap : OperatorNode<N>, py::wrapper<OperatorNode<N>>
             {
+                inline node_ptr hard_copy() const {return this->get_override("hard_copy")();}
                 inline void set_value(int offset=-1, const bool for_comp=false) const {this->get_override("set_value")();}
                 inline void set_test_value(int offset=-1, const bool for_comp=false) const {this->get_override("set_test_value")();}
                 inline void set_value(const double* params, int offset=-1, bool for_comp=false, int depth=1) const {this->get_override("set_value")();}
@@ -138,6 +141,7 @@ namespace sisso
             struct NodeWrap :  Node, py::wrapper<Node>
             {
             public:
+                inline node_ptr hard_copy() const {return this->get_override("hard_copy")();}
                 inline std::string expr() const
                 {
                     return this->get_override("expr")();
@@ -246,6 +250,7 @@ namespace sisso
             template<int N>
             struct OperatorNodeWrap : OperatorNode<N>, py::wrapper<OperatorNode<N>>
             {
+                inline node_ptr hard_copy() const {return this->get_override("hard_copy")();}
                 inline void set_value(int offset=-1, const bool for_comp=false) const
                 {
                     this->get_override("set_value")();
-- 
GitLab