test_model_utils.py 7.79 KB
Newer Older
1
# @author lucasmiranda42
2
3
4
5
6
7
8
9
# encoding: utf-8
# module deepof

"""

Testing module for deepof.model_utils

"""
10
11
12
13

from hypothesis import given
from hypothesis import settings
from hypothesis import strategies as st
lucas_miranda's avatar
lucas_miranda committed
14
from hypothesis.extra.numpy import arrays
lucas_miranda's avatar
lucas_miranda committed
15
import deepof.model_utils
16
import numpy as np
lucas_miranda's avatar
lucas_miranda committed
17
import tensorflow as tf
18
import tensorflow_probability as tfp
lucas_miranda's avatar
lucas_miranda committed
19
from tensorflow.python.framework.ops import EagerTensor
lucas_miranda's avatar
lucas_miranda committed
20

lucas_miranda's avatar
lucas_miranda committed
21
22
23
24
# For coverage.py to work with @tf.function decorated functions and methods,
# graph execution is disabled when running this script with pytest

tf.config.experimental_run_functions_eagerly(True)
25
26
tfpl = tfp.layers
tfd = tfp.distributions
lucas_miranda's avatar
lucas_miranda committed
27

lucas_miranda's avatar
lucas_miranda committed
28
29
30
31

@settings(deadline=None)
@given(
    shape=st.tuples(
32
        st.integers(min_value=5, max_value=10), st.integers(min_value=5, max_value=10)
lucas_miranda's avatar
lucas_miranda committed
33
34
35
    )
)
def test_far_away_uniform_initialiser(shape):
lucas_miranda's avatar
lucas_miranda committed
36
    far = deepof.model_utils.far_away_uniform_initialiser(shape, 0, 15, 1000)
lucas_miranda's avatar
lucas_miranda committed
37
    random = tf.random.uniform(shape, 0, 15)
lucas_miranda's avatar
lucas_miranda committed
38
    assert far.shape == shape
lucas_miranda's avatar
lucas_miranda committed
39
40
41
    assert tf.abs(tf.norm(tf.math.subtract(far[1:], far[:1]))) > tf.abs(
        tf.norm(tf.math.subtract(random[1:], random[:1]))
    )
lucas_miranda's avatar
lucas_miranda committed
42
43


lucas_miranda's avatar
lucas_miranda committed
44
45
46
@settings(deadline=None)
@given(
    tensor=arrays(
lucas_miranda's avatar
lucas_miranda committed
47
48
49
50
51
        shape=(10, 10),
        dtype=float,
        unique=True,
        elements=st.floats(min_value=-300, max_value=300),
    ),
lucas_miranda's avatar
lucas_miranda committed
52
53
54
55
56
57
58
59
60
61
62
)
def test_compute_mmd(tensor):
    tensor1 = tf.cast(tf.convert_to_tensor(tensor), dtype=tf.float32)
    tensor2 = tf.random.uniform(tensor1.shape, -300, 300, dtype=tf.float32)

    mmd_kernel = deepof.model_utils.compute_mmd(tuple([tensor1, tensor2]))
    null_kernel = deepof.model_utils.compute_mmd(tuple([tensor1, tensor1]))

    assert type(mmd_kernel) == EagerTensor
    assert null_kernel == 0

lucas_miranda's avatar
lucas_miranda committed
63

lucas_miranda's avatar
lucas_miranda committed
64
65
66
67
68
69
def test_one_cycle_scheduler():
    cycle1 = deepof.model_utils.one_cycle_scheduler(
        iterations=5, max_rate=1.0, start_rate=0.1, last_iterations=2, last_rate=0.3
    )
    assert type(cycle1._interpolate(1, 2, 0.2, 0.5)) == float

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])

    test_model = tf.keras.Sequential()
    test_model.add(tf.keras.layers.Dense(1))

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    onecycle = deepof.model_utils.one_cycle_scheduler(
        X.shape[0] // 100 * 10, max_rate=0.005,
    )

    fit = test_model.fit(X, y, callbacks=[onecycle], epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History
    assert onecycle.history["lr"][4] > onecycle.history["lr"][1]
    assert onecycle.history["lr"][4] > onecycle.history["lr"][-1]

lucas_miranda's avatar
lucas_miranda committed
89

90
def test_uncorrelated_features_constraint():
91
92
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])
lucas_miranda's avatar
lucas_miranda committed
93

94
    correlations = []
lucas_miranda's avatar
lucas_miranda committed
95

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    for w in range(2):
        test_model = tf.keras.Sequential()
        test_model.add(
            tf.keras.layers.Dense(
                10,
                kernel_constraint=tf.keras.constraints.UnitNorm(axis=1),
                activity_regularizer=deepof.model_utils.uncorrelated_features_constraint(
                    2, weightage=w
                ),
            )
        )

        test_model.compile(
            loss=tf.keras.losses.binary_crossentropy,
            optimizer=tf.keras.optimizers.SGD(),
        )

        fit = test_model.fit(X, y, epochs=10, batch_size=100)
        assert type(fit) == tf.python.keras.callbacks.History

        correlations.append(np.mean(np.corrcoef(test_model.get_weights()[0])))

    assert correlations[0] > correlations[1]


def test_MCDropout():
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])

    test_model = tf.keras.Sequential()
    test_model.add(tf.keras.layers.Dense(10))
    test_model.add(deepof.model_utils.MCDropout(0.5))

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History


def test_dense_transpose():
    X = np.random.uniform(0, 10, [1500, 10])
    y = np.random.randint(0, 2, [1500, 1])

    dense_1 = tf.keras.layers.Dense(10)

    dense_input = tf.keras.layers.Input(shape=(10,))
    dense_test = dense_1(dense_input)
    dense_tran = deepof.model_utils.DenseTranspose(dense_1, output_dim=10)(dense_test)
    test_model = tf.keras.Model(dense_input, dense_tran)

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History


156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def test_KLDivergenceLayer():
    X = tf.random.uniform([1500, 10], 0, 10)
    y = np.random.randint(0, 2, [1500, 1])

    prior = tfd.Independent(
        tfd.Normal(loc=tf.zeros(10), scale=1,), reinterpreted_batch_ndims=1,
    )

    dense_1 = tf.keras.layers.Dense(10)

    i = tf.keras.layers.Input(shape=(10,))
    d = dense_1(i)
    x = tfpl.DistributionLambda(
        lambda dense: tfd.Independent(
            tfd.Normal(loc=dense, scale=1,), reinterpreted_batch_ndims=1,
        )
    )(d)
    x = deepof.model_utils.KLDivergenceLayer(
        prior, weight=tf.keras.backend.variable(1.0, name="kl_beta")
    )(x)
    test_model = tf.keras.Model(i, x)

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History


def test_MMDiscrepancyLayer():
    X = tf.random.uniform([1500, 10], 0, 10)
    y = np.random.randint(0, 2, [1500, 1])

    prior = tfd.Independent(
lucas_miranda's avatar
lucas_miranda committed
191
        tfd.Normal(loc=tf.zeros(10), scale=1,), reinterpreted_batch_ndims=1,
192
193
194
195
196
197
198
199
    )

    dense_1 = tf.keras.layers.Dense(10)

    i = tf.keras.layers.Input(shape=(10,))
    d = dense_1(i)
    x = tfpl.DistributionLambda(
        lambda dense: tfd.Independent(
lucas_miranda's avatar
lucas_miranda committed
200
            tfd.Normal(loc=dense, scale=1,), reinterpreted_batch_ndims=1,
201
202
        )
    )(d)
lucas_miranda's avatar
lucas_miranda committed
203

204
205
206
207
208
209
210
211
212
213
214
215
216
    x = deepof.model_utils.MMDiscrepancyLayer(
        100, prior, beta=tf.keras.backend.variable(1.0, name="kl_beta")
    )(x)
    test_model = tf.keras.Model(i, x)

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History


lucas_miranda's avatar
lucas_miranda committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
def test_dead_neuron_control():
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])

    test_model = tf.keras.Sequential()
    test_model.add(tf.keras.layers.Dense(1))
    test_model.add(deepof.model_utils.Dead_neuron_control())

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History


233
234
235
236
237
238
239
240
241
242
243
244
245
246
def test_entropy_regulariser():
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])

    test_model = tf.keras.Sequential()
    test_model.add(tf.keras.layers.Dense(1))
    test_model.add(deepof.model_utils.Entropy_regulariser(1.0))

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )

    fit = test_model.fit(X, y, epochs=10, batch_size=100)
    assert type(fit) == tf.python.keras.callbacks.History
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262


def test_find_learning_rate():
    X = np.random.uniform(0, 10, [1500, 5])
    y = np.random.randint(0, 2, [1500, 1])

    test_model = tf.keras.Sequential()
    test_model.add(tf.keras.layers.Dense(1))
    test_model.add(deepof.model_utils.Entropy_regulariser(1.0))

    test_model.compile(
        loss=tf.keras.losses.binary_crossentropy, optimizer=tf.keras.optimizers.SGD(),
    )
    test_model.build(X.shape)

    deepof.model_utils.find_learning_rate(test_model, X, y)