diff --git a/test/test_operators/test_fisher_metric.py b/test/test_operators/test_fisher_metric.py
index 6aa0e84f25a79a80532c6f6a8269fb3d03c3b4d6..a787bc1a43cc0de6dd792a3d7f08aaf7ffc02aca 100644
--- a/test/test_operators/test_fisher_metric.py
+++ b/test/test_operators/test_fisher_metric.py
@@ -96,7 +96,8 @@ def energy_tester(pos, get_noisy_data, energy_initializer):
     lin = ift.Linearization.make_var(pos, want_metric=True)
     res2 = _to_array(energy(lin).metric(test_vec).val)
     np.testing.assert_allclose(res/std, res2/std, atol=5)
-    for factor in [0.01, 0.5, 2, 100]:
+    # Test whether one would detect a factor of 2 in the Fisher metric
+    for factor in [0.5, 2]:
         with pytest.raises(AssertionError):
             np.testing.assert_allclose(res/std, factor*res2/std, atol=5)