From 9b33f13aabdb0efe2d5404dfe6166d2c8c5d2a7c Mon Sep 17 00:00:00 2001 From: crawlingcub <86861129+crawlingcub@users.noreply.github.com> Date: Wed, 13 Oct 2021 08:50:42 -0700 Subject: [PATCH] Fixing flaky test (#780) Summary: Running some tests by removing seeds revealed that tests such as `test_single_shapley_int_batch_scalar_float`, `test_single_shapley_int_batch_scalar_tensor_0d`, `test_single_shapley_int_batch_scalar_tensor_1d`, and `test_single_shapley_int_batch_scalar_tensor_int` are flaky. They almost fail 40-50% of the times. To fix these tests, I updated the `n_samples` to 2500 to fix the flakiness. The tests now pass consistently. This only affects the execution time of the test by ~1s. Please let me know if this change seems reasonable. Similar to https://github.com/pytorch/captum/issues/775. Pull Request resolved: https://github.com/pytorch/captum/pull/780 Reviewed By: NarineK, Reubend Differential Revision: D31587778 Pulled By: vivekmig fbshipit-source-id: 843adda41662e881a6bf81ebc4354d6f304e83ce --- tests/attr/test_shapley.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/attr/test_shapley.py b/tests/attr/test_shapley.py index a4e57a5d57..6d894fbec5 100644 --- a/tests/attr/test_shapley.py +++ b/tests/attr/test_shapley.py @@ -349,6 +349,7 @@ def _single_int_input_multi_sample_batch_scalar_shapley_assert( feature_mask=mask, perturbations_per_eval=(1,), target=None, + n_samples=2500, ) def _multi_input_batch_scalar_shapley_assert(self, func: Callable) -> None: