Deal with non-decimal float values in the unseen values calculator

NannyML · Aug 26, 2024 · 08fa87d · 08fa87d
1 parent dde6003
commit 08fa87d
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/nannyml/data_quality/unseen/calculator.py b/nannyml/data_quality/unseen/calculator.py
@@ -263,7 +263,8 @@ def _convert_int_columns_to_categorical(
     int_cols = list(
         filter(
             lambda c: c in column_names
-            and data[c].dtype in ('int_', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64'),
+            and data[c].dtype in ('int_', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64')
+            or (data[c].dtype in ('float_', 'float16', 'float32', 'float64') and (data[c] % 1 == 0).all()),
             data.columns,
         )
     )

diff --git a/tests/data_quality/test_unseen.py b/tests/data_quality/test_unseen.py
@@ -251,3 +251,17 @@ def test_input_dataframes_are_not_altered_by_calculator():  # noqa: D103
     results = calc.calculate(monitored2)  # noqa: F841
     pd.testing.assert_frame_equal(monitored, monitored2)
     pd.testing.assert_frame_equal(reference, reference2)
+
+
+def test_int_values_are_treated_as_categorical():  # noqa: D103
+    reference, analysis, _ = load_synthetic_car_loan_data_quality_dataset()
+    try:
+        _ = UnseenValuesCalculator(
+            column_names=[
+                'repaid',
+            ],
+            timestamp_column_name='timestamp',
+            normalize=False,
+        ).fit(reference)
+    except InvalidArgumentsException:
+        pytest.fail()