diff --git a/CHANGELOG.md b/CHANGELOG.md index f88fd2dde..815d6afde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +### Added + +- Added support for directly estimating the confusion matrix of multiclass classification models using CBPE. + Big thanks to our appreciated alumnus [@cartgr](https://github.com/cartgr) for the effort (and sorry it took soooo long). [(#287)](https://github.com/NannyML/nannyml/issues/287) + +### Fixed + ## [0.9.1] - 2023-07-12 ### Changed diff --git a/docs/_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg b/docs/_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg index 011513bf5..1ed3eb00d 100644 --- a/docs/_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg +++ b/docs/_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg @@ -1 +1 @@ -Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.450.4550.460.4650.470.4750.48Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.450.460.470.480.49Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.0150.020.0250.030.0350.04Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.030.0350.040.0450.05MetricAlertThresholdRealized performanceTimeTimeTimeTimeTrue PositiveTrue NegativeFalse PositiveFalse NegativeRealized True PositiveRealized True NegativeRealized False PositiveRealized False NegativeReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file +Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.450.4550.460.4650.470.4750.48Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.450.460.470.480.49Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.0150.020.0250.030.0350.04Jan 2018Apr 2018Jul 2018Oct 2018Jan 2019Apr 2019Jul 20190.030.0350.040.0450.05MetricAlertThresholdRealized performanceTimeTimeTimeTimeTrue PositiveTrue NegativeFalse PositiveFalse NegativeRealized True PositiveRealized True NegativeRealized False PositiveRealized False NegativeReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file diff --git a/docs/_static/tutorials/performance_calculation/multiclass/tutorial-confusion-matrix-calculation-multiclass.svg b/docs/_static/tutorials/performance_calculation/multiclass/tutorial-confusion-matrix-calculation-multiclass.svg new file mode 100644 index 000000000..ad81e4308 --- /dev/null +++ b/docs/_static/tutorials/performance_calculation/multiclass/tutorial-confusion-matrix-calculation-multiclass.svg @@ -0,0 +1 @@ +May 2020Jul 2020Sep 2020Nov 2020Jan 20210.220.230.240.250.26May 2020Jul 2020Sep 2020Nov 2020Jan 20210.040.0450.050.0550.060.0650.07May 2020Jul 2020Sep 2020Nov 2020Jan 20210.040.0450.050.055May 2020Jul 2020Sep 2020Nov 2020Jan 20210.040.050.060.070.080.090.1May 2020Jul 2020Sep 2020Nov 2020Jan 20210.140.160.180.20.220.240.26May 2020Jul 2020Sep 2020Nov 2020Jan 20210.030.040.050.060.07May 2020Jul 2020Sep 2020Nov 2020Jan 20210.040.050.060.070.080.09May 2020Jul 2020Sep 2020Nov 2020Jan 20210.0350.040.0450.050.0550.060.065May 2020Jul 2020Sep 2020Nov 2020Jan 20210.20.220.240.26MetricAlertThresholdRealized performanceTimeTimeTimeTimeTimeTimeTimeTimeTimetrue class: highstreet_card, predicted class: highstreet_cardtrue class: highstreet_card, predicted class: prepaid_cardtrue class: highstreet_card, predicted class: upmarket_cardtrue class: prepaid_card, predicted class: highstreet_cardtrue class: prepaid_card, predicted class: prepaid_cardtrue class: prepaid_card, predicted class: upmarket_cardtrue class: upmarket_card, predicted class: highstreet_cardtrue class: upmarket_card, predicted class: prepaid_cardtrue class: upmarket_card, predicted class: upmarket_cardRealized true class: highstreet_card, predicted class: highstreet_cardRealized true class: highstreet_card, predicted class: prepaid_cardRealized true class: highstreet_card, predicted class: upmarket_cardRealized true class: prepaid_card, predicted class: highstreet_cardRealized true class: prepaid_card, predicted class: prepaid_cardRealized true class: prepaid_card, predicted class: upmarket_cardRealized true class: upmarket_card, predicted class: highstreet_cardRealized true class: upmarket_card, predicted class: prepaid_cardRealized true class: upmarket_card, predicted class: upmarket_cardReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file diff --git a/docs/_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg b/docs/_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg index 55d3021ac..b8629b10d 100644 --- a/docs/_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg +++ b/docs/_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg @@ -1 +1 @@ -May 2020Jul 2020Sep 2020Nov 2020Jan 20210.750.80.850.9May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.780.80.820.840.860.88May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75MetricAlertThresholdRealized performanceTimeTimeTimeTimeTimeTimeROC AUCF1PrecisionRecallSpecificityAccuracyRealized ROC AUCRealized F1Realized PrecisionRealized RecallRealized SpecificityRealized AccuracyReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file +May 2020Jul 2020Sep 2020Nov 2020Jan 20210.750.80.850.9May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75May 2020Jul 2020Sep 2020Nov 2020Jan 20210.780.80.820.840.860.88May 2020Jul 2020Sep 2020Nov 2020Jan 20210.550.60.650.70.75MetricAlertThresholdRealized performanceTimeTimeTimeTimeTimeTimeROC AUCF1PrecisionRecallSpecificityAccuracyRealized ROC AUCRealized F1Realized PrecisionRealized RecallRealized SpecificityRealized AccuracyReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file diff --git a/docs/_static/tutorials/performance_estimation/binary/tutorial-confusion-matrix-estimation-binary-car-loan-analysis-with-ref.svg b/docs/_static/tutorials/performance_estimation/binary/tutorial-confusion-matrix-estimation-binary-car-loan-analysis-with-ref.svg index d1f35031e..62cad9b11 100644 --- a/docs/_static/tutorials/performance_estimation/binary/tutorial-confusion-matrix-estimation-binary-car-loan-analysis-with-ref.svg +++ b/docs/_static/tutorials/performance_estimation/binary/tutorial-confusion-matrix-estimation-binary-car-loan-analysis-with-ref.svg @@ -1 +1 @@ -Jan 2018Jul 2018Jan 2019Jul 20190.430.440.450.460.470.480.490.5Jan 2018Jul 2018Jan 2019Jul 20190.420.440.460.480.5Jan 2018Jul 2018Jan 2019Jul 20190.0150.020.0250.030.035Jan 2018Jul 2018Jan 2019Jul 20190.030.040.050.060.07MetricAlertThresholdConfidence bandEstimated performance (CBPE)TimeTimeTimeTimeTrue PositiveTrue NegativeFalse PositiveFalse NegativeEstimated True PositiveEstimated True NegativeEstimated False PositiveEstimated False NegativeReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file +Jan 2018Jul 2018Jan 2019Jul 20190.430.440.450.460.470.480.490.5Jan 2018Jul 2018Jan 2019Jul 20190.420.440.460.480.5Jan 2018Jul 2018Jan 2019Jul 20190.0150.020.0250.030.035Jan 2018Jul 2018Jan 2019Jul 20190.030.040.050.060.07MetricAlertThresholdConfidence bandEstimated performance (CBPE)TimeTimeTimeTimeTrue PositiveTrue NegativeFalse PositiveFalse NegativeEstimated True PositiveEstimated True NegativeEstimated False PositiveEstimated False NegativeReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file diff --git a/docs/_static/tutorials/performance_estimation/multiclass/tutorial-confusion-matrix-estimation-multiclass-analysis-with-ref.svg b/docs/_static/tutorials/performance_estimation/multiclass/tutorial-confusion-matrix-estimation-multiclass-analysis-with-ref.svg new file mode 100644 index 000000000..8f735c7c3 --- /dev/null +++ b/docs/_static/tutorials/performance_estimation/multiclass/tutorial-confusion-matrix-estimation-multiclass-analysis-with-ref.svg @@ -0,0 +1 @@ +May 2020Jul 2020Sep 2020Nov 2020Jan 20210.140.160.180.20.220.240.260.28May 2020Jul 2020Sep 2020Nov 2020Jan 20210.030.040.050.060.070.08May 2020Jul 2020Sep 2020Nov 2020Jan 20210.030.040.050.060.07May 2020Jul 2020Sep 2020Nov 2020Jan 20210.040.050.060.07May 2020Jul 2020Sep 2020Nov 2020Jan 20210.220.240.260.28May 2020Jul 2020Sep 2020Nov 2020Jan 20210.0350.040.0450.050.0550.060.065May 2020Jul 2020Sep 2020Nov 2020Jan 20210.030.0350.040.0450.050.055May 2020Jul 2020Sep 2020Nov 2020Jan 20210.030.040.050.060.07May 2020Jul 2020Sep 2020Nov 2020Jan 20210.180.20.220.240.26MetricAlertThresholdConfidence bandEstimated performance (CBPE)TimeTimeTimeTimeTimeTimeTimeTimeTimetrue class: 'prepaid_card', predicted class: 'prepaid_card'true class: 'prepaid_card', predicted class: 'highstreet_card'true class: 'prepaid_card', predicted class: 'upmarket_card'true class: 'highstreet_card', predicted class: 'prepaid_card'true class: 'highstreet_card', predicted class: 'highstreet_card'true class: 'highstreet_card', predicted class: 'upmarket_card'true class: 'upmarket_card', predicted class: 'prepaid_card'true class: 'upmarket_card', predicted class: 'highstreet_card'true class: 'upmarket_card', predicted class: 'upmarket_card'Estimated true class: 'prepaid_card', predicted class: 'prepaid_card'Estimated true class: 'prepaid_card', predicted class: 'highstreet_card'Estimated true class: 'prepaid_card', predicted class: 'upmarket_card'Estimated true class: 'highstreet_card', predicted class: 'prepaid_card'Estimated true class: 'highstreet_card', predicted class: 'highstreet_card'Estimated true class: 'highstreet_card', predicted class: 'upmarket_card'Estimated true class: 'upmarket_card', predicted class: 'prepaid_card'Estimated true class: 'upmarket_card', predicted class: 'highstreet_card'Estimated true class: 'upmarket_card', predicted class: 'upmarket_card'ReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysisReferenceAnalysis \ No newline at end of file diff --git a/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Binary Classification.ipynb b/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Binary Classification.ipynb index e43b48ded..2450f55c3 100644 --- a/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Binary Classification.ipynb +++ b/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Binary Classification.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": { "tags": [] }, @@ -122,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": { "jupyter": { "outputs_hidden": false @@ -151,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": { "jupyter": { "outputs_hidden": false @@ -172,16 +172,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "calc.fit(reference_df)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": { "jupyter": { "outputs_hidden": false @@ -553,7 +564,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": { "tags": [] }, @@ -596,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": { "tags": [] }, @@ -965,7 +976,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "tags": [] }, @@ -1007,11 +1018,3394 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": true, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.4596, + 0.455, + 0.471, + 0.4634, + 0.4674, + 0.458, + 0.4648, + 0.469, + 0.4682, + 0.465, + 0.465 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized True Positive", + "Reference", + "", + "[0:4999]", + "From Jan-01-2018 to Jan-31-2018", + 0.4596, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[5000:9999]", + "From Jan-31-2018 to Mar-02-2018", + 0.455, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[10000:14999]", + "From Mar-02-2018 to Apr-01-2018", + 0.471, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[15000:19999]", + "From Apr-01-2018 to May-02-2018", + 0.4634, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[20000:24999]", + "From May-02-2018 to Jun-01-2018", + 0.4674, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[25000:29999]", + "From Jun-01-2018 to Jul-01-2018", + 0.458, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[30000:34999]", + "From Jul-01-2018 to Jul-31-2018", + 0.4648, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[35000:39999]", + "From Jul-31-2018 to Aug-31-2018", + 0.469, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[40000:44999]", + "From Aug-31-2018 to Sep-30-2018", + 0.4682, + 0.0212 + ], + [ + "Realized True Positive", + "Reference", + "", + "[45000:49999]", + "From Sep-30-2018 to Oct-30-2018", + 0.465, + 0.0212 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-16T03:18:00", + "2018-02-15T09:54:00", + "2018-03-17T16:30:00", + "2018-04-16T23:06:00", + "2018-05-17T05:42:00", + "2018-06-16T12:18:00", + "2018-07-16T18:54:00", + "2018-08-16T01:30:00", + "2018-09-15T08:06:00", + "2018-10-15T14:37:38.424000" + ], + "xaxis": "x", + "y": [ + 0.4596, + 0.455, + 0.471, + 0.4634, + 0.4674, + 0.458, + 0.4648, + 0.469, + 0.4682, + 0.465, + 0.465 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.4812, + 0.4542, + 0.4558, + 0.4592, + 0.4662, + 0.452, + 0.454, + 0.4514, + 0.4498, + 0.4636, + 0.4636 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized True Positive", + "Analysis", + "âš  Drift detected", + "[0:4999]", + "From Oct-30-2018 to Nov-30-2018", + 0.4812, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[5000:9999]", + "From Nov-30-2018 to Dec-30-2018", + 0.4542, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[10000:14999]", + "From Dec-30-2018 to Jan-29-2019", + 0.4558, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[15000:19999]", + "From Jan-29-2019 to Feb-28-2019", + 0.4592, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[20000:24999]", + "From Feb-28-2019 to Mar-31-2019", + 0.4662, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[25000:29999]", + "From Mar-31-2019 to Apr-30-2019", + 0.452, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[30000:34999]", + "From Apr-30-2019 to May-30-2019", + 0.454, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[35000:39999]", + "From May-30-2019 to Jun-29-2019", + 0.4514, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[40000:44999]", + "From Jun-29-2019 to Jul-30-2019", + 0.4498, + 0.0212 + ], + [ + "Realized True Positive", + "Analysis", + "", + "[45000:49999]", + "From Jul-30-2019 to Aug-29-2019", + 0.4636, + 0.0212 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00", + "2018-12-15T03:54:00", + "2019-01-14T10:30:00", + "2019-02-13T17:06:00", + "2019-03-15T23:42:00", + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x", + "y": [ + 0.4812, + 0.4542, + 0.4558, + 0.4592, + 0.4662, + 0.452, + 0.454, + 0.4514, + 0.4498, + 0.4636, + 0.4636 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": true, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00" + ], + "xaxis": "x", + "y": [ + 0.4812 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": true, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037, + 0.47887864308544037 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x", + "y": [ + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596, + 0.4494013569145596 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.4866, + 0.4844, + 0.4752, + 0.4808, + 0.4708, + 0.4862, + 0.4802, + 0.476, + 0.4768, + 0.478, + 0.478 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized True Negative", + "Reference", + "", + "[0:4999]", + "From Jan-01-2018 to Jan-31-2018", + 0.4866, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[5000:9999]", + "From Jan-31-2018 to Mar-02-2018", + 0.4844, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[10000:14999]", + "From Mar-02-2018 to Apr-01-2018", + 0.4752, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[15000:19999]", + "From Apr-01-2018 to May-02-2018", + 0.4808, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[20000:24999]", + "From May-02-2018 to Jun-01-2018", + 0.4708, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[25000:29999]", + "From Jun-01-2018 to Jul-01-2018", + 0.4862, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[30000:34999]", + "From Jul-01-2018 to Jul-31-2018", + 0.4802, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[35000:39999]", + "From Jul-31-2018 to Aug-31-2018", + 0.476, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[40000:44999]", + "From Aug-31-2018 to Sep-30-2018", + 0.4768, + 0.0212 + ], + [ + "Realized True Negative", + "Reference", + "", + "[45000:49999]", + "From Sep-30-2018 to Oct-30-2018", + 0.478, + 0.0212 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-16T03:18:00", + "2018-02-15T09:54:00", + "2018-03-17T16:30:00", + "2018-04-16T23:06:00", + "2018-05-17T05:42:00", + "2018-06-16T12:18:00", + "2018-07-16T18:54:00", + "2018-08-16T01:30:00", + "2018-09-15T08:06:00", + "2018-10-15T14:37:38.424000" + ], + "xaxis": "x2", + "y": [ + 0.4866, + 0.4844, + 0.4752, + 0.4808, + 0.4708, + 0.4862, + 0.4802, + 0.476, + 0.4768, + 0.478, + 0.478 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.461, + 0.488, + 0.4944, + 0.4844, + 0.4748, + 0.4626, + 0.4616, + 0.463, + 0.4726, + 0.4486, + 0.4486 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[0:4999]", + "From Oct-30-2018 to Nov-30-2018", + 0.461, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "", + "[5000:9999]", + "From Nov-30-2018 to Dec-30-2018", + 0.488, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[10000:14999]", + "From Dec-30-2018 to Jan-29-2019", + 0.4944, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "", + "[15000:19999]", + "From Jan-29-2019 to Feb-28-2019", + 0.4844, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "", + "[20000:24999]", + "From Feb-28-2019 to Mar-31-2019", + 0.4748, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[25000:29999]", + "From Mar-31-2019 to Apr-30-2019", + 0.4626, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[30000:34999]", + "From Apr-30-2019 to May-30-2019", + 0.4616, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[35000:39999]", + "From May-30-2019 to Jun-29-2019", + 0.463, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "", + "[40000:44999]", + "From Jun-29-2019 to Jul-30-2019", + 0.4726, + 0.0212 + ], + [ + "Realized True Negative", + "Analysis", + "âš  Drift detected", + "[45000:49999]", + "From Jul-30-2019 to Aug-29-2019", + 0.4486, + 0.0212 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00", + "2018-12-15T03:54:00", + "2019-01-14T10:30:00", + "2019-02-13T17:06:00", + "2019-03-15T23:42:00", + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x2", + "y": [ + 0.461, + 0.488, + 0.4944, + 0.4844, + 0.4748, + 0.4626, + 0.4616, + 0.463, + 0.4726, + 0.4486, + 0.4486 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00", + "2019-01-14T10:30:00", + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x2", + "y": [ + 0.461, + 0.4944, + 0.4626, + 0.4616, + 0.463, + 0.4486 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766, + 0.49411896029134766 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x2", + "y": [ + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523, + 0.4648810397086523 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.019, + 0.0226, + 0.0198, + 0.0196, + 0.022, + 0.0226, + 0.0206, + 0.0216, + 0.0182, + 0.0232, + 0.0232 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized False Positive", + "Reference", + "", + "[0:4999]", + "From Jan-01-2018 to Jan-31-2018", + 0.019, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[5000:9999]", + "From Jan-31-2018 to Mar-02-2018", + 0.0226, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[10000:14999]", + "From Mar-02-2018 to Apr-01-2018", + 0.0198, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[15000:19999]", + "From Apr-01-2018 to May-02-2018", + 0.0196, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[20000:24999]", + "From May-02-2018 to Jun-01-2018", + 0.022, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[25000:29999]", + "From Jun-01-2018 to Jul-01-2018", + 0.0226, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[30000:34999]", + "From Jul-01-2018 to Jul-31-2018", + 0.0206, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[35000:39999]", + "From Jul-31-2018 to Aug-31-2018", + 0.0216, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[40000:44999]", + "From Aug-31-2018 to Sep-30-2018", + 0.0182, + 0.0061 + ], + [ + "Realized False Positive", + "Reference", + "", + "[45000:49999]", + "From Sep-30-2018 to Oct-30-2018", + 0.0232, + 0.0061 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-16T03:18:00", + "2018-02-15T09:54:00", + "2018-03-17T16:30:00", + "2018-04-16T23:06:00", + "2018-05-17T05:42:00", + "2018-06-16T12:18:00", + "2018-07-16T18:54:00", + "2018-08-16T01:30:00", + "2018-09-15T08:06:00", + "2018-10-15T14:37:38.424000" + ], + "xaxis": "x3", + "y": [ + 0.019, + 0.0226, + 0.0198, + 0.0196, + 0.022, + 0.0226, + 0.0206, + 0.0216, + 0.0182, + 0.0232, + 0.0232 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.0218, + 0.0204, + 0.0198, + 0.0192, + 0.0226, + 0.038, + 0.0334, + 0.0364, + 0.031, + 0.0396, + 0.0396 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized False Positive", + "Analysis", + "", + "[0:4999]", + "From Oct-30-2018 to Nov-30-2018", + 0.0218, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "", + "[5000:9999]", + "From Nov-30-2018 to Dec-30-2018", + 0.0204, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "", + "[10000:14999]", + "From Dec-30-2018 to Jan-29-2019", + 0.0198, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "", + "[15000:19999]", + "From Jan-29-2019 to Feb-28-2019", + 0.0192, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "", + "[20000:24999]", + "From Feb-28-2019 to Mar-31-2019", + 0.0226, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "âš  Drift detected", + "[25000:29999]", + "From Mar-31-2019 to Apr-30-2019", + 0.038, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "âš  Drift detected", + "[30000:34999]", + "From Apr-30-2019 to May-30-2019", + 0.0334, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "âš  Drift detected", + "[35000:39999]", + "From May-30-2019 to Jun-29-2019", + 0.0364, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "âš  Drift detected", + "[40000:44999]", + "From Jun-29-2019 to Jul-30-2019", + 0.031, + 0.0061 + ], + [ + "Realized False Positive", + "Analysis", + "âš  Drift detected", + "[45000:49999]", + "From Jul-30-2019 to Aug-29-2019", + 0.0396, + 0.0061 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00", + "2018-12-15T03:54:00", + "2019-01-14T10:30:00", + "2019-02-13T17:06:00", + "2019-03-15T23:42:00", + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x3", + "y": [ + 0.0218, + 0.0204, + 0.0198, + 0.0192, + 0.0226, + 0.038, + 0.0334, + 0.0364, + 0.031, + 0.0396, + 0.0396 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x3", + "y": [ + 0.038, + 0.0334, + 0.0364, + 0.031, + 0.0396 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005, + 0.02581799959167005 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x3", + "y": [ + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995, + 0.01602200040832995 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.0348, + 0.038, + 0.034, + 0.0362, + 0.0398, + 0.0332, + 0.0344, + 0.0334, + 0.0368, + 0.0338, + 0.0338 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized False Negative", + "Reference", + "", + "[0:4999]", + "From Jan-01-2018 to Jan-31-2018", + 0.0348, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[5000:9999]", + "From Jan-31-2018 to Mar-02-2018", + 0.038, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[10000:14999]", + "From Mar-02-2018 to Apr-01-2018", + 0.034, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[15000:19999]", + "From Apr-01-2018 to May-02-2018", + 0.0362, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[20000:24999]", + "From May-02-2018 to Jun-01-2018", + 0.0398, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[25000:29999]", + "From Jun-01-2018 to Jul-01-2018", + 0.0332, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[30000:34999]", + "From Jul-01-2018 to Jul-31-2018", + 0.0344, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[35000:39999]", + "From Jul-31-2018 to Aug-31-2018", + 0.0334, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[40000:44999]", + "From Aug-31-2018 to Sep-30-2018", + 0.0368, + 0.0078 + ], + [ + "Realized False Negative", + "Reference", + "", + "[45000:49999]", + "From Sep-30-2018 to Oct-30-2018", + 0.0338, + 0.0078 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-16T03:18:00", + "2018-02-15T09:54:00", + "2018-03-17T16:30:00", + "2018-04-16T23:06:00", + "2018-05-17T05:42:00", + "2018-06-16T12:18:00", + "2018-07-16T18:54:00", + "2018-08-16T01:30:00", + "2018-09-15T08:06:00", + "2018-10-15T14:37:38.424000" + ], + "xaxis": "x4", + "y": [ + 0.0348, + 0.038, + 0.034, + 0.0362, + 0.0398, + 0.0332, + 0.0344, + 0.0334, + 0.0368, + 0.0338, + 0.0338 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.036, + 0.0374, + 0.03, + 0.0372, + 0.0364, + 0.0474, + 0.051, + 0.0492, + 0.0466, + 0.0482, + 0.0482 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized False Negative", + "Analysis", + "", + "[0:4999]", + "From Oct-30-2018 to Nov-30-2018", + 0.036, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "", + "[5000:9999]", + "From Nov-30-2018 to Dec-30-2018", + 0.0374, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "", + "[10000:14999]", + "From Dec-30-2018 to Jan-29-2019", + 0.03, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "", + "[15000:19999]", + "From Jan-29-2019 to Feb-28-2019", + 0.0372, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "", + "[20000:24999]", + "From Feb-28-2019 to Mar-31-2019", + 0.0364, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "âš  Drift detected", + "[25000:29999]", + "From Mar-31-2019 to Apr-30-2019", + 0.0474, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "âš  Drift detected", + "[30000:34999]", + "From Apr-30-2019 to May-30-2019", + 0.051, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "âš  Drift detected", + "[35000:39999]", + "From May-30-2019 to Jun-29-2019", + 0.0492, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "âš  Drift detected", + "[40000:44999]", + "From Jun-29-2019 to Jul-30-2019", + 0.0466, + 0.0078 + ], + [ + "Realized False Negative", + "Analysis", + "âš  Drift detected", + "[45000:49999]", + "From Jul-30-2019 to Aug-29-2019", + 0.0482, + 0.0078 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-11-14T21:18:00", + "2018-12-15T03:54:00", + "2019-01-14T10:30:00", + "2019-02-13T17:06:00", + "2019-03-15T23:42:00", + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x4", + "y": [ + 0.036, + 0.0374, + 0.03, + 0.0372, + 0.0364, + 0.0474, + 0.051, + 0.0492, + 0.0466, + 0.0482, + 0.0482 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2019-04-15T06:18:00", + "2019-05-15T12:54:00", + "2019-06-14T19:30:00", + "2019-07-15T02:06:00", + "2019-08-14T08:37:38.424000" + ], + "xaxis": "x4", + "y": [ + 0.0474, + 0.051, + 0.0492, + 0.0466, + 0.0482 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-01-01T00:00:00", + "2018-01-31T06:36:00", + "2018-03-02T13:12:00", + "2018-04-01T19:48:00", + "2018-05-02T02:24:00", + "2018-06-01T09:00:00", + "2018-07-01T15:36:00", + "2018-07-31T22:12:00", + "2018-08-31T04:48:00", + "2018-09-30T11:24:00", + "2018-10-30T17:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935, + 0.041691527813262935 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2018-10-30T18:00:00", + "2018-11-30T00:36:00", + "2018-12-30T07:12:00", + "2019-01-29T13:48:00", + "2019-02-28T20:24:00", + "2019-03-31T03:00:00", + "2019-04-30T09:36:00", + "2019-05-30T16:12:00", + "2019-06-29T22:48:00", + "2019-07-30T05:24:00", + "2019-08-29T11:51:16.848000" + ], + "xaxis": "x4", + "y": [ + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074, + 0.029188472186737074 + ], + "yaxis": "y4" + } + ], + "layout": { + "annotations": [ + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized True Positive", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 1, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized True Negative", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.71875, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized False Positive", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.4375, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized False Negative", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.15625, + "yanchor": "bottom", + "yref": "paper" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2018-05-17T05:42:00", + "xref": "x", + "xshift": 10, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2019-03-15T23:42:00", + "xref": "x", + "xshift": 15, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2018-05-17T05:42:00", + "xref": "x2", + "xshift": 10, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2019-03-15T23:42:00", + "xref": "x2", + "xshift": 15, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2018-05-17T05:42:00", + "xref": "x3", + "xshift": 10, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2019-03-15T23:42:00", + "xref": "x3", + "xshift": 15, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2018-05-17T05:42:00", + "xref": "x4", + "xshift": 10, + "y": 1.01, + "yref": "y4 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2019-03-15T23:42:00", + "xref": "x4", + "xshift": 15, + "y": 1.01, + "yref": "y4 domain" + } + ], + "height": 2000, + "hoverlabel": { + "bgcolor": "white", + "font": { + "size": 14 + } + }, + "legend": { + "itemclick": false, + "itemdoubleclick": false, + "traceorder": "grouped" + }, + "paper_bgcolor": "rgba(255,255,255,1)", + "plot_bgcolor": "rgba(255,255,255,1)", + "shapes": [ + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2018-10-30T18:00:00", + "x1": "2018-10-30T18:00:00", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + } + ], + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Realized performance" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis2": { + "anchor": "y2", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis3": { + "anchor": "y3", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis4": { + "anchor": "y4", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0.84375, + 1 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "True Positive" + }, + "zeroline": false + }, + "yaxis2": { + "anchor": "x2", + "domain": [ + 0.5625, + 0.71875 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "True Negative" + }, + "zeroline": false + }, + "yaxis3": { + "anchor": "x3", + "domain": [ + 0.28125, + 0.4375 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "False Positive" + }, + "zeroline": false + }, + "yaxis4": { + "anchor": "x4", + "domain": [ + 0, + 0.15625 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "False Negative" + }, + "zeroline": false + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "figure = results.plot()\n", "figure.show()" @@ -1019,7 +4413,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "jupyter": { "outputs_hidden": false @@ -1028,9 +4422,7 @@ "outputs": [], "source": [ "figure = results.plot()\n", - "figure.write_image(f'../_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg')\n", - "\n", - "# tutorial-perf-est-guide-binary-class-car-loan-analysis\n" + "figure.write_image(f'../_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg')" ] }, { @@ -1057,7 +4449,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb b/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb new file mode 100644 index 000000000..957142693 --- /dev/null +++ b/docs/example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb @@ -0,0 +1,7992 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
acq_channelapp_behavioral_scorerequested_credit_limitapp_channelcredit_bureau_scorestated_incomeis_customertimestampy_pred_proba_prepaid_cardy_pred_proba_highstreet_cardy_pred_proba_upmarket_cardy_predy_true
0Partner31.808232350web30915000True2020-05-02 02:01:300.970.030.00prepaid_cardprepaid_card
1Partner24.382568500mobile41823000True2020-05-02 02:03:330.870.130.00prepaid_cardprepaid_card
2Partner2-0.787575400web50724000False2020-05-02 02:04:490.470.350.18prepaid_cardupmarket_card
\n", + "
" + ], + "text/plain": [ + " acq_channel app_behavioral_score requested_credit_limit app_channel \\\n", + "0 Partner3 1.808232 350 web \n", + "1 Partner2 4.382568 500 mobile \n", + "2 Partner2 -0.787575 400 web \n", + "\n", + " credit_bureau_score stated_income is_customer timestamp \\\n", + "0 309 15000 True 2020-05-02 02:01:30 \n", + "1 418 23000 True 2020-05-02 02:03:33 \n", + "2 507 24000 False 2020-05-02 02:04:49 \n", + "\n", + " y_pred_proba_prepaid_card y_pred_proba_highstreet_card \\\n", + "0 0.97 0.03 \n", + "1 0.87 0.13 \n", + "2 0.47 0.35 \n", + "\n", + " y_pred_proba_upmarket_card y_pred y_true \n", + "0 0.00 prepaid_card prepaid_card \n", + "1 0.00 prepaid_card prepaid_card \n", + "2 0.18 prepaid_card upmarket_card " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import nannyml as nml\n", + "from IPython.display import display\n", + "\n", + "reference_df, analysis_df, analysis_target_df = nml.load_synthetic_multiclass_classification_dataset()\n", + "\n", + "analysis_df = analysis_df.merge(analysis_target_df, left_index=True, right_index=True)\n", + "\n", + "display(reference_df.head(3))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| | acq_channel | app_behavioral_score | requested_credit_limit | app_channel | credit_bureau_score | stated_income | is_customer | timestamp | y_pred_proba_prepaid_card | y_pred_proba_highstreet_card | y_pred_proba_upmarket_card | y_pred | y_true |\n", + "+====+===============+========================+==========================+===============+=======================+=================+===============+=====================+=============================+================================+==============================+==============+===============+\n", + "| 0 | Partner3 | 1.80823 | 350 | web | 309 | 15000 | True | 2020-05-02 02:01:30 | 0.97 | 0.03 | 0 | prepaid_card | prepaid_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| 1 | Partner2 | 4.38257 | 500 | mobile | 418 | 23000 | True | 2020-05-02 02:03:33 | 0.87 | 0.13 | 0 | prepaid_card | prepaid_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| 2 | Partner2 | -0.787575 | 400 | web | 507 | 24000 | False | 2020-05-02 02:04:49 | 0.47 | 0.35 | 0.18 | prepaid_card | upmarket_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n" + ] + } + ], + "source": [ + "print(reference_df.head(3).to_markdown(tablefmt=\"grid\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "calc = nml.PerformanceCalculator(\n", + " y_pred_proba={\n", + " 'prepaid_card': 'y_pred_proba_prepaid_card',\n", + " 'highstreet_card': 'y_pred_proba_highstreet_card',\n", + " 'upmarket_card': 'y_pred_proba_upmarket_card'\n", + " },\n", + " y_pred='y_pred',\n", + " y_true='y_true',\n", + " timestamp_column_name='timestamp',\n", + " problem_type='classification_multiclass',\n", + " metrics=['confusion_matrix'],\n", + " normalize_confusion_matrix='all',\n", + " chunk_size=6000)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "calc.fit(reference_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chunktrue_highstreet_card_pred_highstreet_card...true_upmarket_card_pred_prepaid_cardtrue_upmarket_card_pred_upmarket_card
keychunk_indexstart_indexend_indexstart_dateend_dateperiodtargets_missing_ratesampling_errorvalue...sampling_errorvalueupper_thresholdlower_thresholdalertsampling_errorvalueupper_thresholdlower_thresholdalert
0[0:5999]0059992020-09-01 03:10:012020-09-13 16:15:10analysis0.00.0055550.242833...0.002490.0408330.0441760.03319False0.0056140.2483330.2701060.236227False
1[6000:11999]16000119992020-09-13 16:15:322020-09-25 19:48:42analysis0.00.0055550.256000...0.002490.0390000.0441760.03319False0.0056140.2540000.2701060.236227False
2[12000:17999]212000179992020-09-25 19:50:042020-10-08 02:53:47analysis0.00.0055550.241833...0.002490.0373330.0441760.03319False0.0056140.2573330.2701060.236227False
3[18000:23999]318000239992020-10-08 02:57:342020-10-20 15:48:19analysis0.00.0055550.241667...0.002490.0335000.0441760.03319False0.0056140.2546670.2701060.236227False
4[24000:29999]424000299992020-10-20 15:49:062020-11-01 22:04:40analysis0.00.0055550.248000...0.002490.0381670.0441760.03319False0.0056140.2611670.2701060.236227False
5[30000:35999]530000359992020-11-01 22:04:592020-11-14 03:55:33analysis0.00.0055550.220333...0.002490.0620000.0441760.03319True0.0056140.1941670.2701060.236227True
6[36000:41999]636000419992020-11-14 03:55:492020-11-26 09:19:06analysis0.00.0055550.224333...0.002490.0636670.0441760.03319True0.0056140.1866670.2701060.236227True
7[42000:47999]742000479992020-11-26 09:19:222020-12-08 14:33:56analysis0.00.0055550.232833...0.002490.0636670.0441760.03319True0.0056140.1911670.2701060.236227True
8[48000:53999]848000539992020-12-08 14:34:252020-12-20 18:30:30analysis0.00.0055550.225500...0.002490.0628330.0441760.03319True0.0056140.1943330.2701060.236227True
9[54000:59999]954000599992020-12-20 18:31:092021-01-01 22:57:55analysis0.00.0055550.225667...0.002490.0618330.0441760.03319True0.0056140.1955000.2701060.236227True
\n", + "

10 rows × 53 columns

\n", + "
" + ], + "text/plain": [ + " chunk \\\n", + " key chunk_index start_index end_index start_date \n", + "0 [0:5999] 0 0 5999 2020-09-01 03:10:01 \n", + "1 [6000:11999] 1 6000 11999 2020-09-13 16:15:32 \n", + "2 [12000:17999] 2 12000 17999 2020-09-25 19:50:04 \n", + "3 [18000:23999] 3 18000 23999 2020-10-08 02:57:34 \n", + "4 [24000:29999] 4 24000 29999 2020-10-20 15:49:06 \n", + "5 [30000:35999] 5 30000 35999 2020-11-01 22:04:59 \n", + "6 [36000:41999] 6 36000 41999 2020-11-14 03:55:49 \n", + "7 [42000:47999] 7 42000 47999 2020-11-26 09:19:22 \n", + "8 [48000:53999] 8 48000 53999 2020-12-08 14:34:25 \n", + "9 [54000:59999] 9 54000 59999 2020-12-20 18:31:09 \n", + "\n", + " \\\n", + " end_date period targets_missing_rate \n", + "0 2020-09-13 16:15:10 analysis 0.0 \n", + "1 2020-09-25 19:48:42 analysis 0.0 \n", + "2 2020-10-08 02:53:47 analysis 0.0 \n", + "3 2020-10-20 15:48:19 analysis 0.0 \n", + "4 2020-11-01 22:04:40 analysis 0.0 \n", + "5 2020-11-14 03:55:33 analysis 0.0 \n", + "6 2020-11-26 09:19:06 analysis 0.0 \n", + "7 2020-12-08 14:33:56 analysis 0.0 \n", + "8 2020-12-20 18:30:30 analysis 0.0 \n", + "9 2021-01-01 22:57:55 analysis 0.0 \n", + "\n", + " true_highstreet_card_pred_highstreet_card ... \\\n", + " sampling_error value ... \n", + "0 0.005555 0.242833 ... \n", + "1 0.005555 0.256000 ... \n", + "2 0.005555 0.241833 ... \n", + "3 0.005555 0.241667 ... \n", + "4 0.005555 0.248000 ... \n", + "5 0.005555 0.220333 ... \n", + "6 0.005555 0.224333 ... \n", + "7 0.005555 0.232833 ... \n", + "8 0.005555 0.225500 ... \n", + "9 0.005555 0.225667 ... \n", + "\n", + " true_upmarket_card_pred_prepaid_card \\\n", + " sampling_error value upper_threshold \n", + "0 0.00249 0.040833 0.044176 \n", + "1 0.00249 0.039000 0.044176 \n", + "2 0.00249 0.037333 0.044176 \n", + "3 0.00249 0.033500 0.044176 \n", + "4 0.00249 0.038167 0.044176 \n", + "5 0.00249 0.062000 0.044176 \n", + "6 0.00249 0.063667 0.044176 \n", + "7 0.00249 0.063667 0.044176 \n", + "8 0.00249 0.062833 0.044176 \n", + "9 0.00249 0.061833 0.044176 \n", + "\n", + " true_upmarket_card_pred_upmarket_card \\\n", + " lower_threshold alert sampling_error value \n", + "0 0.03319 False 0.005614 0.248333 \n", + "1 0.03319 False 0.005614 0.254000 \n", + "2 0.03319 False 0.005614 0.257333 \n", + "3 0.03319 False 0.005614 0.254667 \n", + "4 0.03319 False 0.005614 0.261167 \n", + "5 0.03319 True 0.005614 0.194167 \n", + "6 0.03319 True 0.005614 0.186667 \n", + "7 0.03319 True 0.005614 0.191167 \n", + "8 0.03319 True 0.005614 0.194333 \n", + "9 0.03319 True 0.005614 0.195500 \n", + "\n", + " \n", + " upper_threshold lower_threshold alert \n", + "0 0.270106 0.236227 False \n", + "1 0.270106 0.236227 False \n", + "2 0.270106 0.236227 False \n", + "3 0.270106 0.236227 False \n", + "4 0.270106 0.236227 False \n", + "5 0.270106 0.236227 True \n", + "6 0.270106 0.236227 True \n", + "7 0.270106 0.236227 True \n", + "8 0.270106 0.236227 True \n", + "9 0.270106 0.236227 True \n", + "\n", + "[10 rows x 53 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results = calc.calculate(analysis_df)\n", + "display(results.filter(period='analysis').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| | | chunk | | | | | | | | | true_highstreet_card_pred_highstreet_card | | | | | | true_highstreet_card_pred_prepaid_card | | | | | | true_highstreet_card_pred_upmarket_card | | | | | | true_prepaid_card_pred_highstreet_card | | | | | | true_prepaid_card_pred_prepaid_card | | | | | | true_prepaid_card_pred_upmarket_card | | | | | | true_upmarket_card_pred_highstreet_card | | | | | | true_upmarket_card_pred_prepaid_card | | | | | | true_upmarket_card_pred_upmarket_card | | | | |\n", + "| | | key | | chunk_index | | start_index | | end_index | | start_date | | end_date | | period | | targets_missing_rate | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert |\n", + "+====+===============+=================+=================+===============+=====================+=====================+============+==========================+===============================================+===========+=====================+=====================+===========+============================================+===========+=====================+=====================+===========+=============================================+===========+=====================+=====================+===========+============================================+===========+=====================+=====================+===========+=========================================+===========+=====================+=====================+===========+==========================================+===========+=====================+=====================+===========+=============================================+===========+=====================+=====================+===========+==========================================+===========+=====================+=====================+===========+===========================================+===========+=====================+=====================+===========+\n", + "| 0 | [0:5999] | 0 | 0 | 5999 | 2020-09-01 03:10:01 | 2020-09-13 16:15:10 | analysis | 0 | 0.00555482 | 0.242833 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0458333 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0416667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0456667 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.26 | 0.265912 | 0.243455 | False | 0.00239047 | 0.034 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0408333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0408333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.248333 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 1 | [6000:11999] | 1 | 6000 | 11999 | 2020-09-13 16:15:32 | 2020-09-25 19:48:42 | analysis | 0 | 0.00555482 | 0.256 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0435 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0395 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0395 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.253 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0335 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.042 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.039 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.254 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 2 | [12000:17999] | 2 | 12000 | 17999 | 2020-09-25 19:50:04 | 2020-10-08 02:53:47 | analysis | 0 | 0.00555482 | 0.241833 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0416667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0431667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.044 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.2595 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0333333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0418333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0373333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.257333 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 3 | [18000:23999] | 3 | 18000 | 23999 | 2020-10-08 02:57:34 | 2020-10-20 15:48:19 | analysis | 0 | 0.00555482 | 0.241667 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0413333 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0418333 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0395 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.262833 | 0.265912 | 0.243455 | False | 0.00239047 | 0.041 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0436667 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0335 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.254667 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 4 | [24000:29999] | 4 | 24000 | 29999 | 2020-10-20 15:49:06 | 2020-11-01 22:04:40 | analysis | 0 | 0.00555482 | 0.248 | 0.262292 | 0.228341 | False | 0.00265159 | 0.04 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0461667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.045 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.248833 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0323333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0403333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0381667 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.261167 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 5 | [30000:35999] | 5 | 30000 | 35999 | 2020-11-01 22:04:59 | 2020-11-14 03:55:33 | analysis | 0 | 0.00555482 | 0.220333 | 0.262292 | 0.228341 | True | 0.00265159 | 0.0701667 | 0.0486211 | 0.0396456 | True | 0.00262565 | 0.055 | 0.0470817 | 0.039385 | True | 0.00267108 | 0.0898333 | 0.0526699 | 0.0369635 | True | 0.00562464 | 0.146333 | 0.265912 | 0.243455 | True | 0.00239047 | 0.0746667 | 0.0421872 | 0.0289128 | True | 0.00254241 | 0.0875 | 0.046008 | 0.0348254 | True | 0.00248954 | 0.062 | 0.0441763 | 0.0331904 | True | 0.00561357 | 0.194167 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 6 | [36000:41999] | 6 | 36000 | 41999 | 2020-11-14 03:55:49 | 2020-11-26 09:19:06 | analysis | 0 | 0.00555482 | 0.224333 | 0.262292 | 0.228341 | True | 0.00265159 | 0.0673333 | 0.0486211 | 0.0396456 | True | 0.00262565 | 0.053 | 0.0470817 | 0.039385 | True | 0.00267108 | 0.0991667 | 0.0526699 | 0.0369635 | True | 0.00562464 | 0.151333 | 0.265912 | 0.243455 | True | 0.00239047 | 0.0698333 | 0.0421872 | 0.0289128 | True | 0.00254241 | 0.0846667 | 0.046008 | 0.0348254 | True | 0.00248954 | 0.0636667 | 0.0441763 | 0.0331904 | True | 0.00561357 | 0.186667 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 7 | [42000:47999] | 7 | 42000 | 47999 | 2020-11-26 09:19:22 | 2020-12-08 14:33:56 | analysis | 0 | 0.00555482 | 0.232833 | 0.262292 | 0.228341 | False | 0.00265159 | 0.066 | 0.0486211 | 0.0396456 | True | 0.00262565 | 0.0503333 | 0.0470817 | 0.039385 | True | 0.00267108 | 0.0916667 | 0.0526699 | 0.0369635 | True | 0.00562464 | 0.145833 | 0.265912 | 0.243455 | True | 0.00239047 | 0.0693333 | 0.0421872 | 0.0289128 | True | 0.00254241 | 0.0891667 | 0.046008 | 0.0348254 | True | 0.00248954 | 0.0636667 | 0.0441763 | 0.0331904 | True | 0.00561357 | 0.191167 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 8 | [48000:53999] | 8 | 48000 | 53999 | 2020-12-08 14:34:25 | 2020-12-20 18:30:30 | analysis | 0 | 0.00555482 | 0.2255 | 0.262292 | 0.228341 | True | 0.00265159 | 0.0686667 | 0.0486211 | 0.0396456 | True | 0.00262565 | 0.052 | 0.0470817 | 0.039385 | True | 0.00267108 | 0.0898333 | 0.0526699 | 0.0369635 | True | 0.00562464 | 0.148 | 0.265912 | 0.243455 | True | 0.00239047 | 0.0733333 | 0.0421872 | 0.0289128 | True | 0.00254241 | 0.0855 | 0.046008 | 0.0348254 | True | 0.00248954 | 0.0628333 | 0.0441763 | 0.0331904 | True | 0.00561357 | 0.194333 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 9 | [54000:59999] | 9 | 54000 | 59999 | 2020-12-20 18:31:09 | 2021-01-01 22:57:55 | analysis | 0 | 0.00555482 | 0.225667 | 0.262292 | 0.228341 | True | 0.00265159 | 0.065 | 0.0486211 | 0.0396456 | True | 0.00262565 | 0.0543333 | 0.0470817 | 0.039385 | True | 0.00267108 | 0.0946667 | 0.0526699 | 0.0369635 | True | 0.00562464 | 0.144833 | 0.265912 | 0.243455 | True | 0.00239047 | 0.0726667 | 0.0421872 | 0.0289128 | True | 0.00254241 | 0.0855 | 0.046008 | 0.0348254 | True | 0.00248954 | 0.0618333 | 0.0441763 | 0.0331904 | True | 0.00561357 | 0.1955 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n" + ] + } + ], + "source": [ + "from docs.utils import print_multi_index_markdown\n", + "print_multi_index_markdown(results.filter(period='analysis').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chunktrue_highstreet_card_pred_highstreet_card...true_upmarket_card_pred_prepaid_cardtrue_upmarket_card_pred_upmarket_card
keychunk_indexstart_indexend_indexstart_dateend_dateperiodtargets_missing_ratesampling_errorvalue...sampling_errorvalueupper_thresholdlower_thresholdalertsampling_errorvalueupper_thresholdlower_thresholdalert
0[0:5999]0059992020-05-02 02:01:302020-05-14 12:25:35reference0.00.0055550.244000...0.002490.0393330.0441760.03319False0.0056140.2530000.2701060.236227False
1[6000:11999]16000119992020-05-14 12:29:252020-05-26 18:27:42reference0.00.0055550.253833...0.002490.0366670.0441760.03319False0.0056140.2471670.2701060.236227False
2[12000:17999]212000179992020-05-26 18:31:062020-06-07 19:55:45reference0.00.0055550.246333...0.002490.0408330.0441760.03319False0.0056140.2553330.2701060.236227False
3[18000:23999]318000239992020-06-07 19:58:392020-06-19 19:42:20reference0.00.0055550.245500...0.002490.0415000.0441760.03319False0.0056140.2466670.2701060.236227False
4[24000:29999]424000299992020-06-19 19:44:142020-07-02 01:58:05reference0.00.0055550.242000...0.002490.0385000.0441760.03319False0.0056140.2605000.2701060.236227False
5[30000:35999]530000359992020-07-02 02:06:562020-07-14 08:14:04reference0.00.0055550.242833...0.002490.0401670.0441760.03319False0.0056140.2521670.2701060.236227False
6[36000:41999]636000419992020-07-14 08:14:082020-07-26 12:55:42reference0.00.0055550.247333...0.002490.0355000.0441760.03319False0.0056140.2443330.2701060.236227False
7[42000:47999]742000479992020-07-26 12:57:372020-08-07 16:32:15reference0.00.0055550.255667...0.002490.0368330.0441760.03319False0.0056140.2526670.2701060.236227False
8[48000:53999]848000539992020-08-07 16:33:442020-08-20 00:06:08reference0.00.0055550.236333...0.002490.0393330.0441760.03319False0.0056140.2575000.2701060.236227False
9[54000:59999]954000599992020-08-20 00:07:582020-09-01 03:03:23reference0.00.0055550.239333...0.002490.0381670.0441760.03319False0.0056140.2623330.2701060.236227False
\n", + "

10 rows × 53 columns

\n", + "
" + ], + "text/plain": [ + " chunk \\\n", + " key chunk_index start_index end_index start_date \n", + "0 [0:5999] 0 0 5999 2020-05-02 02:01:30 \n", + "1 [6000:11999] 1 6000 11999 2020-05-14 12:29:25 \n", + "2 [12000:17999] 2 12000 17999 2020-05-26 18:31:06 \n", + "3 [18000:23999] 3 18000 23999 2020-06-07 19:58:39 \n", + "4 [24000:29999] 4 24000 29999 2020-06-19 19:44:14 \n", + "5 [30000:35999] 5 30000 35999 2020-07-02 02:06:56 \n", + "6 [36000:41999] 6 36000 41999 2020-07-14 08:14:08 \n", + "7 [42000:47999] 7 42000 47999 2020-07-26 12:57:37 \n", + "8 [48000:53999] 8 48000 53999 2020-08-07 16:33:44 \n", + "9 [54000:59999] 9 54000 59999 2020-08-20 00:07:58 \n", + "\n", + " \\\n", + " end_date period targets_missing_rate \n", + "0 2020-05-14 12:25:35 reference 0.0 \n", + "1 2020-05-26 18:27:42 reference 0.0 \n", + "2 2020-06-07 19:55:45 reference 0.0 \n", + "3 2020-06-19 19:42:20 reference 0.0 \n", + "4 2020-07-02 01:58:05 reference 0.0 \n", + "5 2020-07-14 08:14:04 reference 0.0 \n", + "6 2020-07-26 12:55:42 reference 0.0 \n", + "7 2020-08-07 16:32:15 reference 0.0 \n", + "8 2020-08-20 00:06:08 reference 0.0 \n", + "9 2020-09-01 03:03:23 reference 0.0 \n", + "\n", + " true_highstreet_card_pred_highstreet_card ... \\\n", + " sampling_error value ... \n", + "0 0.005555 0.244000 ... \n", + "1 0.005555 0.253833 ... \n", + "2 0.005555 0.246333 ... \n", + "3 0.005555 0.245500 ... \n", + "4 0.005555 0.242000 ... \n", + "5 0.005555 0.242833 ... \n", + "6 0.005555 0.247333 ... \n", + "7 0.005555 0.255667 ... \n", + "8 0.005555 0.236333 ... \n", + "9 0.005555 0.239333 ... \n", + "\n", + " true_upmarket_card_pred_prepaid_card \\\n", + " sampling_error value upper_threshold \n", + "0 0.00249 0.039333 0.044176 \n", + "1 0.00249 0.036667 0.044176 \n", + "2 0.00249 0.040833 0.044176 \n", + "3 0.00249 0.041500 0.044176 \n", + "4 0.00249 0.038500 0.044176 \n", + "5 0.00249 0.040167 0.044176 \n", + "6 0.00249 0.035500 0.044176 \n", + "7 0.00249 0.036833 0.044176 \n", + "8 0.00249 0.039333 0.044176 \n", + "9 0.00249 0.038167 0.044176 \n", + "\n", + " true_upmarket_card_pred_upmarket_card \\\n", + " lower_threshold alert sampling_error value \n", + "0 0.03319 False 0.005614 0.253000 \n", + "1 0.03319 False 0.005614 0.247167 \n", + "2 0.03319 False 0.005614 0.255333 \n", + "3 0.03319 False 0.005614 0.246667 \n", + "4 0.03319 False 0.005614 0.260500 \n", + "5 0.03319 False 0.005614 0.252167 \n", + "6 0.03319 False 0.005614 0.244333 \n", + "7 0.03319 False 0.005614 0.252667 \n", + "8 0.03319 False 0.005614 0.257500 \n", + "9 0.03319 False 0.005614 0.262333 \n", + "\n", + " \n", + " upper_threshold lower_threshold alert \n", + "0 0.270106 0.236227 False \n", + "1 0.270106 0.236227 False \n", + "2 0.270106 0.236227 False \n", + "3 0.270106 0.236227 False \n", + "4 0.270106 0.236227 False \n", + "5 0.270106 0.236227 False \n", + "6 0.270106 0.236227 False \n", + "7 0.270106 0.236227 False \n", + "8 0.270106 0.236227 False \n", + "9 0.270106 0.236227 False \n", + "\n", + "[10 rows x 53 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(results.filter(period='reference').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| | | chunk | | | | | | | | | true_highstreet_card_pred_highstreet_card | | | | | | true_highstreet_card_pred_prepaid_card | | | | | | true_highstreet_card_pred_upmarket_card | | | | | | true_prepaid_card_pred_highstreet_card | | | | | | true_prepaid_card_pred_prepaid_card | | | | | | true_prepaid_card_pred_upmarket_card | | | | | | true_upmarket_card_pred_highstreet_card | | | | | | true_upmarket_card_pred_prepaid_card | | | | | | true_upmarket_card_pred_upmarket_card | | | | |\n", + "| | | key | | chunk_index | | start_index | | end_index | | start_date | | end_date | | period | | targets_missing_rate | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert | | sampling_error | | value | | upper_threshold | | lower_threshold | | alert |\n", + "+====+===============+=================+=================+===============+=====================+=====================+============+==========================+===============================================+===========+=====================+=====================+===========+============================================+===========+=====================+=====================+===========+=============================================+===========+=====================+=====================+===========+============================================+===========+=====================+=====================+===========+=========================================+===========+=====================+=====================+===========+==========================================+===========+=====================+=====================+===========+=============================================+===========+=====================+=====================+===========+==========================================+===========+=====================+=====================+===========+===========================================+===========+=====================+=====================+===========+\n", + "| 0 | [0:5999] | 0 | 0 | 5999 | 2020-05-02 02:01:30 | 2020-05-14 12:25:35 | reference | 0 | 0.00555482 | 0.244 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0453333 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0435 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.05 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.2535 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0346667 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0366667 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0393333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.253 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 1 | [6000:11999] | 1 | 6000 | 11999 | 2020-05-14 12:29:25 | 2020-05-26 18:27:42 | reference | 0 | 0.00555482 | 0.253833 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0458333 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.042 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0473333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.25 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0363333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0408333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0366667 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.247167 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 2 | [12000:17999] | 2 | 12000 | 17999 | 2020-05-26 18:31:06 | 2020-06-07 19:55:45 | reference | 0 | 0.00555482 | 0.246333 | 0.262292 | 0.228341 | False | 0.00265159 | 0.041 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0428333 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0455 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.2555 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0303333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0423333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0408333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.255333 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 3 | [18000:23999] | 3 | 18000 | 23999 | 2020-06-07 19:58:39 | 2020-06-19 19:42:20 | reference | 0 | 0.00555482 | 0.2455 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0426667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0426667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0448333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.258167 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0345 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0435 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0415 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.246667 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 4 | [24000:29999] | 4 | 24000 | 29999 | 2020-06-19 19:44:14 | 2020-07-02 01:58:05 | reference | 0 | 0.00555482 | 0.242 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0435 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0421667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0403333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.256833 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0346667 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0415 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0385 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.2605 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 5 | [30000:35999] | 5 | 30000 | 35999 | 2020-07-02 02:06:56 | 2020-07-14 08:14:04 | reference | 0 | 0.00555482 | 0.242833 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0461667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.044 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0455 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.253667 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0353333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0401667 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0401667 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.252167 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 6 | [36000:41999] | 6 | 36000 | 41999 | 2020-07-14 08:14:08 | 2020-07-26 12:55:42 | reference | 0 | 0.00555482 | 0.247333 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0446667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0463333 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0428333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.261167 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0366667 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0411667 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0355 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.244333 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 7 | [42000:47999] | 7 | 42000 | 47999 | 2020-07-26 12:57:37 | 2020-08-07 16:32:15 | reference | 0 | 0.00555482 | 0.255667 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0443333 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0431667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0441667 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.2475 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0373333 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0383333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0368333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.252667 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 8 | [48000:53999] | 8 | 48000 | 53999 | 2020-08-07 16:33:44 | 2020-08-20 00:06:08 | reference | 0 | 0.00555482 | 0.236333 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0446667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.044 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0458333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.253833 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0391667 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0393333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0393333 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.2575 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n", + "| 9 | [54000:59999] | 9 | 54000 | 59999 | 2020-08-20 00:07:58 | 2020-09-01 03:03:23 | reference | 0 | 0.00555482 | 0.239333 | 0.262292 | 0.228341 | False | 0.00265159 | 0.0431667 | 0.0486211 | 0.0396456 | False | 0.00262565 | 0.0416667 | 0.0470817 | 0.039385 | False | 0.00267108 | 0.0418333 | 0.0526699 | 0.0369635 | False | 0.00562464 | 0.256667 | 0.265912 | 0.243455 | False | 0.00239047 | 0.0365 | 0.0421872 | 0.0289128 | False | 0.00254241 | 0.0403333 | 0.046008 | 0.0348254 | False | 0.00248954 | 0.0381667 | 0.0441763 | 0.0331904 | False | 0.00561357 | 0.262333 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+--------------------------+-----------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+--------------------------------------------+-----------+---------------------+---------------------+-----------+-----------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+---------------------------------------------+-----------+---------------------+---------------------+-----------+------------------------------------------+-----------+---------------------+---------------------+-----------+-------------------------------------------+-----------+---------------------+---------------------+-----------+\n" + ] + } + ], + "source": [ + "print_multi_index_markdown(results.filter(period='reference').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.244, + 0.25383333333333336, + 0.24633333333333332, + 0.2455, + 0.242, + 0.24283333333333335, + 0.24733333333333332, + 0.25566666666666665, + 0.23633333333333334, + 0.23933333333333334, + 0.23933333333333334 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.244, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.2538, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.2463, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.2455, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.242, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.2428, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.2473, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.2557, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.2363, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.2393, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x", + "y": [ + 0.244, + 0.25383333333333336, + 0.24633333333333332, + 0.2455, + 0.242, + 0.24283333333333335, + 0.24733333333333332, + 0.25566666666666665, + 0.23633333333333334, + 0.23933333333333334, + 0.23933333333333334 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.24283333333333335, + 0.256, + 0.24183333333333334, + 0.24166666666666667, + 0.248, + 0.22033333333333333, + 0.22433333333333333, + 0.23283333333333334, + 0.2255, + 0.22566666666666665, + 0.22566666666666665 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.2428, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.256, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.2418, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.2417, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.248, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.2203, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.2243, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.2328, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.2255, + 0.0167 + ], + [ + "Realized true class: highstreet_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.2257, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x", + "y": [ + 0.24283333333333335, + 0.256, + 0.24183333333333334, + 0.24166666666666667, + 0.248, + 0.22033333333333333, + 0.22433333333333333, + 0.23283333333333334, + 0.2255, + 0.22566666666666665, + 0.22566666666666665 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x", + "y": [ + 0.22033333333333333, + 0.22433333333333333, + 0.2255, + 0.22566666666666665 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355, + 0.2622923109843355 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792, + 0.22834102234899792 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.04533333333333334, + 0.04583333333333333, + 0.041, + 0.042666666666666665, + 0.0435, + 0.04616666666666667, + 0.04466666666666667, + 0.044333333333333336, + 0.04466666666666667, + 0.043166666666666666, + 0.043166666666666666 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.0453, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.0458, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.041, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0427, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0435, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.0462, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0447, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0443, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.0447, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0432, + 0.008 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x2", + "y": [ + 0.04533333333333334, + 0.04583333333333333, + 0.041, + 0.042666666666666665, + 0.0435, + 0.04616666666666667, + 0.04466666666666667, + 0.044333333333333336, + 0.04466666666666667, + 0.043166666666666666, + 0.043166666666666666 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.04583333333333333, + 0.0435, + 0.041666666666666664, + 0.04133333333333333, + 0.04, + 0.07016666666666667, + 0.06733333333333333, + 0.066, + 0.06866666666666667, + 0.065, + 0.065 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.0458, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.0435, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.0417, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.0413, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.04, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.0702, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.0673, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.066, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.0687, + 0.008 + ], + [ + "Realized true class: highstreet_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.065, + 0.008 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x2", + "y": [ + 0.04583333333333333, + 0.0435, + 0.041666666666666664, + 0.04133333333333333, + 0.04, + 0.07016666666666667, + 0.06733333333333333, + 0.066, + 0.06866666666666667, + 0.065, + 0.065 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x2", + "y": [ + 0.07016666666666667, + 0.06733333333333333, + 0.066, + 0.06866666666666667, + 0.065 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672, + 0.04862109446779672 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966, + 0.039645572198869966 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.0435, + 0.042, + 0.042833333333333334, + 0.042666666666666665, + 0.042166666666666665, + 0.044, + 0.04633333333333333, + 0.043166666666666666, + 0.044, + 0.041666666666666664, + 0.041666666666666664 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.0435, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.042, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.0428, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0427, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0422, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.044, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0463, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0432, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.044, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0417, + 0.0079 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x3", + "y": [ + 0.0435, + 0.042, + 0.042833333333333334, + 0.042666666666666665, + 0.042166666666666665, + 0.044, + 0.04633333333333333, + 0.043166666666666666, + 0.044, + 0.041666666666666664, + 0.041666666666666664 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.041666666666666664, + 0.0395, + 0.043166666666666666, + 0.041833333333333333, + 0.04616666666666667, + 0.055, + 0.053, + 0.050333333333333334, + 0.052, + 0.05433333333333333, + 0.05433333333333333 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.0417, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.0395, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.0432, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.0418, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.0462, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.055, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.053, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.0503, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.052, + 0.0079 + ], + [ + "Realized true class: highstreet_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.0543, + 0.0079 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x3", + "y": [ + 0.041666666666666664, + 0.0395, + 0.043166666666666666, + 0.041833333333333333, + 0.04616666666666667, + 0.055, + 0.053, + 0.050333333333333334, + 0.052, + 0.05433333333333333, + 0.05433333333333333 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x3", + "y": [ + 0.055, + 0.053, + 0.050333333333333334, + 0.052, + 0.05433333333333333 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121, + 0.0470817096143121 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456, + 0.03938495705235456 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.05, + 0.04733333333333333, + 0.0455, + 0.044833333333333336, + 0.04033333333333333, + 0.0455, + 0.042833333333333334, + 0.04416666666666667, + 0.04583333333333333, + 0.041833333333333333, + 0.041833333333333333 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.05, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.0473, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.0455, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0448, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0403, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.0455, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0428, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0442, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.0458, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0418, + 0.008 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x4", + "y": [ + 0.05, + 0.04733333333333333, + 0.0455, + 0.044833333333333336, + 0.04033333333333333, + 0.0455, + 0.042833333333333334, + 0.04416666666666667, + 0.04583333333333333, + 0.041833333333333333, + 0.041833333333333333 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.04566666666666667, + 0.0395, + 0.044, + 0.0395, + 0.045, + 0.08983333333333333, + 0.09916666666666667, + 0.09166666666666666, + 0.08983333333333333, + 0.09466666666666666, + 0.09466666666666666 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.0457, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.0395, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.044, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.0395, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.045, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.0898, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.0992, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.0917, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.0898, + 0.008 + ], + [ + "Realized true class: prepaid_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.0947, + 0.008 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x4", + "y": [ + 0.04566666666666667, + 0.0395, + 0.044, + 0.0395, + 0.045, + 0.08983333333333333, + 0.09916666666666667, + 0.09166666666666666, + 0.08983333333333333, + 0.09466666666666666, + 0.09466666666666666 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x4", + "y": [ + 0.08983333333333333, + 0.09916666666666667, + 0.09166666666666666, + 0.08983333333333333, + 0.09466666666666666 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172, + 0.05266985073429172 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161, + 0.03696348259904161 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.2535, + 0.25, + 0.2555, + 0.25816666666666666, + 0.25683333333333336, + 0.25366666666666665, + 0.26116666666666666, + 0.2475, + 0.25383333333333336, + 0.25666666666666665, + 0.25666666666666665 + ], + "yaxis": "y5" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.2535, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.25, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.2555, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.2582, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.2568, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.2537, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.2612, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.2475, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.2538, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.2567, + 0.0169 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x5", + "y": [ + 0.2535, + 0.25, + 0.2555, + 0.25816666666666666, + 0.25683333333333336, + 0.25366666666666665, + 0.26116666666666666, + 0.2475, + 0.25383333333333336, + 0.25666666666666665, + 0.25666666666666665 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.26, + 0.253, + 0.2595, + 0.2628333333333333, + 0.24883333333333332, + 0.14633333333333334, + 0.15133333333333332, + 0.14583333333333334, + 0.148, + 0.14483333333333334, + 0.14483333333333334 + ], + "yaxis": "y5" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.26, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.253, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.2595, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.2628, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.2488, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.1463, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.1513, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.1458, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.148, + 0.0169 + ], + [ + "Realized true class: prepaid_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.1448, + 0.0169 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x5", + "y": [ + 0.26, + 0.253, + 0.2595, + 0.2628333333333333, + 0.24883333333333332, + 0.14633333333333334, + 0.15133333333333332, + 0.14583333333333334, + 0.148, + 0.14483333333333334, + 0.14483333333333334 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x5", + "y": [ + 0.14633333333333334, + 0.15133333333333332, + 0.14583333333333334, + 0.148, + 0.14483333333333334 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143, + 0.2659115344359143 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244, + 0.24345513223075244 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.034666666666666665, + 0.036333333333333336, + 0.030333333333333334, + 0.0345, + 0.034666666666666665, + 0.035333333333333335, + 0.03666666666666667, + 0.037333333333333336, + 0.03916666666666667, + 0.0365, + 0.0365 + ], + "yaxis": "y6" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.0347, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.0363, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.0303, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0345, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0347, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.0353, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0367, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0373, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.0392, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0365, + 0.0072 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x6", + "y": [ + 0.034666666666666665, + 0.036333333333333336, + 0.030333333333333334, + 0.0345, + 0.034666666666666665, + 0.035333333333333335, + 0.03666666666666667, + 0.037333333333333336, + 0.03916666666666667, + 0.0365, + 0.0365 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.034, + 0.0335, + 0.03333333333333333, + 0.041, + 0.03233333333333333, + 0.07466666666666667, + 0.06983333333333333, + 0.06933333333333333, + 0.07333333333333333, + 0.07266666666666667, + 0.07266666666666667 + ], + "yaxis": "y6" + }, + { + "customdata": [ + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.034, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.0335, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.0333, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.041, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.0323, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.0747, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.0698, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.0693, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.0733, + 0.0072 + ], + [ + "Realized true class: prepaid_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.0727, + 0.0072 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x6", + "y": [ + 0.034, + 0.0335, + 0.03333333333333333, + 0.041, + 0.03233333333333333, + 0.07466666666666667, + 0.06983333333333333, + 0.06933333333333333, + 0.07333333333333333, + 0.07266666666666667, + 0.07266666666666667 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x6", + "y": [ + 0.07466666666666667, + 0.06983333333333333, + 0.06933333333333333, + 0.07333333333333333, + 0.07266666666666667 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665, + 0.04218720573735665 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357, + 0.028912794262643357 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x7", + "y": [ + 0.03666666666666667, + 0.04083333333333333, + 0.042333333333333334, + 0.0435, + 0.0415, + 0.04016666666666667, + 0.041166666666666664, + 0.03833333333333333, + 0.03933333333333333, + 0.04033333333333333, + 0.04033333333333333 + ], + "yaxis": "y7" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.0367, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.0408, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.0423, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0435, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0415, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.0402, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0412, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0383, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.0393, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0403, + 0.0076 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x7", + "y": [ + 0.03666666666666667, + 0.04083333333333333, + 0.042333333333333334, + 0.0435, + 0.0415, + 0.04016666666666667, + 0.041166666666666664, + 0.03833333333333333, + 0.03933333333333333, + 0.04033333333333333, + 0.04033333333333333 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x7", + "y": [ + 0.04083333333333333, + 0.042, + 0.041833333333333333, + 0.043666666666666666, + 0.04033333333333333, + 0.0875, + 0.08466666666666667, + 0.08916666666666667, + 0.0855, + 0.0855, + 0.0855 + ], + "yaxis": "y7" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.0408, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.042, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.0418, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.0437, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.0403, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.0875, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.0847, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.0892, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.0855, + 0.0076 + ], + [ + "Realized true class: upmarket_card, predicted class: highstreet_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.0855, + 0.0076 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x7", + "y": [ + 0.04083333333333333, + 0.042, + 0.041833333333333333, + 0.043666666666666666, + 0.04033333333333333, + 0.0875, + 0.08466666666666667, + 0.08916666666666667, + 0.0855, + 0.0855, + 0.0855 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x7", + "y": [ + 0.0875, + 0.08466666666666667, + 0.08916666666666667, + 0.0855, + 0.0855 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x7", + "y": [ + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x7", + "y": [ + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x7", + "y": [ + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385, + 0.04600795453262385 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x7", + "y": [ + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949, + 0.03482537880070949 + ], + "yaxis": "y7" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x8", + "y": [ + 0.03933333333333333, + 0.03666666666666667, + 0.04083333333333333, + 0.0415, + 0.0385, + 0.04016666666666667, + 0.0355, + 0.036833333333333336, + 0.03933333333333333, + 0.03816666666666667, + 0.03816666666666667 + ], + "yaxis": "y8" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.0393, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.0367, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.0408, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.0415, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.0385, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.0402, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.0355, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.0368, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.0393, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.0382, + 0.0075 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x8", + "y": [ + 0.03933333333333333, + 0.03666666666666667, + 0.04083333333333333, + 0.0415, + 0.0385, + 0.04016666666666667, + 0.0355, + 0.036833333333333336, + 0.03933333333333333, + 0.03816666666666667, + 0.03816666666666667 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x8", + "y": [ + 0.04083333333333333, + 0.039, + 0.037333333333333336, + 0.0335, + 0.03816666666666667, + 0.062, + 0.06366666666666666, + 0.06366666666666666, + 0.06283333333333334, + 0.06183333333333333, + 0.06183333333333333 + ], + "yaxis": "y8" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.0408, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.039, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.0373, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.0335, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.0382, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.062, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.0637, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.0637, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.0628, + 0.0075 + ], + [ + "Realized true class: upmarket_card, predicted class: prepaid_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.0618, + 0.0075 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x8", + "y": [ + 0.04083333333333333, + 0.039, + 0.037333333333333336, + 0.0335, + 0.03816666666666667, + 0.062, + 0.06366666666666666, + 0.06366666666666666, + 0.06283333333333334, + 0.06183333333333333, + 0.06183333333333333 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x8", + "y": [ + 0.062, + 0.06366666666666666, + 0.06366666666666666, + 0.06283333333333334, + 0.06183333333333333 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x8", + "y": [ + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x8", + "y": [ + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x8", + "y": [ + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135, + 0.0441762833604135 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x8", + "y": [ + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166, + 0.033190383306253166 + ], + "yaxis": "y8" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x9", + "y": [ + 0.253, + 0.24716666666666667, + 0.25533333333333336, + 0.24666666666666667, + 0.2605, + 0.25216666666666665, + 0.24433333333333335, + 0.25266666666666665, + 0.2575, + 0.2623333333333333, + 0.2623333333333333 + ], + "yaxis": "y9" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.253, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.2472, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.2553, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.2467, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.2605, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.2522, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.2443, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.2527, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.2575, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.2623, + 0.0168 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x9", + "y": [ + 0.253, + 0.24716666666666667, + 0.25533333333333336, + 0.24666666666666667, + 0.2605, + 0.25216666666666665, + 0.24433333333333335, + 0.25266666666666665, + 0.2575, + 0.2623333333333333, + 0.2623333333333333 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x9", + "y": [ + 0.24833333333333332, + 0.254, + 0.25733333333333336, + 0.25466666666666665, + 0.26116666666666666, + 0.19416666666666665, + 0.18666666666666668, + 0.19116666666666668, + 0.19433333333333333, + 0.1955, + 0.1955 + ], + "yaxis": "y9" + }, + { + "customdata": [ + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.2483, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.254, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.2573, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.2547, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.2612, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.1942, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.1867, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.1912, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.1943, + 0.0168 + ], + [ + "Realized true class: upmarket_card, predicted class: upmarket_card", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.1955, + 0.0168 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x9", + "y": [ + 0.24833333333333332, + 0.254, + 0.25733333333333336, + 0.25466666666666665, + 0.26116666666666666, + 0.19416666666666665, + 0.18666666666666668, + 0.19116666666666668, + 0.19433333333333333, + 0.1955, + 0.1955 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x9", + "y": [ + 0.19416666666666665, + 0.18666666666666668, + 0.19116666666666668, + 0.19433333333333333, + 0.1955 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x9", + "y": [ + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x9", + "y": [ + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x9", + "y": [ + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454, + 0.2701062652451454 + ], + "yaxis": "y9" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x9", + "y": [ + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789, + 0.23622706808818789 + ], + "yaxis": "y9" + } + ], + "layout": { + "annotations": [ + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: highstreet_card, predicted class: highstreet_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 1, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: highstreet_card, predicted class: prepaid_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.8827160493827161, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: highstreet_card, predicted class: upmarket_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.7654320987654322, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: prepaid_card, predicted class: highstreet_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.6481481481481483, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: prepaid_card, predicted class: prepaid_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.5308641975308642, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: prepaid_card, predicted class: upmarket_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.41358024691358025, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: upmarket_card, predicted class: highstreet_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.2962962962962963, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: upmarket_card, predicted class: prepaid_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.17901234567901234, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized true class: upmarket_card, predicted class: upmarket_card", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.0617283950617284, + "yanchor": "bottom", + "yref": "paper" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x", + "xshift": 10, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x", + "xshift": 15, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x2", + "xshift": 10, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x2", + "xshift": 15, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x3", + "xshift": 10, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x3", + "xshift": 15, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x4", + "xshift": 10, + "y": 1.01, + "yref": "y4 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x4", + "xshift": 15, + "y": 1.01, + "yref": "y4 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x5", + "xshift": 10, + "y": 1.01, + "yref": "y5 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x5", + "xshift": 15, + "y": 1.01, + "yref": "y5 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x6", + "xshift": 10, + "y": 1.01, + "yref": "y6 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x6", + "xshift": 15, + "y": 1.01, + "yref": "y6 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x7", + "xshift": 10, + "y": 1.01, + "yref": "y7 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x7", + "xshift": 15, + "y": 1.01, + "yref": "y7 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x8", + "xshift": 10, + "y": 1.01, + "yref": "y8 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x8", + "xshift": 15, + "y": 1.01, + "yref": "y8 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x9", + "xshift": 10, + "y": 1.01, + "yref": "y9 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x9", + "xshift": 15, + "y": 1.01, + "yref": "y9 domain" + } + ], + "height": 4500, + "hoverlabel": { + "bgcolor": "white", + "font": { + "size": 14 + } + }, + "legend": { + "itemclick": false, + "itemdoubleclick": false, + "traceorder": "grouped" + }, + "paper_bgcolor": "rgba(255,255,255,1)", + "plot_bgcolor": "rgba(255,255,255,1)", + "shapes": [ + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x6", + "y0": 0, + "y1": 1, + "yref": "y6 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x6", + "y0": 0, + "y1": 1, + "yref": "y6 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x7", + "y0": 0, + "y1": 1, + "yref": "y7 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x6", + "y0": 0, + "y1": 1, + "yref": "y6 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x7", + "y0": 0, + "y1": 1, + "yref": "y7 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x8", + "y0": 0, + "y1": 1, + "yref": "y8 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x6", + "y0": 0, + "y1": 1, + "yref": "y6 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x7", + "y0": 0, + "y1": 1, + "yref": "y7 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x8", + "y0": 0, + "y1": 1, + "yref": "y8 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x9", + "y0": 0, + "y1": 1, + "yref": "y9 domain" + } + ], + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Realized performance" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis2": { + "anchor": "y2", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis3": { + "anchor": "y3", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis4": { + "anchor": "y4", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis5": { + "anchor": "y5", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis6": { + "anchor": "y6", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis7": { + "anchor": "y7", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis8": { + "anchor": "y8", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis9": { + "anchor": "y9", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0.9382716049382716, + 1 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: highstreet_card, predicted class: highstreet_card" + }, + "zeroline": false + }, + "yaxis2": { + "anchor": "x2", + "domain": [ + 0.8209876543209876, + 0.8827160493827161 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: highstreet_card, predicted class: prepaid_card" + }, + "zeroline": false + }, + "yaxis3": { + "anchor": "x3", + "domain": [ + 0.7037037037037037, + 0.7654320987654322 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: highstreet_card, predicted class: upmarket_card" + }, + "zeroline": false + }, + "yaxis4": { + "anchor": "x4", + "domain": [ + 0.5864197530864198, + 0.6481481481481483 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: prepaid_card, predicted class: highstreet_card" + }, + "zeroline": false + }, + "yaxis5": { + "anchor": "x5", + "domain": [ + 0.4691358024691358, + 0.5308641975308642 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: prepaid_card, predicted class: prepaid_card" + }, + "zeroline": false + }, + "yaxis6": { + "anchor": "x6", + "domain": [ + 0.35185185185185186, + 0.41358024691358025 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: prepaid_card, predicted class: upmarket_card" + }, + "zeroline": false + }, + "yaxis7": { + "anchor": "x7", + "domain": [ + 0.2345679012345679, + 0.2962962962962963 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: upmarket_card, predicted class: highstreet_card" + }, + "zeroline": false + }, + "yaxis8": { + "anchor": "x8", + "domain": [ + 0.11728395061728394, + 0.17901234567901234 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: upmarket_card, predicted class: prepaid_card" + }, + "zeroline": false + }, + "yaxis9": { + "anchor": "x9", + "domain": [ + 0, + 0.0617283950617284 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "true class: upmarket_card, predicted class: upmarket_card" + }, + "zeroline": false + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "figure = results.plot()\n", + "figure.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "figure = results.plot()\n", + "figure.write_image(f'../_static/tutorials/performance_calculation/multiclass/tutorial-confusion-matrix-calculation-multiclass.svg')\n", + "\n", + "# tutorial-perf-est-guide-binary-class-car-loan-analysis\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb b/docs/example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb new file mode 100644 index 000000000..e032da5bd --- /dev/null +++ b/docs/example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
acq_channelapp_behavioral_scorerequested_credit_limitapp_channelcredit_bureau_scorestated_incomeis_customertimestampy_pred_proba_prepaid_cardy_pred_proba_highstreet_cardy_pred_proba_upmarket_cardy_predy_true
0Partner31.808232350web30915000True2020-05-02 02:01:300.970.030.00prepaid_cardprepaid_card
1Partner24.382568500mobile41823000True2020-05-02 02:03:330.870.130.00prepaid_cardprepaid_card
2Partner2-0.787575400web50724000False2020-05-02 02:04:490.470.350.18prepaid_cardupmarket_card
\n", + "
" + ], + "text/plain": [ + " acq_channel app_behavioral_score requested_credit_limit app_channel \\\n", + "0 Partner3 1.808232 350 web \n", + "1 Partner2 4.382568 500 mobile \n", + "2 Partner2 -0.787575 400 web \n", + "\n", + " credit_bureau_score stated_income is_customer timestamp \\\n", + "0 309 15000 True 2020-05-02 02:01:30 \n", + "1 418 23000 True 2020-05-02 02:03:33 \n", + "2 507 24000 False 2020-05-02 02:04:49 \n", + "\n", + " y_pred_proba_prepaid_card y_pred_proba_highstreet_card \\\n", + "0 0.97 0.03 \n", + "1 0.87 0.13 \n", + "2 0.47 0.35 \n", + "\n", + " y_pred_proba_upmarket_card y_pred y_true \n", + "0 0.00 prepaid_card prepaid_card \n", + "1 0.00 prepaid_card prepaid_card \n", + "2 0.18 prepaid_card upmarket_card " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import nannyml as nml\n", + "from IPython.display import display\n", + "\n", + "reference_df, analysis_df, _ = nml.load_synthetic_multiclass_classification_dataset()\n", + "\n", + "display(reference_df.head(3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| | acq_channel | app_behavioral_score | requested_credit_limit | app_channel | credit_bureau_score | stated_income | is_customer | timestamp | y_pred_proba_prepaid_card | y_pred_proba_highstreet_card | y_pred_proba_upmarket_card | y_pred | y_true |\n", + "+====+===============+========================+==========================+===============+=======================+=================+===============+=====================+=============================+================================+==============================+==============+===============+\n", + "| 0 | Partner3 | 1.80823 | 350 | web | 309 | 15000 | True | 2020-05-02 02:01:30 | 0.97 | 0.03 | 0 | prepaid_card | prepaid_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| 1 | Partner2 | 4.38257 | 500 | mobile | 418 | 23000 | True | 2020-05-02 02:03:33 | 0.87 | 0.13 | 0 | prepaid_card | prepaid_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n", + "| 2 | Partner2 | -0.787575 | 400 | web | 507 | 24000 | False | 2020-05-02 02:04:49 | 0.47 | 0.35 | 0.18 | prepaid_card | upmarket_card |\n", + "+----+---------------+------------------------+--------------------------+---------------+-----------------------+-----------------+---------------+---------------------+-----------------------------+--------------------------------+------------------------------+--------------+---------------+\n" + ] + } + ], + "source": [ + "print(reference_df.head(3).to_markdown(tablefmt=\"grid\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "estimator = nml.CBPE(\n", + " y_pred_proba={\n", + " 'prepaid_card': 'y_pred_proba_prepaid_card',\n", + " 'highstreet_card': 'y_pred_proba_highstreet_card',\n", + " 'upmarket_card': 'y_pred_proba_upmarket_card'},\n", + " y_pred='y_pred',\n", + " y_true='y_true',\n", + " timestamp_column_name='timestamp',\n", + " problem_type='classification_multiclass',\n", + " metrics=['confusion_matrix'],\n", + " normalize_confusion_matrix='all',\n", + " chunk_size=6000,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "estimator.fit(reference_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chunktrue_prepaid_card_pred_prepaid_card...true_upmarket_card_pred_highstreet_cardtrue_upmarket_card_pred_upmarket_card
keychunk_indexstart_indexend_indexstart_dateend_dateperiodvaluesampling_errorrealized...lower_thresholdalertvaluesampling_errorrealizedupper_confidence_boundarylower_confidence_boundaryupper_thresholdlower_thresholdalert
0[0:5999]0059992020-09-01 03:10:012020-09-13 16:15:10analysis0.2616740.005555NaN...0.034825False0.2444250.005614NaN0.2612660.2275840.2701060.236227False
1[6000:11999]16000119992020-09-13 16:15:322020-09-25 19:48:42analysis0.2529420.005555NaN...0.034825False0.2474300.005614NaN0.2642710.2305890.2701060.236227False
2[12000:17999]212000179992020-09-25 19:50:042020-10-08 02:53:47analysis0.2568460.005555NaN...0.034825False0.2536690.005614NaN0.2705090.2368280.2701060.236227False
3[18000:23999]318000239992020-10-08 02:57:342020-10-20 15:48:19analysis0.2574720.005555NaN...0.034825False0.2566980.005614NaN0.2735390.2398580.2701060.236227False
4[24000:29999]424000299992020-10-20 15:49:062020-11-01 22:04:40analysis0.2460730.005555NaN...0.034825False0.2568430.005614NaN0.2736830.2400020.2701060.236227False
5[30000:35999]530000359992020-11-01 22:04:592020-11-14 03:55:33analysis0.1667770.005555NaN...0.034825True0.2054660.005614NaN0.2223060.1886250.2701060.236227True
6[36000:41999]636000419992020-11-14 03:55:492020-11-26 09:19:06analysis0.1693170.005555NaN...0.034825True0.1973820.005614NaN0.2142230.1805410.2701060.236227True
7[42000:47999]742000479992020-11-26 09:19:222020-12-08 14:33:56analysis0.1650420.005555NaN...0.034825True0.1960220.005614NaN0.2128620.1791810.2701060.236227True
8[48000:53999]848000539992020-12-08 14:34:252020-12-20 18:30:30analysis0.1680320.005555NaN...0.034825True0.2024740.005614NaN0.2193150.1856340.2701060.236227True
9[54000:59999]954000599992020-12-20 18:31:092021-01-01 22:57:55analysis0.1628610.005555NaN...0.034825True0.2062380.005614NaN0.2230780.1893970.2701060.236227True
\n", + "

10 rows × 79 columns

\n", + "
" + ], + "text/plain": [ + " chunk \\\n", + " key chunk_index start_index end_index start_date \n", + "0 [0:5999] 0 0 5999 2020-09-01 03:10:01 \n", + "1 [6000:11999] 1 6000 11999 2020-09-13 16:15:32 \n", + "2 [12000:17999] 2 12000 17999 2020-09-25 19:50:04 \n", + "3 [18000:23999] 3 18000 23999 2020-10-08 02:57:34 \n", + "4 [24000:29999] 4 24000 29999 2020-10-20 15:49:06 \n", + "5 [30000:35999] 5 30000 35999 2020-11-01 22:04:59 \n", + "6 [36000:41999] 6 36000 41999 2020-11-14 03:55:49 \n", + "7 [42000:47999] 7 42000 47999 2020-11-26 09:19:22 \n", + "8 [48000:53999] 8 48000 53999 2020-12-08 14:34:25 \n", + "9 [54000:59999] 9 54000 59999 2020-12-20 18:31:09 \n", + "\n", + " true_prepaid_card_pred_prepaid_card \\\n", + " end_date period value \n", + "0 2020-09-13 16:15:10 analysis 0.261674 \n", + "1 2020-09-25 19:48:42 analysis 0.252942 \n", + "2 2020-10-08 02:53:47 analysis 0.256846 \n", + "3 2020-10-20 15:48:19 analysis 0.257472 \n", + "4 2020-11-01 22:04:40 analysis 0.246073 \n", + "5 2020-11-14 03:55:33 analysis 0.166777 \n", + "6 2020-11-26 09:19:06 analysis 0.169317 \n", + "7 2020-12-08 14:33:56 analysis 0.165042 \n", + "8 2020-12-20 18:30:30 analysis 0.168032 \n", + "9 2021-01-01 22:57:55 analysis 0.162861 \n", + "\n", + " ... true_upmarket_card_pred_highstreet_card \\\n", + " sampling_error realized ... lower_threshold alert \n", + "0 0.005555 NaN ... 0.034825 False \n", + "1 0.005555 NaN ... 0.034825 False \n", + "2 0.005555 NaN ... 0.034825 False \n", + "3 0.005555 NaN ... 0.034825 False \n", + "4 0.005555 NaN ... 0.034825 False \n", + "5 0.005555 NaN ... 0.034825 True \n", + "6 0.005555 NaN ... 0.034825 True \n", + "7 0.005555 NaN ... 0.034825 True \n", + "8 0.005555 NaN ... 0.034825 True \n", + "9 0.005555 NaN ... 0.034825 True \n", + "\n", + " true_upmarket_card_pred_upmarket_card \\\n", + " value sampling_error realized \n", + "0 0.244425 0.005614 NaN \n", + "1 0.247430 0.005614 NaN \n", + "2 0.253669 0.005614 NaN \n", + "3 0.256698 0.005614 NaN \n", + "4 0.256843 0.005614 NaN \n", + "5 0.205466 0.005614 NaN \n", + "6 0.197382 0.005614 NaN \n", + "7 0.196022 0.005614 NaN \n", + "8 0.202474 0.005614 NaN \n", + "9 0.206238 0.005614 NaN \n", + "\n", + " \\\n", + " upper_confidence_boundary lower_confidence_boundary upper_threshold \n", + "0 0.261266 0.227584 0.270106 \n", + "1 0.264271 0.230589 0.270106 \n", + "2 0.270509 0.236828 0.270106 \n", + "3 0.273539 0.239858 0.270106 \n", + "4 0.273683 0.240002 0.270106 \n", + "5 0.222306 0.188625 0.270106 \n", + "6 0.214223 0.180541 0.270106 \n", + "7 0.212862 0.179181 0.270106 \n", + "8 0.219315 0.185634 0.270106 \n", + "9 0.223078 0.189397 0.270106 \n", + "\n", + " \n", + " lower_threshold alert \n", + "0 0.236227 False \n", + "1 0.236227 False \n", + "2 0.236227 False \n", + "3 0.236227 False \n", + "4 0.236227 False \n", + "5 0.236227 True \n", + "6 0.236227 True \n", + "7 0.236227 True \n", + "8 0.236227 True \n", + "9 0.236227 True \n", + "\n", + "[10 rows x 79 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results = estimator.estimate(analysis_df)\n", + "display(results.filter(period='analysis').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| | | chunk | | | | | | | | true_prepaid_card_pred_prepaid_card | | | | | | | | | true_prepaid_card_pred_highstreet_card | | | | | | | | | true_prepaid_card_pred_upmarket_card | | | | | | | | | true_highstreet_card_pred_prepaid_card | | | | | | | | | true_highstreet_card_pred_highstreet_card | | | | | | | | | true_highstreet_card_pred_upmarket_card | | | | | | | | | true_upmarket_card_pred_prepaid_card | | | | | | | | | true_upmarket_card_pred_highstreet_card | | | | | | | | | true_upmarket_card_pred_upmarket_card | | | | | | | |\n", + "| | | key | | chunk_index | | start_index | | end_index | | start_date | | end_date | | period | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert | | value | | sampling_error | | realized | | upper_confidence_boundary | | lower_confidence_boundary | | upper_threshold | | lower_threshold | | alert |\n", + "+====+===============+=================+=================+===============+=====================+=====================+============+=========================================+====================+==============+===============================+===============================+=====================+=====================+===========+============================================+====================+==============+===============================+===============================+=====================+=====================+===========+==========================================+====================+==============+===============================+===============================+=====================+=====================+===========+============================================+====================+==============+===============================+===============================+=====================+=====================+===========+===============================================+====================+==============+===============================+===============================+=====================+=====================+===========+=============================================+====================+==============+===============================+===============================+=====================+=====================+===========+==========================================+====================+==============+===============================+===============================+=====================+=====================+===========+=============================================+====================+==============+===============================+===============================+=====================+=====================+===========+===========================================+====================+==============+===============================+===============================+=====================+=====================+===========+\n", + "| 0 | [0:5999] | 0 | 0 | 5999 | 2020-09-01 03:10:01 | 2020-09-13 16:15:10 | analysis | 0.261674 | 0.00555482 | nan | 0.278339 | 0.24501 | 0.265912 | 0.243455 | False | 0.0415311 | 0.00265159 | nan | 0.0494858 | 0.0335763 | 0.0526699 | 0.0369635 | False | 0.0379462 | 0.00262565 | nan | 0.0458231 | 0.0300692 | 0.0421872 | 0.0289128 | False | 0.0453291 | 0.00267108 | nan | 0.0533423 | 0.0373158 | 0.0486211 | 0.0396456 | False | 0.247291 | 0.00562464 | nan | 0.264165 | 0.230417 | 0.262292 | 0.228341 | False | 0.0416287 | 0.00239047 | nan | 0.0488002 | 0.0344573 | 0.0470817 | 0.039385 | False | 0.0396632 | 0.00254241 | nan | 0.0472904 | 0.0320359 | 0.0441763 | 0.0331904 | False | 0.0405114 | 0.00248954 | nan | 0.04798 | 0.0330428 | 0.046008 | 0.0348254 | False | 0.244425 | 0.00561357 | nan | 0.261266 | 0.227584 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 1 | [6000:11999] | 1 | 6000 | 11999 | 2020-09-13 16:15:32 | 2020-09-25 19:48:42 | analysis | 0.252942 | 0.00555482 | nan | 0.269606 | 0.236277 | 0.265912 | 0.243455 | False | 0.039106 | 0.00265159 | nan | 0.0470608 | 0.0311512 | 0.0526699 | 0.0369635 | False | 0.0380279 | 0.00262565 | nan | 0.0459048 | 0.0301509 | 0.0421872 | 0.0289128 | False | 0.0438881 | 0.00267108 | nan | 0.0519014 | 0.0358749 | 0.0486211 | 0.0396456 | False | 0.256042 | 0.00562464 | nan | 0.272916 | 0.239169 | 0.262292 | 0.228341 | False | 0.0415422 | 0.00239047 | nan | 0.0487137 | 0.0343708 | 0.0470817 | 0.039385 | False | 0.0386701 | 0.00254241 | nan | 0.0462974 | 0.0310429 | 0.0441763 | 0.0331904 | False | 0.0423516 | 0.00248954 | nan | 0.0498202 | 0.0348829 | 0.046008 | 0.0348254 | False | 0.24743 | 0.00561357 | nan | 0.264271 | 0.230589 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 2 | [12000:17999] | 2 | 12000 | 17999 | 2020-09-25 19:50:04 | 2020-10-08 02:53:47 | analysis | 0.256846 | 0.00555482 | nan | 0.27351 | 0.240181 | 0.265912 | 0.243455 | False | 0.0402242 | 0.00265159 | nan | 0.0481789 | 0.0322694 | 0.0526699 | 0.0369635 | False | 0.0373997 | 0.00262565 | nan | 0.0452766 | 0.0295227 | 0.0421872 | 0.0289128 | False | 0.0425611 | 0.00267108 | nan | 0.0505744 | 0.0345479 | 0.0486211 | 0.0396456 | False | 0.247692 | 0.00562464 | nan | 0.264566 | 0.230818 | 0.262292 | 0.228341 | False | 0.0427649 | 0.00239047 | nan | 0.0499363 | 0.0355935 | 0.0470817 | 0.039385 | False | 0.0390932 | 0.00254241 | nan | 0.0467204 | 0.0314659 | 0.0441763 | 0.0331904 | False | 0.0397506 | 0.00248954 | nan | 0.0472193 | 0.032282 | 0.046008 | 0.0348254 | False | 0.253669 | 0.00561357 | nan | 0.270509 | 0.236828 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 3 | [18000:23999] | 3 | 18000 | 23999 | 2020-10-08 02:57:34 | 2020-10-20 15:48:19 | analysis | 0.257472 | 0.00555482 | nan | 0.274136 | 0.240807 | 0.265912 | 0.243455 | False | 0.0406657 | 0.00265159 | nan | 0.0486205 | 0.032711 | 0.0526699 | 0.0369635 | False | 0.0394442 | 0.00262565 | nan | 0.0473212 | 0.0315673 | 0.0421872 | 0.0289128 | False | 0.0415748 | 0.00267108 | nan | 0.0495881 | 0.0335616 | 0.0486211 | 0.0396456 | False | 0.242519 | 0.00562464 | nan | 0.259393 | 0.225645 | 0.262292 | 0.228341 | False | 0.0413573 | 0.00239047 | nan | 0.0485288 | 0.0341859 | 0.0470817 | 0.039385 | False | 0.03862 | 0.00254241 | nan | 0.0462473 | 0.0309928 | 0.0441763 | 0.0331904 | False | 0.041649 | 0.00248954 | nan | 0.0491176 | 0.0341804 | 0.046008 | 0.0348254 | False | 0.256698 | 0.00561357 | nan | 0.273539 | 0.239858 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 4 | [24000:29999] | 4 | 24000 | 29999 | 2020-10-20 15:49:06 | 2020-11-01 22:04:40 | analysis | 0.246073 | 0.00555482 | nan | 0.262738 | 0.229409 | 0.265912 | 0.243455 | False | 0.0418344 | 0.00265159 | nan | 0.0497891 | 0.0338796 | 0.0526699 | 0.0369635 | False | 0.0380074 | 0.00262565 | nan | 0.0458843 | 0.0301304 | 0.0421872 | 0.0289128 | False | 0.0427521 | 0.00267108 | nan | 0.0507653 | 0.0347388 | 0.0486211 | 0.0396456 | False | 0.250714 | 0.00562464 | nan | 0.267588 | 0.23384 | 0.262292 | 0.228341 | False | 0.0448168 | 0.00239047 | nan | 0.0519882 | 0.0376454 | 0.0470817 | 0.039385 | False | 0.0381747 | 0.00254241 | nan | 0.0458019 | 0.0305475 | 0.0441763 | 0.0331904 | False | 0.040785 | 0.00248954 | nan | 0.0482537 | 0.0333164 | 0.046008 | 0.0348254 | False | 0.256843 | 0.00561357 | nan | 0.273683 | 0.240002 | 0.270106 | 0.236227 | False |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 5 | [30000:35999] | 5 | 30000 | 35999 | 2020-11-01 22:04:59 | 2020-11-14 03:55:33 | analysis | 0.166777 | 0.00555482 | nan | 0.183441 | 0.150112 | 0.265912 | 0.243455 | True | 0.0710278 | 0.00265159 | nan | 0.0789826 | 0.0630731 | 0.0526699 | 0.0369635 | True | 0.0623 | 0.00262565 | nan | 0.070177 | 0.0544231 | 0.0421872 | 0.0289128 | True | 0.0621998 | 0.00267108 | nan | 0.070213 | 0.0541865 | 0.0486211 | 0.0396456 | True | 0.263583 | 0.00562464 | nan | 0.280457 | 0.24671 | 0.262292 | 0.228341 | True | 0.0560677 | 0.00239047 | nan | 0.0632392 | 0.0488963 | 0.0470817 | 0.039385 | True | 0.0495235 | 0.00254241 | nan | 0.0571508 | 0.0418963 | 0.0441763 | 0.0331904 | True | 0.0630554 | 0.00248954 | nan | 0.070524 | 0.0555868 | 0.046008 | 0.0348254 | True | 0.205466 | 0.00561357 | nan | 0.222306 | 0.188625 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 6 | [36000:41999] | 6 | 36000 | 41999 | 2020-11-14 03:55:49 | 2020-11-26 09:19:06 | analysis | 0.169317 | 0.00555482 | nan | 0.185981 | 0.152652 | 0.265912 | 0.243455 | True | 0.0740581 | 0.00265159 | nan | 0.0820128 | 0.0661033 | 0.0526699 | 0.0369635 | True | 0.0585906 | 0.00262565 | nan | 0.0664675 | 0.0507136 | 0.0421872 | 0.0289128 | True | 0.0627503 | 0.00267108 | nan | 0.0707635 | 0.054737 | 0.0486211 | 0.0396456 | True | 0.269904 | 0.00562464 | nan | 0.286778 | 0.25303 | 0.262292 | 0.228341 | True | 0.0535273 | 0.00239047 | nan | 0.0606988 | 0.0463559 | 0.0470817 | 0.039385 | True | 0.0502663 | 0.00254241 | nan | 0.0578936 | 0.0426391 | 0.0441763 | 0.0331904 | True | 0.0642049 | 0.00248954 | nan | 0.0716735 | 0.0567363 | 0.046008 | 0.0348254 | True | 0.197382 | 0.00561357 | nan | 0.214223 | 0.180541 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 7 | [42000:47999] | 7 | 42000 | 47999 | 2020-11-26 09:19:22 | 2020-12-08 14:33:56 | analysis | 0.165042 | 0.00555482 | nan | 0.181707 | 0.148378 | 0.265912 | 0.243455 | True | 0.0729706 | 0.00265159 | nan | 0.0809254 | 0.0650159 | 0.0526699 | 0.0369635 | True | 0.0598608 | 0.00262565 | nan | 0.0677378 | 0.0519839 | 0.0421872 | 0.0289128 | True | 0.06332 | 0.00267108 | nan | 0.0713333 | 0.0553068 | 0.0486211 | 0.0396456 | True | 0.275653 | 0.00562464 | nan | 0.292527 | 0.258779 | 0.262292 | 0.228341 | True | 0.0549509 | 0.00239047 | nan | 0.0621223 | 0.0477794 | 0.0470817 | 0.039385 | True | 0.0471376 | 0.00254241 | nan | 0.0547648 | 0.0395104 | 0.0441763 | 0.0331904 | True | 0.0650432 | 0.00248954 | nan | 0.0725119 | 0.0575746 | 0.046008 | 0.0348254 | True | 0.196022 | 0.00561357 | nan | 0.212862 | 0.179181 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 8 | [48000:53999] | 8 | 48000 | 53999 | 2020-12-08 14:34:25 | 2020-12-20 18:30:30 | analysis | 0.168032 | 0.00555482 | nan | 0.184696 | 0.151367 | 0.265912 | 0.243455 | True | 0.0727937 | 0.00265159 | nan | 0.0807484 | 0.0648389 | 0.0526699 | 0.0369635 | True | 0.0607362 | 0.00262565 | nan | 0.0686131 | 0.0528592 | 0.0421872 | 0.0289128 | True | 0.0638833 | 0.00267108 | nan | 0.0718965 | 0.05587 | 0.0486211 | 0.0396456 | True | 0.266011 | 0.00562464 | nan | 0.282885 | 0.249137 | 0.262292 | 0.228341 | True | 0.056456 | 0.00239047 | nan | 0.0636275 | 0.0492846 | 0.0470817 | 0.039385 | True | 0.0475851 | 0.00254241 | nan | 0.0552123 | 0.0399578 | 0.0441763 | 0.0331904 | True | 0.0620285 | 0.00248954 | nan | 0.0694972 | 0.0545599 | 0.046008 | 0.0348254 | True | 0.202474 | 0.00561357 | nan | 0.219315 | 0.185634 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n", + "| 9 | [54000:59999] | 9 | 54000 | 59999 | 2020-12-20 18:31:09 | 2021-01-01 22:57:55 | analysis | 0.162861 | 0.00555482 | nan | 0.179525 | 0.146196 | 0.265912 | 0.243455 | True | 0.0720836 | 0.00265159 | nan | 0.0800383 | 0.0641288 | 0.0526699 | 0.0369635 | True | 0.0601227 | 0.00262565 | nan | 0.0679997 | 0.0522458 | 0.0421872 | 0.0289128 | True | 0.0596525 | 0.00267108 | nan | 0.0676658 | 0.0516393 | 0.0486211 | 0.0396456 | True | 0.270229 | 0.00562464 | nan | 0.287103 | 0.253355 | 0.262292 | 0.228341 | True | 0.0561397 | 0.00239047 | nan | 0.0633112 | 0.0489683 | 0.0470817 | 0.039385 | True | 0.0491533 | 0.00254241 | nan | 0.0567806 | 0.0415261 | 0.0441763 | 0.0331904 | True | 0.0635208 | 0.00248954 | nan | 0.0709894 | 0.0560522 | 0.046008 | 0.0348254 | True | 0.206238 | 0.00561357 | nan | 0.223078 | 0.189397 | 0.270106 | 0.236227 | True |\n", + "+----+---------------+-----------------+-----------------+---------------+---------------------+---------------------+------------+-----------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+--------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-----------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+---------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+-------------------------------------------+--------------------+--------------+-------------------------------+-------------------------------+---------------------+---------------------+-----------+\n" + ] + } + ], + "source": [ + "from docs.utils import print_multi_index_markdown\n", + "print_multi_index_markdown(results.filter(period='analysis').to_df())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_fig = results.plot()\n", + "metric_fig.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_fig = results.plot()\n", + "metric_fig.write_image(file=f\"../_static/tutorials/performance_estimation/multiclass/tutorial-confusion-matrix-estimation-multiclass-analysis-with-ref.svg\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "EMD", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb b/docs/example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb index 67ce324fd..6d1fa35f5 100644 --- a/docs/example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb +++ b/docs/example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": { "tags": [] }, @@ -135,7 +135,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": { "jupyter": { "outputs_hidden": false @@ -164,13 +164,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": { "jupyter": { "outputs_hidden": false } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "calc = nml.PerformanceCalculator(\n", " y_pred_proba={\n", @@ -190,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": { "jupyter": { "outputs_hidden": false @@ -562,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": { "jupyter": { "outputs_hidden": false @@ -607,7 +618,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": { "jupyter": { "outputs_hidden": false @@ -978,7 +989,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": { "jupyter": { "outputs_hidden": false @@ -1022,13 +1033,4754 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "jupyter": { "outputs_hidden": false } }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.9047604867815185, + 0.9059169458383876, + 0.9093288408003372, + 0.906731007520671, + 0.9105774459778013, + 0.9045772973310676, + 0.906673477964433, + 0.9087028273995331, + 0.9050723814398817, + 0.9097489570920305, + 0.9097489570920305 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized ROC AUC", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.9048, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.9059, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.9093, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.9067, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.9106, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.9046, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.9067, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.9087, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.9051, + 0.0064 + ], + [ + "Realized ROC AUC", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.9097, + 0.0064 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x", + "y": [ + 0.9047604867815185, + 0.9059169458383876, + 0.9093288408003372, + 0.906731007520671, + 0.9105774459778013, + 0.9045772973310676, + 0.906673477964433, + 0.9087028273995331, + 0.9050723814398817, + 0.9097489570920305, + 0.9097489570920305 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.9075949825507551, + 0.9105338078866886, + 0.9094139354389862, + 0.9115770759369158, + 0.9075329211230677, + 0.7485876652639005, + 0.7511366577551554, + 0.7563992252467017, + 0.7585606818895146, + 0.75393731620128, + 0.75393731620128 + ], + "yaxis": "y" + }, + { + "customdata": [ + [ + "Realized ROC AUC", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.9076, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.9105, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.9094, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.9116, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.9075, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.7486, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.7511, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.7564, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.7586, + 0.0064 + ], + [ + "Realized ROC AUC", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.7539, + 0.0064 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x", + "y": [ + 0.9075949825507551, + 0.9105338078866886, + 0.9094139354389862, + 0.9115770759369158, + 0.9075329211230677, + 0.7485876652639005, + 0.7511366577551554, + 0.7563992252467017, + 0.7585606818895146, + 0.75393731620128, + 0.75393731620128 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x", + "y": [ + 0.7485876652639005, + 0.7511366577551554, + 0.7563992252467017, + 0.7585606818895146, + 0.75393731620128 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": true, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x", + "y": [ + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074, + 0.9135156728918074 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x", + "y": [ + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325, + 0.900902260737325 + ], + "yaxis": "y" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.7505323557358875, + 0.7511484640126803, + 0.7571400512722031, + 0.7502740596433638, + 0.7591436975867962, + 0.7486300734249819, + 0.7527627058654386, + 0.7558827980804009, + 0.7474196344836103, + 0.7580550513577399, + 0.7580550513577399 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized F1", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.7505, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.7511, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.7571, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.7503, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.7591, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.7486, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.7528, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.7559, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.7474, + 0.017 + ], + [ + "Realized F1", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.7581, + 0.017 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x2", + "y": [ + 0.7505323557358875, + 0.7511484640126803, + 0.7571400512722031, + 0.7502740596433638, + 0.7591436975867962, + 0.7486300734249819, + 0.7527627058654386, + 0.7558827980804009, + 0.7474196344836103, + 0.7580550513577399, + 0.7580550513577399 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.7511029725567399, + 0.763045484661255, + 0.7584871119660588, + 0.7589438350894463, + 0.7579637647538195, + 0.5571123937358071, + 0.5591478583434738, + 0.5650552384148319, + 0.5638970959051086, + 0.5616437963495821, + 0.5616437963495821 + ], + "yaxis": "y2" + }, + { + "customdata": [ + [ + "Realized F1", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.7511, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.763, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.7585, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.7589, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.758, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.5571, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.5591, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.5651, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.5639, + 0.017 + ], + [ + "Realized F1", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.5616, + 0.017 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x2", + "y": [ + 0.7511029725567399, + 0.763045484661255, + 0.7584871119660588, + 0.7589438350894463, + 0.7579637647538195, + 0.5571123937358071, + 0.5591478583434738, + 0.5650552384148319, + 0.5638970959051086, + 0.5616437963495821, + 0.5616437963495821 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x2", + "y": [ + 0.5571123937358071, + 0.5591478583434738, + 0.5650552384148319, + 0.5638970959051086, + 0.5616437963495821 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x2", + "y": [ + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994, + 0.7649438592270994 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x2", + "y": [ + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521, + 0.741253919065521 + ], + "yaxis": "y2" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.7504997877145659, + 0.7511423421157434, + 0.7572795318438779, + 0.7504146231097971, + 0.7591747693336198, + 0.7486602875939433, + 0.7526838124010954, + 0.75581974760633, + 0.7474411436189555, + 0.758077211499174, + 0.758077211499174 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized Precision", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.7505, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.7511, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.7573, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.7504, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.7592, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.7487, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.7527, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.7558, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.7474, + 0.0167 + ], + [ + "Realized Precision", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.7581, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x3", + "y": [ + 0.7504997877145659, + 0.7511423421157434, + 0.7572795318438779, + 0.7504146231097971, + 0.7591747693336198, + 0.7486602875939433, + 0.7526838124010954, + 0.75581974760633, + 0.7474411436189555, + 0.758077211499174, + 0.758077211499174 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.7512696944958154, + 0.763125096320063, + 0.7585026532532758, + 0.7589734049826605, + 0.7579497585294958, + 0.5596958396310442, + 0.5629149496148481, + 0.5690688453209285, + 0.5666732930015904, + 0.5651292932033183, + 0.5651292932033183 + ], + "yaxis": "y3" + }, + { + "customdata": [ + [ + "Realized Precision", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.7513, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.7631, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.7585, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.759, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.7579, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.5597, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.5629, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.5691, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.5667, + 0.0167 + ], + [ + "Realized Precision", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.5651, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x3", + "y": [ + 0.7512696944958154, + 0.763125096320063, + 0.7585026532532758, + 0.7589734049826605, + 0.7579497585294958, + 0.5596958396310442, + 0.5629149496148481, + 0.5690688453209285, + 0.5666732930015904, + 0.5651292932033183, + 0.5651292932033183 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x3", + "y": [ + 0.5596958396310442, + 0.5629149496148481, + 0.5690688453209285, + 0.5666732930015904, + 0.5651292932033183 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x3", + "y": [ + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447, + 0.7649778261800447 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x3", + "y": [ + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757, + 0.7412608251873757 + ], + "yaxis": "y3" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.7505756565812259, + 0.7511570710053205, + 0.7571742725136351, + 0.750240785251214, + 0.7591965868948084, + 0.7486570029800302, + 0.7529438552767722, + 0.7560061060614122, + 0.7474359276798309, + 0.7580515492401371, + 0.7580515492401371 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized Recall", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.7506, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.7512, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.7572, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.7502, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.7592, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.7487, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.7529, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.756, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.7474, + 0.0167 + ], + [ + "Realized Recall", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.7581, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x4", + "y": [ + 0.7505756565812259, + 0.7511570710053205, + 0.7571742725136351, + 0.750240785251214, + 0.7591965868948084, + 0.7486570029800302, + 0.7529438552767722, + 0.7560061060614122, + 0.7474359276798309, + 0.7580515492401371, + 0.7580515492401371 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.751032542219961, + 0.7631482722478959, + 0.7584839425208738, + 0.7589862629444336, + 0.7579793699050997, + 0.5578286791733232, + 0.5601696314300518, + 0.5659426670075174, + 0.5647234307451827, + 0.5627719243579322, + 0.5627719243579322 + ], + "yaxis": "y4" + }, + { + "customdata": [ + [ + "Realized Recall", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.751, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.7631, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.7585, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.759, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.758, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.5578, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.5602, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.5659, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.5647, + 0.0167 + ], + [ + "Realized Recall", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.5628, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x4", + "y": [ + 0.751032542219961, + 0.7631482722478959, + 0.7584839425208738, + 0.7589862629444336, + 0.7579793699050997, + 0.5578286791733232, + 0.5601696314300518, + 0.5659426670075174, + 0.5647234307451827, + 0.5627719243579322, + 0.5627719243579322 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x4", + "y": [ + 0.5578286791733232, + 0.5601696314300518, + 0.5659426670075174, + 0.5647234307451827, + 0.5627719243579322 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x4", + "y": [ + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402, + 0.7650263362299402 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x4", + "y": [ + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369, + 0.7412614264669369 + ], + "yaxis": "y4" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.8752257186404249, + 0.8754239802495872, + 0.8786275988842677, + 0.8751632156297671, + 0.8796861029875632, + 0.8743285866322269, + 0.8764072702343472, + 0.8778905538531246, + 0.8738321876216824, + 0.8791772887984904, + 0.8791772887984904 + ], + "yaxis": "y5" + }, + { + "customdata": [ + [ + "Realized Specificity", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.8752, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.8754, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.8786, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.8752, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.8797, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.8743, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.8764, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.8779, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.8738, + 0.009 + ], + [ + "Realized Specificity", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.8792, + 0.009 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x5", + "y": [ + 0.8752257186404249, + 0.8754239802495872, + 0.8786275988842677, + 0.8751632156297671, + 0.8796861029875632, + 0.8743285866322269, + 0.8764072702343472, + 0.8778905538531246, + 0.8738321876216824, + 0.8791772887984904, + 0.8791772887984904 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.8755496969740512, + 0.8815079899320031, + 0.8793672267258789, + 0.8796300273907566, + 0.8789858259316669, + 0.7799051783587029, + 0.7806759535472807, + 0.7842227303455768, + 0.7834220052802948, + 0.7824290248406834, + 0.7824290248406834 + ], + "yaxis": "y5" + }, + { + "customdata": [ + [ + "Realized Specificity", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.8755, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.8815, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.8794, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.8796, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.879, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.7799, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.7807, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.7842, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.7834, + 0.009 + ], + [ + "Realized Specificity", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.7824, + 0.009 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x5", + "y": [ + 0.8755496969740512, + 0.8815079899320031, + 0.8793672267258789, + 0.8796300273907566, + 0.8789858259316669, + 0.7799051783587029, + 0.7806759535472807, + 0.7842227303455768, + 0.7834220052802948, + 0.7824290248406834, + 0.7824290248406834 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x5", + "y": [ + 0.7799051783587029, + 0.7806759535472807, + 0.7842227303455768, + 0.7834220052802948, + 0.7824290248406834 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x5", + "y": [ + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304, + 0.8825860497313304 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x5", + "y": [ + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661, + 0.8705664509749661 + ], + "yaxis": "y5" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_reference", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.7505, + 0.751, + 0.7571666666666667, + 0.7503333333333333, + 0.7593333333333333, + 0.7486666666666667, + 0.7528333333333334, + 0.7558333333333334, + 0.7476666666666667, + 0.7583333333333333, + 0.7583333333333333 + ], + "yaxis": "y6" + }, + { + "customdata": [ + [ + "Realized Accuracy", + "Reference", + "", + "[0:5999]", + "From May-02-2020 to May-14-2020", + 0.7505, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[6000:11999]", + "From May-14-2020 to May-26-2020", + 0.751, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[12000:17999]", + "From May-26-2020 to Jun-07-2020", + 0.7572, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[18000:23999]", + "From Jun-07-2020 to Jun-19-2020", + 0.7503, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[24000:29999]", + "From Jun-19-2020 to Jul-02-2020", + 0.7593, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[30000:35999]", + "From Jul-02-2020 to Jul-14-2020", + 0.7487, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[36000:41999]", + "From Jul-14-2020 to Jul-26-2020", + 0.7528, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[42000:47999]", + "From Jul-26-2020 to Aug-07-2020", + 0.7558, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[48000:53999]", + "From Aug-07-2020 to Aug-20-2020", + 0.7477, + 0.0167 + ], + [ + "Realized Accuracy", + "Reference", + "", + "[54000:59999]", + "From Aug-20-2020 to Sep-01-2020", + 0.7583, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_reference", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-08T07:15:27.500000", + "2020-05-20T15:30:15.500000", + "2020-06-01T19:14:52.500000", + "2020-06-13T19:51:26.500000", + "2020-06-25T22:55:35", + "2020-07-08T05:10:32", + "2020-07-20T10:35:52.500000", + "2020-08-01T14:45:40.500000", + "2020-08-13T20:20:51", + "2020-08-26T01:35:40.500000" + ], + "xaxis": "x6", + "y": [ + 0.7505, + 0.751, + 0.7571666666666667, + 0.7503333333333333, + 0.7593333333333333, + 0.7486666666666667, + 0.7528333333333334, + 0.7558333333333334, + 0.7476666666666667, + 0.7583333333333333, + 0.7583333333333333 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "metric_analysis", + "line": { + "color": "#00c8e5", + "shape": "hv", + "width": 2 + }, + "mode": "lines", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.7511666666666666, + 0.763, + 0.7586666666666667, + 0.7591666666666667, + 0.758, + 0.5608333333333333, + 0.5623333333333334, + 0.5698333333333333, + 0.5678333333333333, + 0.566, + 0.566 + ], + "yaxis": "y6" + }, + { + "customdata": [ + [ + "Realized Accuracy", + "Analysis", + "", + "[0:5999]", + "From Sep-01-2020 to Sep-13-2020", + 0.7512, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "", + "[6000:11999]", + "From Sep-13-2020 to Sep-25-2020", + 0.763, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "", + "[12000:17999]", + "From Sep-25-2020 to Oct-08-2020", + 0.7587, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "", + "[18000:23999]", + "From Oct-08-2020 to Oct-20-2020", + 0.7592, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "", + "[24000:29999]", + "From Oct-20-2020 to Nov-01-2020", + 0.758, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "âš  Drift detected", + "[30000:35999]", + "From Nov-01-2020 to Nov-14-2020", + 0.5608, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "âš  Drift detected", + "[36000:41999]", + "From Nov-14-2020 to Nov-26-2020", + 0.5623, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "âš  Drift detected", + "[42000:47999]", + "From Nov-26-2020 to Dec-08-2020", + 0.5698, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "âš  Drift detected", + "[48000:53999]", + "From Dec-08-2020 to Dec-20-2020", + 0.5678, + 0.0167 + ], + [ + "Realized Accuracy", + "Analysis", + "âš  Drift detected", + "[54000:59999]", + "From Dec-20-2020 to Jan-01-2021", + 0.566, + 0.0167 + ] + ], + "hovertemplate": "%{customdata[1]}     %{customdata[2]}
Chunk: %{customdata[3]}     %{customdata[4]}
%{customdata[0]}: %{customdata[5]}
Confidence band: +/- %{customdata[6]}
", + "legendgroup": "metric_analysis", + "marker": { + "color": "#00c8e5", + "size": 5, + "symbol": "circle" + }, + "mode": "markers", + "name": "Metric", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-07T09:42:46.500000", + "2020-09-19T18:02:48", + "2020-10-01T23:23:49", + "2020-10-14T09:23:20", + "2020-10-26T18:57:02.500000", + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x6", + "y": [ + 0.7511666666666666, + 0.763, + 0.7586666666666667, + 0.7591666666666667, + 0.758, + 0.5608333333333333, + 0.5623333333333334, + 0.5698333333333333, + 0.5678333333333333, + 0.566, + 0.566 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "alert", + "marker": { + "color": "#DD4040", + "size": 8, + "symbol": "diamond" + }, + "mode": "markers", + "name": "Alert", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-11-08T01:00:24", + "2020-11-20T06:37:35.500000", + "2020-12-02T11:56:53.500000", + "2020-12-14T16:32:47", + "2020-12-26T20:44:32" + ], + "xaxis": "x6", + "y": [ + 0.5608333333333333, + 0.5623333333333334, + 0.5698333333333333, + 0.5678333333333333, + 0.566 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-05-02T02:01:30", + "2020-05-14T12:29:25", + "2020-05-26T18:31:06", + "2020-06-07T19:58:39", + "2020-06-19T19:44:14", + "2020-07-02T02:06:56", + "2020-07-14T08:14:08", + "2020-07-26T12:57:37", + "2020-08-07T16:33:44", + "2020-08-20T00:07:58", + "2020-09-01T03:03:23" + ], + "xaxis": "x6", + "y": [ + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984, + 0.7651019085996984 + ], + "yaxis": "y6" + }, + { + "hoverinfo": "skip", + "legendgroup": "thresh", + "line": { + "color": "#DD4040", + "dash": "dash", + "width": 2 + }, + "mode": "lines", + "name": "Threshold", + "showlegend": false, + "type": "scatter", + "x": [ + "2020-09-01T03:10:01", + "2020-09-13T16:15:32", + "2020-09-25T19:50:04", + "2020-10-08T02:57:34", + "2020-10-20T15:49:06", + "2020-11-01T22:04:59", + "2020-11-14T03:55:49", + "2020-11-26T09:19:22", + "2020-12-08T14:34:25", + "2020-12-20T18:31:09", + "2021-01-01T22:57:55" + ], + "xaxis": "x6", + "y": [ + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349, + 0.7412314247336349 + ], + "yaxis": "y6" + } + ], + "layout": { + "annotations": [ + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized ROC AUC", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.9999999999999999, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized F1", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.8194444444444444, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized Precision", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.638888888888889, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized Recall", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.4583333333333333, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized Specificity", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.2777777777777778, + "yanchor": "bottom", + "yref": "paper" + }, + { + "font": { + "size": 16 + }, + "showarrow": false, + "text": "Realized Accuracy", + "x": 0.5, + "xanchor": "center", + "xref": "paper", + "y": 0.09722222222222222, + "yanchor": "bottom", + "yref": "paper" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x", + "xshift": 10, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x", + "xshift": 15, + "y": 1.01, + "yref": "y domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x2", + "xshift": 10, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x2", + "xshift": 15, + "y": 1.01, + "yref": "y2 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x3", + "xshift": 10, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x3", + "xshift": 15, + "y": 1.01, + "yref": "y3 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x4", + "xshift": 10, + "y": 1.01, + "yref": "y4 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x4", + "xshift": 15, + "y": 1.01, + "yref": "y4 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x5", + "xshift": 10, + "y": 1.01, + "yref": "y5 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x5", + "xshift": 15, + "y": 1.01, + "yref": "y5 domain" + }, + { + "showarrow": false, + "text": "Reference", + "x": "2020-06-26T01:40:31.700000", + "xref": "x6", + "xshift": 10, + "y": 1.01, + "yref": "y6 domain" + }, + { + "showarrow": false, + "text": "Analysis", + "x": "2020-10-26T15:02:48.100000", + "xref": "x6", + "xshift": 15, + "y": 1.01, + "yref": "y6 domain" + } + ], + "height": 3000, + "hoverlabel": { + "bgcolor": "white", + "font": { + "size": 14 + } + }, + "legend": { + "itemclick": false, + "itemdoubleclick": false, + "traceorder": "grouped" + }, + "paper_bgcolor": "rgba(255,255,255,1)", + "plot_bgcolor": "rgba(255,255,255,1)", + "shapes": [ + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x", + "y0": 0, + "y1": 1, + "yref": "y domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x2", + "y0": 0, + "y1": 1, + "yref": "y2 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x3", + "y0": 0, + "y1": 1, + "yref": "y3 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x4", + "y0": 0, + "y1": 1, + "yref": "y4 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x5", + "y0": 0, + "y1": 1, + "yref": "y5 domain" + }, + { + "layer": "below", + "line": { + "color": "#666666", + "width": 1 + }, + "type": "line", + "x0": "2020-09-01T03:10:01", + "x1": "2020-09-01T03:10:01", + "xref": "x6", + "y0": 0, + "y1": 1, + "yref": "y6 domain" + } + ], + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Realized performance" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis2": { + "anchor": "y2", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis3": { + "anchor": "y3", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis4": { + "anchor": "y4", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis5": { + "anchor": "y5", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "xaxis6": { + "anchor": "y6", + "domain": [ + 0, + 1 + ], + "linecolor": "#3b0280", + "matches": "x", + "mirror": true, + "showgrid": false, + "title": { + "text": "Time" + }, + "zeroline": false + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0.9027777777777777, + 0.9999999999999999 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "ROC AUC" + }, + "zeroline": false + }, + "yaxis2": { + "anchor": "x2", + "domain": [ + 0.7222222222222222, + 0.8194444444444444 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "F1" + }, + "zeroline": false + }, + "yaxis3": { + "anchor": "x3", + "domain": [ + 0.5416666666666667, + 0.638888888888889 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "Precision" + }, + "zeroline": false + }, + "yaxis4": { + "anchor": "x4", + "domain": [ + 0.3611111111111111, + 0.4583333333333333 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "Recall" + }, + "zeroline": false + }, + "yaxis5": { + "anchor": "x5", + "domain": [ + 0.18055555555555555, + 0.2777777777777778 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "Specificity" + }, + "zeroline": false + }, + "yaxis6": { + "anchor": "x6", + "domain": [ + 0, + 0.09722222222222222 + ], + "linecolor": "#3b0280", + "mirror": true, + "showgrid": false, + "title": { + "text": "Accuracy" + }, + "zeroline": false + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "figure = results.plot()\n", "figure.show()" @@ -1036,7 +5788,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "jupyter": { "outputs_hidden": false diff --git a/docs/requirements.txt b/docs/requirements.txt index 953dfedc1..56646f0ce 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ sphinx-toggleprompt==0.2.0 sphinx-copybutton==0.5.0 sphinx-toolbox==3.2.0 sphinx==4.5.0 +sphinx-rtd-theme==1.3.0 diff --git a/docs/tutorials/performance_calculation/multiclass_performance_calculation.rst b/docs/tutorials/performance_calculation/multiclass_performance_calculation.rst index c00eb5489..16b10d3c7 100644 --- a/docs/tutorials/performance_calculation/multiclass_performance_calculation.rst +++ b/docs/tutorials/performance_calculation/multiclass_performance_calculation.rst @@ -4,133 +4,25 @@ Monitoring Realized Performance for Multiclass Classification ================================================================ -.. note:: - The following example uses :term:`timestamps`. - These are optional but have an impact on the way data is chunked and results are plotted. - You can read more about them in the :ref:`data requirements`. +We currently support the following **standard** metrics for multiclass classification performance calculation: + * **roc_auc** + * **f1** + * **precision** + * **recall** + * **specificity** + * **accuracy** +For more information about estimating these metrics, refer to the :ref:`multiclass-standard-metric-calculation` section. -Just The Code -============== +We also support the following *complex* metric for multiclass classification performance calculation: -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 1 3 4 6 8 + * **confusion_matrix** -.. admonition:: **Advanced configuration** - :class: hint +For more information about estimating this metrics, refer to the :ref:`multiclass-confusion-matrix-estimation` section. - - To learn how :class:`~nannyml.chunk.Chunk` works and to set up custom chunkings check out the :ref:`chunking tutorial ` - - To learn how :class:`~nannyml.thresholds.ConstantThreshold` works and to set up custom threshold check out the :ref:`thresholds tutorial ` +.. toctree:: + :maxdepth: 2 -Walkthrough -============= - - -For simplicity this guide is based on a synthetic dataset where the monitored model predicts -which type of credit card product new customers should be assigned to. -Check out :ref:`Credit Card Dataset` to learn more about this dataset. - -In order to monitor a model, NannyML needs to learn about it from a reference dataset. -Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. -You can read more about this in our section on :ref:`data periods` - -The ``analysis_targets_df`` dataframe contains the target results of the analysis period. This is kept separate in the -synthetic data because it is not used during :ref:`performance estimation`. But it is required -to calculate performance, so the first thing we need to in this case is set up the right data in the right dataframes. -The analysis target values are joined on the analysis frame by the **identifier** column. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 1 - -.. nbtable:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cell: 2 - -Next a :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is created using a list of metrics to calculate -(or just one metric), the data columns required for these metrics, and an optional :ref:`chunking` specification. - -The list of metrics specifies which performance metrics of the monitored model will be calculated. -The following metrics are currently supported: - -- ``roc_auc`` - one-vs-the-rest, macro-averaged -- ``f1`` - macro-averaged -- ``precision`` - macro-averaged -- ``recall`` - macro-averaged -- ``specificity`` - macro-averaged -- ``accuracy`` - -For more information on metrics, check the :mod:`~nannyml.performance_calculation.metrics` module. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 3 - - -The new :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is fitted using the -:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.fit` method on the reference data. - -The fitted :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` can then be used to calculate -realized performance metrics on all data which has target values available with the -:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.calculate` method. -NannyML can output a dataframe that contains all the results of the analysis data. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 4 - -.. nbtable:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cell: 5 - -The results from the reference data are also available. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 6 - -.. nbtable:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cell: 7 - -Apart from chunk-related data, the results data have a set of columns for each calculated metric. - - - **targets_missing_rate** - The fraction of missing target data. - - **value** - the realized metric value for a specific chunk. - - **sampling_error** - the estimate of the :term:`Sampling Error`. - - **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant - performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in - the reference partition. The thresholds are 3 standard deviations away from the mean performance calculated on - chunks. - They are calculated during ``fit`` phase. You can also set up custom thresholds using constant or standard deviations thresholds, - to learn more about it check out our :ref:`tutorial on thresholds`. - - **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses - upper or lower threshold. - -The results can be plotted for visual inspection: - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb - :cells: 8 - -.. image:: /_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg - - -Insights -======== - -After reviewing the performance calculation results, we should be able to clearly see how the model is performing against -the targets, according to whatever metrics we wish to track. - - -What Next -========= - -If we decide further investigation is needed, the :ref:`Data Drift` functionality can help us to see -what feature changes may be contributing to any performance changes. We can also plot the realized performance -and :ref:`compare it with the estimated results`. - -It is also wise to check whether the model's performance is satisfactory -according to business requirements. This is an ad-hoc investigation that is not covered by NannyML. + multiclass_performance_calculation/standard_metric_calculation + multiclass_performance_calculation/confusion_matrix_calculation diff --git a/docs/tutorials/performance_calculation/multiclass_performance_calculation/confusion_matrix_calculation.rst b/docs/tutorials/performance_calculation/multiclass_performance_calculation/confusion_matrix_calculation.rst new file mode 100644 index 000000000..8b1c2798a --- /dev/null +++ b/docs/tutorials/performance_calculation/multiclass_performance_calculation/confusion_matrix_calculation.rst @@ -0,0 +1,163 @@ +.. _multiclass-confusion-matrix-calculation: + +=================================================================== +Calculating Confusion Matrix Elements for Multiclass Classification +=================================================================== + +This tutorial explains how to use NannyML to calculate the :term:`confusion matrix` for multiclass classification +models. + +.. note:: + The following example uses :term:`timestamps`. + These are optional but have an impact on the way data is chunked and results are plotted. + You can read more about them in the :ref:`data requirements`. + +.. _confusion-matrix-calculation-multiclass-just-the-code: + +Just The Code +---------------- + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 1 3 4 5 7 9 + + +Walkthrough +-------------- + +For simplicity this guide is based on a synthetic dataset where the monitored model predicts +which type of credit card product new customers should be assigned to. +Check out :ref:`Credit Card Dataset` to learn more about this dataset. + +In order to monitor a model, NannyML needs to learn about it from a reference dataset. Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. +You can read more about this in our section on :ref:`data periods`. + +The ``analysis_targets`` dataframe contains the target results of the analysis period. This is kept separate in the synthetic data because it is +not used during :ref:`performance estimation`. But it is required to calculate performance, so the first thing we need to in this case is set up the right data in the right dataframes. + +The analysis target values are joined on the analysis frame by their index. Your dataset may already contain the **target** column, so you may skip this join. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 1 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cell: 2 + +Next a :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is created using +the following: + + - **y_pred_proba:** the name of the column in the reference data that + contains the predicted probabilities. + - **y_pred:** the name of the column in the reference data that + contains the predicted classes. + - **y_true:** the name of the column in the reference data that + contains the true classes. + - **timestamp_column_name (Optional):** the name of the column in the reference data that + contains timestamps. + - **problem_type:** the type of problem being monitored. In this example we + will monitor a binary classification problem. + - **metrics:** a list of metrics to calculate. In this example we + will calculate the ``confusion_matrix`` metric. + - **normalize_confusion_matrix (Optional):** how to normalize the confusion matrix. + The normalization options are: + + * **None** : returns counts for each cell + * **"true"** : normalize over the true class of observations. + * **"pred"** : normalize over the predicted class of observations + * **"all"** : normalize over all observations + + - **chunk_size (Optional):** the number of observations in each chunk of data + used to calculate performance. For more information about + :term:`chunking` other chunking options check out the :ref:`chunking tutorial`. + - **thresholds (Optional):** the thresholds used to calculate the alert flag. For more information about + thresholds, check out the :ref:`thresholds tutorial`. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 3 + + +The new :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is fitted using the +:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.fit` method on the ``reference`` data. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 4 + +The fitted :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` can then be used to calculate +realized performance metrics on all data which has target values available with the +:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.calculate` method. +NannyML can output a dataframe that contains all the results of the analysis data. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 5 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cell: 6 + +The results from the reference data are also available. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 7 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cell: 8 + +Apart from chunk and period-related columns, the results data have a set of columns for each +calculated metric. + +- **targets_missing_rate** - The fraction of missing target data. +- **value** - the realized metric value for a specific chunk. +- **sampling_error** - the estimate of the :term:`Sampling Error`. +- **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant + performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in + the **reference** partition. The thresholds are 3 standard deviations away from the mean performance calculated on + chunks. + They are calculated during **fit** phase. +- **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses + upper or lower threshold. + +The results can be plotted for visual inspection. Our plot contains several key elements. + +* *The purple step plot* shows the performance in each chunk of the analysis period. Thick squared point + markers indicate the middle of these chunks. + +* *The blue step plot* shows the performance in each chunk of the reference period. Thick squared point markers indicate + the middle of these chunks. + +* *The gray vertical line* splits the reference and analysis periods. + +* *The red horizontal dashed lines* show upper and lower thresholds for alerting purposes. + +* *The red diamond-shaped point markers* in the middle of a chunk indicate that an alert has been raised. Alerts are caused by the performance crossing the upper or lower threshold. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Calculating Confusion Matrix - Multiclass Classification.ipynb + :cells: 9 + +.. image:: /_static/tutorials/performance_calculation/binary/tutorial-confusion-matrix-calculation-binary-car-loan-analysis.svg + +Additional information such as the chunk index range and chunk date range (if timestamps were provided) is shown in the hover for each chunk (these are +interactive plots, though only static views are included here). + +Insights +-------- + +After reviewing the performance calculation results, we should be able to clearly see how the model is performing against +the targets, according to whatever metrics we wish to track. + + +What's Next +----------- + +If we decide further investigation is needed, the :ref:`Data Drift` functionality can help us to see +what feature changes may be contributing to any performance changes. + +It is also wise to check whether the model's performance is satisfactory +according to business requirements. This is an ad-hoc investigation that is not covered by NannyML. diff --git a/docs/tutorials/performance_calculation/multiclass_performance_calculation/standard_metric_calculation.rst b/docs/tutorials/performance_calculation/multiclass_performance_calculation/standard_metric_calculation.rst new file mode 100644 index 000000000..d42fdc79f --- /dev/null +++ b/docs/tutorials/performance_calculation/multiclass_performance_calculation/standard_metric_calculation.rst @@ -0,0 +1,136 @@ +.. _multiclass-standard-metric-calculation: + +====================================================================== +Calculating Standard Performance Metrics for Multiclass Classification +====================================================================== + +.. note:: + The following example uses :term:`timestamps`. + These are optional but have an impact on the way data is chunked and results are plotted. + You can read more about them in the :ref:`data requirements`. + + + +Just The Code +============== + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 1 3 4 6 8 + +.. admonition:: **Advanced configuration** + :class: hint + + - To learn how :class:`~nannyml.chunk.Chunk` works and to set up custom chunkings check out the :ref:`chunking tutorial ` + - To learn how :class:`~nannyml.thresholds.ConstantThreshold` works and to set up custom threshold check out the :ref:`thresholds tutorial ` + +Walkthrough +============= + + +For simplicity this guide is based on a synthetic dataset where the monitored model predicts +which type of credit card product new customers should be assigned to. +Check out :ref:`Credit Card Dataset` to learn more about this dataset. + +In order to monitor a model, NannyML needs to learn about it from a reference dataset. +Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. +You can read more about this in our section on :ref:`data periods` + +The ``analysis_targets_df`` dataframe contains the target results of the analysis period. This is kept separate in the +synthetic data because it is not used during :ref:`performance estimation`. But it is required +to calculate performance, so the first thing we need to in this case is set up the right data in the right dataframes. +The analysis target values are joined on the analysis frame by the **identifier** column. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 1 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cell: 2 + +Next a :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is created using a list of metrics to calculate +(or just one metric), the data columns required for these metrics, and an optional :ref:`chunking` specification. + +The list of metrics specifies which performance metrics of the monitored model will be calculated. +The following metrics are currently supported: + +- ``roc_auc`` - one-vs-the-rest, macro-averaged +- ``f1`` - macro-averaged +- ``precision`` - macro-averaged +- ``recall`` - macro-averaged +- ``specificity`` - macro-averaged +- ``accuracy`` + +For more information on metrics, check the :mod:`~nannyml.performance_calculation.metrics` module. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 3 + + +The new :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` is fitted using the +:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.fit` method on the reference data. + +The fitted :class:`~nannyml.performance_calculation.calculator.PerformanceCalculator` can then be used to calculate +realized performance metrics on all data which has target values available with the +:meth:`~nannyml.performance_calculation.calculator.PerformanceCalculator.calculate` method. +NannyML can output a dataframe that contains all the results of the analysis data. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 4 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cell: 5 + +The results from the reference data are also available. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 6 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cell: 7 + +Apart from chunk-related data, the results data have a set of columns for each calculated metric. + + - **targets_missing_rate** - The fraction of missing target data. + - **value** - the realized metric value for a specific chunk. + - **sampling_error** - the estimate of the :term:`Sampling Error`. + - **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant + performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in + the reference partition. The thresholds are 3 standard deviations away from the mean performance calculated on + chunks. + They are calculated during ``fit`` phase. You can also set up custom thresholds using constant or standard deviations thresholds, + to learn more about it check out our :ref:`tutorial on thresholds`. + - **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses + upper or lower threshold. + +The results can be plotted for visual inspection: + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Realized Performance - Multiclass Classification.ipynb + :cells: 8 + +.. image:: /_static/tutorials/performance_calculation/multiclass/tutorial-performance-calculation-multiclass.svg + + +Insights +======== + +After reviewing the performance calculation results, we should be able to clearly see how the model is performing against +the targets, according to whatever metrics we wish to track. + + +What Next +========= + +If we decide further investigation is needed, the :ref:`Data Drift` functionality can help us to see +what feature changes may be contributing to any performance changes. We can also plot the realized performance +and :ref:`compare it with the estimated results`. + +It is also wise to check whether the model's performance is satisfactory +according to business requirements. This is an ad-hoc investigation that is not covered by NannyML. \ No newline at end of file diff --git a/docs/tutorials/performance_estimation.rst b/docs/tutorials/performance_estimation.rst index e02b21d99..b72e14b73 100644 --- a/docs/tutorials/performance_estimation.rst +++ b/docs/tutorials/performance_estimation.rst @@ -5,7 +5,7 @@ Estimating Performance ============================================ .. toctree:: - :maxdepth: 3 + :maxdepth: 2 performance_estimation/why_estimate_performance performance_estimation/binary_performance_estimation diff --git a/docs/tutorials/performance_estimation/multiclass_performance_estimation.rst b/docs/tutorials/performance_estimation/multiclass_performance_estimation.rst index 49e575b05..e87136bcd 100644 --- a/docs/tutorials/performance_estimation/multiclass_performance_estimation.rst +++ b/docs/tutorials/performance_estimation/multiclass_performance_estimation.rst @@ -1,144 +1,29 @@ .. _multiclass-performance-estimation: -==================================================== +================================================ Estimating Performance for Multiclass Classification -==================================================== +================================================ -This tutorial explains how to use NannyML to estimate the performance of binary classification -models in the absence of target data. To find out how :class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE` estimates performance, read the :ref:`explanation of Confidence-based -Performance Estimation`. +We currently support the following **standard** metrics for multiclass classification performance estimation: -.. note:: - The following example uses :term:`timestamps`. - These are optional but have an impact on the way data is chunked and results are plotted. - You can read more about them in the :ref:`data requirements`. + * **roc_auc** - one-vs-the-rest, macro-averaged + * **f1** - macro-averaged + * **precision** - macro-averaged + * **recall** - macro-averaged + * **specificity** - macro-averaged + * **accuracy** +For more information about estimating these metrics, refer to the :ref:`standard-metric-estimation` section. -Just The Code -------------- +We also support the following *complex* metrics for multiclass classification performance estimation: -.. nbimport:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cells: 1 3 4 6 + * **confusion_matrix** -.. admonition:: **Advanced configuration** - :class: hint +For more information about estimating the confusion matrix for multiclass problems, +refer to the :ref:`multiclass-confusion-matrix-estimation` section. - - To learn how :class:`~nannyml.chunk.Chunk` works and to set up custom chunkings check out the :ref:`chunking tutorial ` - - To learn how :class:`~nannyml.thresholds.ConstantThreshold` works and to set up custom threshold check out the :ref:`thresholds tutorial ` +.. toctree:: + :maxdepth: 2 -Walkthrough ------------ - - -For simplicity this guide is based on a synthetic dataset where the monitored model predicts -which type of credit card product new customers should be assigned to. -Check out :ref:`Credit Card Dataset` to learn more about this dataset. - -In order to monitor a model, NannyML needs to learn about it and set expectations from a reference dataset. -Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. -You can read more about this in our section on :ref:`data periods`. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cells: 1 - -.. nbtable:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cell: 2 - -Next we create the Confidence-based Performance Estimation (:class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE`) -estimator with a list of metrics, and an optional :term:`chunking` specification. For more information about -chunking check out the :ref:`chunking tutorial` and it's :ref:`advanced guide`. - -.. note:: - The list of metrics specifies which performance metrics of the monitored model will be estimated. - The following metrics are currently supported: - - - ``roc_auc`` - one-vs-the-rest, macro-averaged - - ``f1`` - macro-averaged - - ``precision`` - macro-averaged - - ``recall`` - macro-averaged - - ``specificity`` - macro-averaged - - ``accuracy`` - - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cells: 3 - -The :class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE` -estimator is then fitted using the -:meth:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE.fit` method on the reference data. - -The fitted ``estimator`` can be used to estimate performance on other data, for which performance cannot be calculated. -Typically, this would be used on the latest production data where target is missing. In our example this is -the ``analysis_df`` data. - -NannyML can then output a dataframe that contains all the results. Let's have a look at the results for analysis period -only. - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cells: 4 - -.. nbtable:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cell: 5 - -Apart from chunk-related data, the results data have the following columns for each metric -that was estimated: - - - **value** - the estimate of a metric for a specific chunk. - - **sampling_error** - the estimate of the :term:`Sampling Error`. - - **realized** - when **target** values are available for a chunk, the realized performance metric will also - be calculated and included within the results. - - **upper_confidence_boundary** and **lower_confidence_boundary** - These values show the :term:`Confidence Band` of the relevant metric - and are equal to estimated value +/- 3 times the estimated :term:`Sampling Error`. - - **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant - performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in - the reference partition. By default, the thresholds are 3 standard deviations away from the mean performance calculated on - chunks. They are calculated during ``fit`` phase. You can also set up custom thresholds using constant or standard deviations thresholds, - to learn more about it check out our :ref:`tutorial on thresholds`. - - **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses - upper or lower threshold. - - -These results can be also plotted. Our plot contains several key elements. - -* The purple dashed step plot shows the estimated performance in each chunk of the analysis period. Thick squared point - markers indicate the middle of these chunks. - -* The black vertical line splits the reference and analysis periods. - -* The low-saturated colored area around the estimated performance indicates the :ref:`sampling error`. - -* The red horizontal dashed lines show upper and lower thresholds for alerting purposes. - -* If the estimated performance crosses the upper or lower threshold an alert is raised which is indicated with a red - diamond-shaped point marker in the middle of the chunk. - -Description of tabular results above explains how the -:term:`confidence bands` and thresholds are calculated. Additional information is shown in the hover (these are -interactive plots, though only static views are included here). - - -.. nbimport:: - :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb - :cells: 6 - -.. image:: ../../_static/tutorials/performance_estimation/multiclass_synthetic.svg - -Insights --------- - -After reviewing the performance estimation results, we should be able to see any indications of performance change that -NannyML has detected based upon the model's inputs and outputs alone. - - -What's next ------------ - -The :ref:`Data Drift` functionality can help us to understand whether data drift is causing the performance problem. -When the target values become available we can -:ref:`compared realized and performance results`. + multiclass_performance_estimation/standard_metric_estimation + multiclass_performance_estimation/confusion_matrix_estimation diff --git a/docs/tutorials/performance_estimation/multiclass_performance_estimation/confusion_matrix_estimation.rst b/docs/tutorials/performance_estimation/multiclass_performance_estimation/confusion_matrix_estimation.rst new file mode 100644 index 000000000..571bb1672 --- /dev/null +++ b/docs/tutorials/performance_estimation/multiclass_performance_estimation/confusion_matrix_estimation.rst @@ -0,0 +1,168 @@ +.. _multiclass-confusion-matrix-estimation: + +================================================================== +Estimating Confusion Matrix Elements for Multiclass Classification +================================================================== + +This tutorial explains how to use NannyML to estimate the :term:`confusion matrix` for multiclass classification +models in the absence of target data. To find out how CBPE estimates performance, read the :ref:`explanation of Confidence-based +Performance Estimation`. + +.. note:: + The following example uses :term:`timestamps`. + These are optional but have an impact on the way data is chunked and results are plotted. + You can read more about them in the :ref:`data requirements`. + +.. _confusion-matrix-estimation-multiclass-just-the-code: + +Just The Code +---------------- + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 1 3 4 5 7 + + +Walkthrough +-------------- + +For simplicity this guide is based on a synthetic dataset where the monitored model predicts +which type of credit card product new customers should be assigned to. +Check out :ref:`Credit Card Dataset` to learn more about this dataset. + +In order to monitor a model, NannyML needs to learn about it from a reference dataset. Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. +You can read more about this in our section on :ref:`data periods`. + +We start by loading the dataset we'll be using: + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 1 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cell: 2 + +Next we create the Confidence-based Performance Estimation +(:class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE`) +estimator. To initialize an estimator that estimates the **confusion_matrix**, we specify the following +parameters: + + - **y_pred_proba:** a dictionary that maps the class names to the + name of the column in the reference data that contains the + predicted probabilities for that class. + - **y_pred:** the name of the column in the reference data that + contains the predicted classes. + - **y_true:** the name of the column in the reference data that + contains the true classes. + - **timestamp_column_name (Optional):** the name of the column in the reference data that + contains timestamps. + - **metrics:** a list of metrics to estimate. In this example we + will estimate the ``confusion_matrix`` metric. + - **chunk_size (Optional):** the number of observations in each chunk of data + used to estimate performance. For more information about + :term:`chunking` configurations check out the :ref:`chunking tutorial`. + - **problem_type:** the type of problem being monitored. In this example we + will monitor a multiclass classification problem. + - **normalize_confusion_matrix (Optional):** how to normalize the confusion matrix. + The normalization options are: + + * **None** : returns counts for each cell + * **"true"** : normalize over the true class of observations. + * **"pred"** : normalize over the predicted class of observations + * **"all"** : normalize over all observations + + - **thresholds (Optional):** the thresholds used to calculate the alert flag. For more information about + thresholds, check out the :ref:`thresholds tutorial`. + +.. note:: + Since we are estimating the confusion matrix, the count values + in each cell of the confusion matrix are estimates. We normalize the + estimates just as if they were true counts. This means that when we + normalize over the true class, the estimates in each row will sum to 1. + When we normalize over the predicted class, the estimates in each + column will sum to 1. When we normalize over all observations, the + estimates in the entire matrix will sum to 1. + + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 3 + +The :class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE` +estimator is then fitted using the +:meth:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE.fit` method on the ``reference`` data. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 4 + +The fitted ``estimator`` can be used to estimate performance on other data, for which performance cannot be calculated. +Typically, this would be used on the latest production data where target is missing. In our example this is +the ``analysis_df`` data. + +NannyML can then output a dataframe that contains all the results. Let's have a look at the results for analysis period +only. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 5 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cell: 6 + +Apart from chunk-related data, the results data have the following columns for each metric +that was estimated: + + - **value** - the estimate of a metric for a specific chunk. + - **sampling_error** - the estimate of the :term:`Sampling Error`. + - **realized** - when **target** values are available for a chunk, the realized performance metric will also + be calculated and included within the results. + - **upper_confidence_boundary** and **lower_confidence_boundary** - These values show the :term:`confidence band` of the relevant metric + and are equal to estimated value +/- 3 times the estimated :term:`sampling error`. + - **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant + performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in + the **reference** partition. The thresholds are 3 standard deviations away from the mean performance calculated on + chunks. + The thresholds are calculated during **fit** phase. + - **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses + upper or lower threshold. + +These results can be also plotted. Our plot contains several key elements. + +* *The purple step plot* shows the estimated performance in each chunk of the analysis period. Thick squared point + markers indicate the middle of these chunks. + +* *The low-saturated purple area* around the estimated performance in the analysis period corresponds to the :term:`confidence band` which is + calculated as the estimated performance +/- 3 times the estimated :term:`Sampling Error`. + +* *The gray vertical line* splits the reference and analysis periods. + +* *The red horizontal dashed lines* show upper and lower thresholds for alerting purposes. + +* *The red diamond-shaped point markers* in the middle of a chunk indicate that an alert has been raised. Alerts are caused by the estimated performance crossing the upper or lower threshold. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Confusion Matrix - Multiclass Classification.ipynb + :cells: 7 + +.. image:: ../../../_static/tutorials/performance_estimation/multiclass/tutorial-confusion-matrix-estimation-multiclass-analysis-with-ref.svg + +Additional information such as the chunk index range and chunk date range (if timestamps were provided) is shown in the hover for each chunk (these are +interactive plots, though only static views are included here). + +Insights +-------- + +After reviewing the performance estimation results, we should be able to see any indications of performance change that +NannyML has detected based upon the model's inputs and outputs alone. + + +What's next +----------- + +The :ref:`Data Drift` functionality can help us to understand whether data drift is causing the performance problem. +When the target values become available we can use +:ref:`realized performance calculation` to +:ref:`compare realized and estimated confusion matrix results`. diff --git a/docs/tutorials/performance_estimation/multiclass_performance_estimation/standard_metric_estimation.rst b/docs/tutorials/performance_estimation/multiclass_performance_estimation/standard_metric_estimation.rst new file mode 100644 index 000000000..75ae505fd --- /dev/null +++ b/docs/tutorials/performance_estimation/multiclass_performance_estimation/standard_metric_estimation.rst @@ -0,0 +1,144 @@ +.. _multiclass_standard-metric-estimation: + +==================================================== +Estimating Performance for Multiclass Classification +==================================================== + +This tutorial explains how to use NannyML to estimate the performance of binary classification +models in the absence of target data. To find out how :class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE` estimates performance, read the :ref:`explanation of Confidence-based +Performance Estimation`. + +.. note:: + The following example uses :term:`timestamps`. + These are optional but have an impact on the way data is chunked and results are plotted. + You can read more about them in the :ref:`data requirements`. + + +Just The Code +------------- + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cells: 1 3 4 6 + +.. admonition:: **Advanced configuration** + :class: hint + + - To learn how :class:`~nannyml.chunk.Chunk` works and to set up custom chunkings check out the :ref:`chunking tutorial ` + - To learn how :class:`~nannyml.thresholds.ConstantThreshold` works and to set up custom threshold check out the :ref:`thresholds tutorial ` + +Walkthrough +----------- + + +For simplicity this guide is based on a synthetic dataset where the monitored model predicts +which type of credit card product new customers should be assigned to. +Check out :ref:`Credit Card Dataset` to learn more about this dataset. + +In order to monitor a model, NannyML needs to learn about it and set expectations from a reference dataset. +Then it can monitor the data that is subject to actual analysis, provided as the analysis dataset. +You can read more about this in our section on :ref:`data periods`. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cells: 1 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cell: 2 + +Next we create the Confidence-based Performance Estimation (:class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE`) +estimator with a list of metrics, and an optional :term:`chunking` specification. For more information about +chunking check out the :ref:`chunking tutorial` and it's :ref:`advanced guide`. + +.. note:: + The list of metrics specifies which performance metrics of the monitored model will be estimated. + The following metrics are currently supported: + + - ``roc_auc`` - one-vs-the-rest, macro-averaged + - ``f1`` - macro-averaged + - ``precision`` - macro-averaged + - ``recall`` - macro-averaged + - ``specificity`` - macro-averaged + - ``accuracy`` + + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cells: 3 + +The :class:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE` +estimator is then fitted using the +:meth:`~nannyml.performance_estimation.confidence_based.cbpe.CBPE.fit` method on the reference data. + +The fitted ``estimator`` can be used to estimate performance on other data, for which performance cannot be calculated. +Typically, this would be used on the latest production data where target is missing. In our example this is +the ``analysis_df`` data. + +NannyML can then output a dataframe that contains all the results. Let's have a look at the results for analysis period +only. + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cells: 4 + +.. nbtable:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cell: 5 + +Apart from chunk-related data, the results data have the following columns for each metric +that was estimated: + + - **value** - the estimate of a metric for a specific chunk. + - **sampling_error** - the estimate of the :term:`Sampling Error`. + - **realized** - when **target** values are available for a chunk, the realized performance metric will also + be calculated and included within the results. + - **upper_confidence_boundary** and **lower_confidence_boundary** - These values show the :term:`Confidence Band` of the relevant metric + and are equal to estimated value +/- 3 times the estimated :term:`Sampling Error`. + - **upper_threshold** and **lower_threshold** - crossing these thresholds will raise an alert on significant + performance change. The thresholds are calculated based on the actual performance of the monitored model on chunks in + the reference partition. By default, the thresholds are 3 standard deviations away from the mean performance calculated on + chunks. They are calculated during ``fit`` phase. You can also set up custom thresholds using constant or standard deviations thresholds, + to learn more about it check out our :ref:`tutorial on thresholds`. + - **alert** - flag indicating potentially significant performance change. ``True`` if estimated performance crosses + upper or lower threshold. + + +These results can be also plotted. Our plot contains several key elements. + +* The purple dashed step plot shows the estimated performance in each chunk of the analysis period. Thick squared point + markers indicate the middle of these chunks. + +* The black vertical line splits the reference and analysis periods. + +* The low-saturated colored area around the estimated performance indicates the :ref:`sampling error`. + +* The red horizontal dashed lines show upper and lower thresholds for alerting purposes. + +* If the estimated performance crosses the upper or lower threshold an alert is raised which is indicated with a red + diamond-shaped point marker in the middle of the chunk. + +Description of tabular results above explains how the +:term:`confidence bands` and thresholds are calculated. Additional information is shown in the hover (these are +interactive plots, though only static views are included here). + + +.. nbimport:: + :path: ./example_notebooks/Tutorial - Estimating Performance - Multiclass Classification.ipynb + :cells: 6 + +.. image:: ../../../_static/tutorials/performance_estimation/multiclass_synthetic.svg + +Insights +-------- + +After reviewing the performance estimation results, we should be able to see any indications of performance change that +NannyML has detected based upon the model's inputs and outputs alone. + + +What's next +----------- + +The :ref:`Data Drift` functionality can help us to understand whether data drift is causing the performance problem. +When the target values become available we can +:ref:`compared realized and performance results`. diff --git a/nannyml/performance_calculation/metrics/binary_classification.py b/nannyml/performance_calculation/metrics/binary_classification.py index 2b4d97bc1..44b04aa60 100644 --- a/nannyml/performance_calculation/metrics/binary_classification.py +++ b/nannyml/performance_calculation/metrics/binary_classification.py @@ -1,12 +1,12 @@ # Author: Niels Nuyttens # # License: Apache Software License 2.0 +import warnings from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import pandas as pd from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score -import warnings from nannyml._typing import ProblemType from nannyml.base import _list_missing diff --git a/nannyml/performance_calculation/metrics/multiclass_classification.py b/nannyml/performance_calculation/metrics/multiclass_classification.py index 63510120c..33ba37849 100644 --- a/nannyml/performance_calculation/metrics/multiclass_classification.py +++ b/nannyml/performance_calculation/metrics/multiclass_classification.py @@ -7,13 +7,14 @@ # License: Apache Software License 2.0 """Module containing metric utilities and implementations.""" +import warnings from typing import Dict, List, Optional, Tuple, Union # noqa: TYP001 import numpy as np import pandas as pd -import warnings from sklearn.metrics import ( accuracy_score, + confusion_matrix, f1_score, multilabel_confusion_matrix, precision_score, @@ -24,6 +25,7 @@ from nannyml._typing import ProblemType, class_labels, model_output_column_names from nannyml.base import _list_missing +from nannyml.chunk import Chunker from nannyml.exceptions import InvalidArgumentsException from nannyml.performance_calculation.metrics.base import Metric, MetricFactory, _common_data_cleaning from nannyml.sampling_error.multiclass_classification import ( @@ -39,8 +41,10 @@ recall_sampling_error_components, specificity_sampling_error, specificity_sampling_error_components, + multiclass_confusion_matrix_sampling_error, + multiclass_confusion_matrix_sampling_error_components, ) -from nannyml.thresholds import Threshold +from nannyml.thresholds import Threshold, calculate_threshold_values @MetricFactory.register(metric='roc_auc', use_case=ProblemType.CLASSIFICATION_MULTICLASS) @@ -571,3 +575,162 @@ def _calculate(self, data: pd.DataFrame): def _sampling_error(self, data: pd.DataFrame) -> float: return accuracy_sampling_error(self._sampling_error_components, data) + + +@MetricFactory.register('confusion_matrix', ProblemType.CLASSIFICATION_MULTICLASS) +class MulticlassClassificationConfusionMatrix(Metric): + def __init__( + self, + y_true: str, + y_pred: str, + threshold: Threshold, + y_pred_proba: Optional[Union[str, Dict[str, str]]] = None, + normalize_confusion_matrix: Optional[str] = None, + **kwargs, + ): + + """Creates a new confusion matrix instance.""" + super().__init__( + name='confusion_matrix', + y_true=y_true, + y_pred=y_pred, + threshold=threshold, + y_pred_proba=y_pred_proba, + components=[("None", "none")], + ) + + self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix + + self.classes: Optional[List[str]] = None + + def __str__(self): + return "confusion_matrix" + + def fit(self, reference_data: pd.DataFrame, chunker: Chunker): + + # _fit + # realized perf on chunks + # set thresholds + + self._fit(reference_data) + + reference_chunks = chunker.split(reference_data) + reference_chunk_results = np.asarray([self._calculate(chunk.data) for chunk in reference_chunks]) + + self.alert_thresholds = self._multiclass_confusion_matrix_alert_thresholds( + reference_chunk_results=reference_chunk_results, + ) + + def _multiclass_confusion_matrix_alert_thresholds( + self, + reference_chunk_results: np.ndarray, + ) -> Dict[str, Tuple[Optional[float], Optional[float]]]: + """Calculate the alert thresholds for the confusion matrix. + + Args: + reference_chunk_results: The confusion matrix for each chunk of the reference data. + + Returns: + The alert thresholds for the confusion matrix. + """ + alert_thresholds = {} + + if self.classes is None: + raise ValueError("classes must be set before calling this method") + + num_classes = len(self.classes) + + for i in range(num_classes): + for j in range(num_classes): + lower_threshold_value, upper_threshold_value = calculate_threshold_values( + threshold=self.threshold, + data=reference_chunk_results[:, i, j], + lower_threshold_value_limit=self.lower_threshold_value_limit, + upper_threshold_value_limit=self.upper_threshold_value_limit, + ) + alert_thresholds[f'true_{self.classes[i]}_pred_{self.classes[j]}'] = ( + lower_threshold_value, + upper_threshold_value, + ) + + return alert_thresholds + + def _fit(self, reference_data: pd.DataFrame): + _list_missing([self.y_true, self.y_pred], reference_data) + + self.sampling_error_components = multiclass_confusion_matrix_sampling_error_components( + y_true_reference=reference_data[self.y_true], + y_pred_reference=reference_data[self.y_pred], + normalize_confusion_matrix=self.normalize_confusion_matrix, + ) + + self.classes = sorted(reference_data[self.y_true].unique()) + + self.components = self._get_components(self.classes) + + def _get_components(self, classes: List[str]) -> List[Tuple[str, str]]: + components = [] + + for true_class in classes: + for pred_class in classes: + components.append( + ( + f"true class: '{true_class}', predicted class: '{pred_class}'", + f'true_{true_class}_pred_{pred_class}', + ) + ) + + return components + + def _calculate(self, data: pd.DataFrame) -> Union[np.ndarray, float]: + _list_missing([self.y_true, self.y_pred], data) + + y_true = data[self.y_true] + y_pred = data[self.y_pred] + + if y_pred.isna().all().any(): + raise InvalidArgumentsException( + f"could not calculate metric {self.display_name}: prediction column contains no data" + ) + + if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1): + return np.nan + else: + cm = confusion_matrix(y_true, y_pred, labels=self.classes, normalize=self.normalize_confusion_matrix) + return cm + + def get_chunk_record(self, chunk_data: pd.DataFrame) -> Dict[str, Union[float, bool]]: + + if self.classes is None: + raise ValueError("classes must be set before calling this method") + + sampling_errors = multiclass_confusion_matrix_sampling_error(self.sampling_error_components, chunk_data) + realized_cm = self._calculate(chunk_data) + + if isinstance(realized_cm, float): + realized_cm = np.full((len(self.classes), len(self.classes)), np.nan) + + chunk_record = {} + + for true_class in self.classes: + for pred_class in self.classes: + + column_name = f'true_{true_class}_pred_{pred_class}' + + chunk_record[f"{column_name}_sampling_error"] = sampling_errors[ + self.classes.index(true_class), self.classes.index(pred_class) + ] + + chunk_record[f"{column_name}"] = realized_cm[ + self.classes.index(true_class), self.classes.index(pred_class) + ] + + lower_threshold, upper_threshold = self.alert_thresholds[f"true_{true_class}_pred_{pred_class}"] + chunk_record[f"{column_name}_upper_threshold"] = upper_threshold + chunk_record[f"{column_name}_lower_threshold"] = lower_threshold + + chunk_record[f"{column_name}_alert"] = ( + self.alert_thresholds is not None and (chunk_record[f"{column_name}"] < lower_threshold) + ) or (self.alert_thresholds is not None and (chunk_record[f"{column_name}"] > upper_threshold)) + + return chunk_record diff --git a/nannyml/performance_estimation/confidence_based/metrics.py b/nannyml/performance_estimation/confidence_based/metrics.py index 4f9620ac8..a7316f9e8 100644 --- a/nannyml/performance_estimation/confidence_based/metrics.py +++ b/nannyml/performance_estimation/confidence_based/metrics.py @@ -11,6 +11,7 @@ import abc import logging +import warnings from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union import numpy as np @@ -402,6 +403,11 @@ def _realized_performance(self, data: pd.DataFrame) -> float: y_pred_proba, _, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized ROC-AUC.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized ROC-AUC.") return np.NaN return roc_auc_score(y_true, y_pred_proba) @@ -494,6 +500,15 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized F1 score.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized F1 score.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized F1 score.") return np.NaN return f1_score(y_true=y_true, y_pred=y_pred) @@ -570,6 +585,15 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized precision.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized precision.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized precision.") return np.NaN return precision_score(y_true=y_true, y_pred=y_pred) @@ -644,6 +668,15 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized recall.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as recall precision.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as recall precision.") return np.NaN return recall_score(y_true=y_true, y_pred=y_pred) @@ -718,10 +751,19 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized specificity.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized specificity.") return np.NaN - conf_matrix = confusion_matrix(y_true=y_true, y_pred=y_pred) - return conf_matrix[1, 1] / (conf_matrix[1, 0] + conf_matrix[1, 1]) + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized specificity.") + return np.NaN + + tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel() + return tn / (tn + fp) def estimate_specificity(y_pred: pd.DataFrame, y_pred_proba: pd.DataFrame) -> float: @@ -797,6 +839,15 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized accuracy.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized accuracy.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized accuracy.") return np.NaN return accuracy_score(y_true=y_true, y_pred=y_pred) @@ -961,6 +1012,15 @@ def _true_positive_realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized confusion matrix.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized confusion matrix.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized confusion matrix.") return np.NaN num_tp = np.sum(np.logical_and(y_pred, y_true)) @@ -980,6 +1040,7 @@ def _true_negative_realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized confusion matrix.") return np.NaN num_tn = np.sum(np.logical_and(np.logical_not(y_pred), np.logical_not(y_true))) @@ -999,6 +1060,15 @@ def _false_positive_realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized confusion matrix.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized confusion matrix.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized confusion matrix.") return np.NaN num_tp = np.sum(np.logical_and(y_pred, y_true)) @@ -1018,6 +1088,15 @@ def _false_negative_realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized confusion matrix.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized confusion matrix.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized confusion matrix.") return np.NaN num_tp = np.sum(np.logical_and(y_pred, y_true)) @@ -1500,6 +1579,15 @@ def _realized_performance(self, data: pd.DataFrame) -> float: _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba) if y_true is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized business value.") + return np.NaN + + if y_true.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized business value.") + return np.NaN + + if y_pred.nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized business value.") return np.NaN tp_value = self.business_value_matrix[1, 1] @@ -1677,7 +1765,13 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized ROC-AUC.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized ROC-AUC.") return np.NaN _, y_pred_probas, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) @@ -1734,7 +1828,17 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized F1 score.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized F1 score.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized F1 score.") return np.NaN y_pred, _, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) @@ -1791,7 +1895,17 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized precision.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized precision.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized precision.") return np.NaN y_pred, _, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) @@ -1848,7 +1962,17 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized recall.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized recall.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized recall.") return np.NaN y_pred, _, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) @@ -1905,7 +2029,17 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized specificity.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized specificity.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized specificity.") return np.NaN y_pred, _, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) @@ -1964,9 +2098,300 @@ def _sampling_error(self, data: pd.DataFrame) -> float: def _realized_performance(self, data: pd.DataFrame) -> float: data = self._ensure_targets(data) + if data is None: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized accuracy.") return np.NaN - y_pred, _, labels = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized accuracy.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized accuracy.") + return np.NaN + y_pred, _, _ = _get_multiclass_uncalibrated_predictions(data, self.y_pred, self.y_pred_proba) return accuracy_score(data[self.y_true], y_pred) + + +@MetricFactory.register('confusion_matrix', ProblemType.CLASSIFICATION_MULTICLASS) +class MulticlassClassificationConfusionMatrix(Metric): + def __init__( + self, + y_pred_proba: ModelOutputsType, + y_pred: str, + y_true: str, + chunker: Chunker, + threshold: Threshold, + timestamp_column_name: Optional[str] = None, + normalize_confusion_matrix: Optional[str] = None, + **kwargs, + ): + + if isinstance(y_pred_proba, str): + raise ValueError( + "y_pred_proba must be a dictionary with class labels as keys and pred_proba column names as values" + ) + + self.classes = list(y_pred_proba.keys()) + + super().__init__( + name='confusion_matrix', + y_pred_proba=y_pred_proba, + y_pred=y_pred, + y_true=y_true, + timestamp_column_name=timestamp_column_name, + chunker=chunker, + threshold=threshold, + components=self._get_components(self.classes), + lower_threshold_value_limit=0, + ) + + self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix + + def _get_components(self, classes: List[str]) -> List[Tuple[str, str]]: + components = [] + + for true_class in classes: + for pred_class in classes: + components.append( + ( + f"true class: '{true_class}', predicted class: '{pred_class}'", + f'true_{true_class}_pred_{pred_class}', + ) + ) + + return components + + def fit(self, reference_data: pd.DataFrame): # override the superclass fit method + """Fits a Metric on reference data. + Parameters + ---------- + reference_data: pd.DataFrame + The reference data used for fitting. Must have target data available. + """ + # Calculate alert thresholds + reference_chunks = self.chunker.split( + reference_data, + ) + + self.alert_thresholds = self._multiclass_confusion_matrix_alert_thresholds(reference_chunks) + + # Calculate confidence bands + self.confidence_deviations = self._multiclass_confusion_matrix_confidence_deviations(reference_chunks) + + # Delegate to confusion matrix subclass + self._fit(reference_data) # could probably put _fit functionality here since overide fit method + + return + + def _fit(self, reference_data: pd.DataFrame): + + self._confusion_matrix_sampling_error_components = mse.multiclass_confusion_matrix_sampling_error_components( + y_true_reference=reference_data[self.y_true], + y_pred_reference=reference_data[self.y_pred], + normalize_confusion_matrix=self.normalize_confusion_matrix, + ) + + def _multiclass_confusion_matrix_alert_thresholds( + self, reference_chunks: List[Chunk] + ) -> Dict[str, Tuple[Optional[float], Optional[float]]]: + + realized_chunk_performance = np.asarray( + [self._multi_class_confusion_matrix_realized_performance(chunk.data) for chunk in reference_chunks] + ) + + alert_thresholds = {} + + num_classes = len(self.classes) + + for i in range(num_classes): + for j in range(num_classes): + lower_threshold_value, upper_threshold_value = calculate_threshold_values( + threshold=self.threshold, + data=realized_chunk_performance[:, i, j], + lower_threshold_value_limit=self.lower_threshold_value_limit, + upper_threshold_value_limit=self.upper_threshold_value_limit, + ) + alert_thresholds[f'true_{self.classes[i]}_pred_{self.classes[j]}'] = ( + lower_threshold_value, + upper_threshold_value, + ) + + return alert_thresholds + + def _multi_class_confusion_matrix_realized_performance(self, data: pd.DataFrame) -> Union[np.ndarray, float]: + if data is None or self.y_true not in data.columns: + warnings.warn("No 'y_true' values given for chunk, returning NaN as realized precision.") + return np.NaN + + if data[self.y_true].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_true', returning NaN as realized precision.") + return np.NaN + + if data[self.y_pred].nunique() <= 1: + warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized precision.") + return np.NaN + + cm = confusion_matrix( + data[self.y_true], data[self.y_pred], labels=self.classes, normalize=self.normalize_confusion_matrix + ) + + return cm + + def _multiclass_confusion_matrix_confidence_deviations( + self, + reference_chunks: List[Chunk], + ) -> Dict[str, float]: + + confidence_deviations = {} + + num_classes = len(self.classes) + + for i in range(num_classes): + for j in range(num_classes): + + confidence_deviations[f'true_{self.classes[i]}_pred_{self.classes[j]}'] = np.std( + [self._get_multiclass_confusion_matrix_estimate(chunk.data)[i, j] for chunk in reference_chunks] + ) + + return confidence_deviations + + def _get_multiclass_confusion_matrix_estimate(self, chunk_data: pd.DataFrame) -> np.ndarray: + + if isinstance(self.y_pred_proba, str): + raise ValueError( + "y_pred_proba must be a dictionary with class labels as keys and pred_proba column names as values" + ) + + y_pred_proba = {key: chunk_data[value] for key, value in self.y_pred_proba.items()} + + y_pred = chunk_data[self.y_pred] + + num_classes = len(self.classes) + + est_confusion_matrix = np.zeros((num_classes, num_classes)) + + for i in range(num_classes): + for j in range(num_classes): + est_confusion_matrix[i, j] = np.mean( + np.where( + (y_pred == self.classes[j]), + y_pred_proba[self.classes[i]], + 0, + ) + ) + + if self.normalize_confusion_matrix is None: + normalized_est_confusion_matrix = est_confusion_matrix * len(y_pred) + + elif self.normalize_confusion_matrix == 'true': + normalized_est_confusion_matrix = est_confusion_matrix / np.sum(est_confusion_matrix, axis=1)[:, None] + + elif self.normalize_confusion_matrix == 'pred': + normalized_est_confusion_matrix = est_confusion_matrix / np.sum(est_confusion_matrix, axis=0)[None, :] + elif self.normalize_confusion_matrix == 'all': + normalized_est_confusion_matrix = est_confusion_matrix / np.sum(est_confusion_matrix) + else: + raise ValueError( + f'normalize_confusion_matrix should be one of None, "true", \ + "pred", or "all", but got {self.normalize_confusion_matrix}' + ) + + return normalized_est_confusion_matrix + + def get_chunk_record(self, chunk_data: pd.DataFrame) -> Dict: + + chunk_record = {} + + estimated_cm = self._get_multiclass_confusion_matrix_estimate(chunk_data) + realized_cm = self._multi_class_confusion_matrix_realized_performance(chunk_data) + + sampling_error = mse.multiclass_confusion_matrix_sampling_error( + self._confusion_matrix_sampling_error_components, + chunk_data, + ) + + for true_class in self.classes: + for pred_class in self.classes: + + chunk_record[f'estimated_true_{true_class}_pred_{pred_class}'] = estimated_cm[ + self.classes.index(true_class), self.classes.index(pred_class) + ] + + chunk_record[f'sampling_error_true_{true_class}_pred_{pred_class}'] = sampling_error[ + self.classes.index(true_class), self.classes.index(pred_class) + ] + + # check if realized_cm is nan + if isinstance(realized_cm, np.ndarray): + chunk_record[f'realized_true_{true_class}_pred_{pred_class}'] = realized_cm[ + self.classes.index(true_class), self.classes.index(pred_class) + ] + else: + chunk_record[f'realized_true_{true_class}_pred_{pred_class}'] = realized_cm + + upper_confidence_boundary = ( + estimated_cm[self.classes.index(true_class), self.classes.index(pred_class)] + + SAMPLING_ERROR_RANGE + * sampling_error[self.classes.index(true_class), self.classes.index(pred_class)] + ) + + if self.normalize_confusion_matrix is None: + chunk_record[ + f'upper_confidence_boundary_true_{true_class}_pred_{pred_class}' + ] = upper_confidence_boundary + else: + chunk_record[f'upper_confidence_boundary_true_{true_class}_pred_{pred_class}'] = min( + self.confidence_upper_bound, upper_confidence_boundary + ) + + lower_confidence_boundary = ( + estimated_cm[self.classes.index(true_class), self.classes.index(pred_class)] + - SAMPLING_ERROR_RANGE + * sampling_error[self.classes.index(true_class), self.classes.index(pred_class)] + ) + + if self.normalize_confusion_matrix is None: + chunk_record[ + f'lower_confidence_boundary_true_{true_class}_pred_{pred_class}' + ] = lower_confidence_boundary + else: + chunk_record[f'lower_confidence_boundary_true_{true_class}_pred_{pred_class}'] = max( + self.confidence_lower_bound, lower_confidence_boundary + ) + + chunk_record[f'upper_threshold_true_{true_class}_pred_{pred_class}'] = self.alert_thresholds[ + f'true_{true_class}_pred_{pred_class}' + ][1] + chunk_record[f'lower_threshold_true_{true_class}_pred_{pred_class}'] = self.alert_thresholds[ + f'true_{true_class}_pred_{pred_class}' + ][0] + + # do alerts + chunk_record[f'alert_true_{true_class}_pred_{pred_class}'] = ( + self.alert_thresholds is not None + and ( + estimated_cm[self.classes.index(true_class), self.classes.index(pred_class)] + > self.alert_thresholds[f'true_{true_class}_pred_{pred_class}'][1] + ) + or ( + self.alert_thresholds is not None + and ( + estimated_cm[self.classes.index(true_class), self.classes.index(pred_class)] + < self.alert_thresholds[f'true_{true_class}_pred_{pred_class}'][0] + ) + ) + ) + + return chunk_record + + def _estimate(self, data: pd.DataFrame): + pass + + def _sampling_error(self, data: pd.DataFrame) -> float: + return 0.0 + + def _realized_performance(self, data: pd.DataFrame) -> float: + return 0.0 diff --git a/nannyml/sampling_error/multiclass_classification.py b/nannyml/sampling_error/multiclass_classification.py index 3ca9edd43..318cbf10f 100644 --- a/nannyml/sampling_error/multiclass_classification.py +++ b/nannyml/sampling_error/multiclass_classification.py @@ -2,10 +2,11 @@ # Jakub Bialek # # License: Apache Software License 2.0 -from typing import List, Tuple +from typing import List, Tuple, Union import numpy as np import pandas as pd +from sklearn.metrics import confusion_matrix def _standard_deviation_of_variances(components: List[Tuple], data) -> float: @@ -345,3 +346,59 @@ def accuracy_sampling_error(sampling_error_components: Tuple, data) -> float: """ return sampling_error_components[0] / np.sqrt(len(data)) + + +def multiclass_confusion_matrix_sampling_error_components( + y_true_reference: List[pd.Series], y_pred_reference: List[pd.Series], normalize_confusion_matrix: Union[str, None] +): + + cm = confusion_matrix(y_true_reference, y_pred_reference) + + true_marginal = cm.sum(axis=1)[:, None] + pred_marginal = cm.sum(axis=0)[None, :] + + num_observations = len(y_true_reference) + + if normalize_confusion_matrix == 'true': + relevant_proportions = true_marginal / num_observations + elif normalize_confusion_matrix == 'pred': + relevant_proportions = pred_marginal / num_observations + elif normalize_confusion_matrix == 'all': + relevant_proportions = 1 + else: + relevant_proportions = None + + n_rows, n_cols = cm.shape + + stds = np.zeros((n_rows, n_cols)) + + for i in range(n_rows): + for j in range(n_cols): + + if normalize_confusion_matrix == 'true': + obs_level_array = np.zeros(true_marginal[i, 0], dtype=int) + elif normalize_confusion_matrix == 'pred': + obs_level_array = np.zeros(pred_marginal[0, j], dtype=int) + elif normalize_confusion_matrix == 'all': + obs_level_array = np.zeros(num_observations, dtype=int) + else: + obs_level_array = np.zeros(num_observations, dtype=int) + + end_index = cm[i, j] + obs_level_array[:end_index] = 1 + + stds[i, j] = np.std(obs_level_array) + + return stds, relevant_proportions + + +def multiclass_confusion_matrix_sampling_error(sampling_error_components: Tuple, data): + + reference_stds, relevant_proportions = sampling_error_components + + if relevant_proportions is None: + standard_errors = (reference_stds / np.sqrt(len(data))) * len(data) + else: + standard_errors = reference_stds / np.sqrt(len(data) * relevant_proportions) + + return standard_errors diff --git a/nannyml/thresholds.py b/nannyml/thresholds.py index b39e7bff6..e27c27869 100644 --- a/nannyml/thresholds.py +++ b/nannyml/thresholds.py @@ -153,7 +153,7 @@ def __init__( self, std_lower_multiplier: Optional[Union[float, int]] = 3, std_upper_multiplier: Optional[Union[float, int]] = 3, - offset_from: Callable[[np.ndarray], Any] = np.mean, + offset_from: Callable[[np.ndarray], Any] = np.nanmean, ): """Creates a new StandardDeviationThreshold instance. @@ -166,7 +166,7 @@ def __init__( The number the standard deviation of the input array will be multiplied with to form the upper offset. This value will be added to the aggregate of the input array. Defaults to 3. - offset_from: Callable[[np.ndarray], Any], default=np.mean + offset_from: Callable[[np.ndarray], Any], default=np.nanmean A function that will be applied to the input array to aggregate it into a single value. Adding the upper offset to this value will yield the upper threshold, subtracting the lower offset will yield the lower threshold. @@ -180,7 +180,7 @@ def __init__( def thresholds(self, data: np.ndarray, **kwargs) -> Tuple[Optional[float], Optional[float]]: aggregate = self.offset_from(data) - std = np.std(data) + std = np.nanstd(data) lower_threshold = aggregate - std * self.std_lower_multiplier if self.std_lower_multiplier is not None else None @@ -259,6 +259,7 @@ def calculate_threshold_values( metric_name: Optional[str], default=None When provided the metric name will be included within any log messages for additional clarity. """ + lower_threshold_value, upper_threshold_value = threshold.thresholds(data) if ( diff --git a/nannyml/usage_logging.py b/nannyml/usage_logging.py index d242c387a..f5450fbf4 100644 --- a/nannyml/usage_logging.py +++ b/nannyml/usage_logging.py @@ -141,6 +141,9 @@ def __init__(self, write_key: Optional[str] = None, machine_metadata: Optional[D segment_analytics.max_retries = 1 + segment_analytics.timeout = 3 + segment_analytics.max_retries = 1 + if machine_metadata is not None: self._identify(machine_metadata) @@ -252,16 +255,17 @@ def _is_nannyml_cloud(): def _get_runtime_environment(): - if _is_running_in_aks(): - return 'aks' - if _is_running_in_eks(): - return 'eks' - if _is_running_in_kubernetes(): - return 'kubernetes' + if _is_running_in_notebook(): + return 'notebook' elif _is_running_in_docker(): return 'docker' - elif _is_running_in_notebook(): - return 'notebook' + elif _is_running_in_kubernetes(): + if _is_running_in_aks(): + return 'aks' + elif _is_running_in_eks(): + return 'eks' + else: + return 'kubernetes' else: return 'native' @@ -286,7 +290,7 @@ def _is_running_in_aks(): try: metadata = requests.get( - 'http://169.254.169.254/metadata/instance?api-version=2021-02-01', headers={'Metadata': 'true'} + 'http://169.254.169.254/metadata/instance?api-version=2021-02-01', headers={'Metadata': 'true'}, timeout=5 ) return metadata.status_code == 200 except Exception: @@ -298,7 +302,9 @@ def _is_running_in_eks(): try: token = requests.put( - 'http://169.254.169.254/latest/api/token', headers={'X-aws-ec2-metadata-token-ttl-seconds': 21600} + 'http://169.254.169.254/latest/api/token', + headers={'X-aws-ec2-metadata-token-ttl-seconds': 21600}, + timeout=5, ).raw() metadata = requests.get('http://169.254.169.254/latest/meta-data/', headers={'X-aws-ec2-metadata-token': token}) diff --git a/tests/performance_calculation/metrics/test_multiclass_classification.py b/tests/performance_calculation/metrics/test_multiclass_classification.py index 081a66564..a143a15e1 100644 --- a/tests/performance_calculation/metrics/test_multiclass_classification.py +++ b/tests/performance_calculation/metrics/test_multiclass_classification.py @@ -23,6 +23,7 @@ MulticlassClassificationPrecision, MulticlassClassificationRecall, MulticlassClassificationSpecificity, + MulticlassClassificationConfusionMatrix, ) from nannyml.thresholds import ConstantThreshold, StandardDeviationThreshold @@ -45,7 +46,7 @@ def performance_calculator() -> PerformanceCalculator: }, y_pred='y_pred', y_true='y_true', - metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy'], + metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy', 'confusion_matrix'], problem_type='classification_multiclass', ) @@ -60,7 +61,7 @@ def realized_performance_metrics(multiclass_data) -> pd.DataFrame: }, y_pred='y_pred', y_true='y_true', - metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy'], + metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy', 'confusion_matrix'], problem_type='classification_multiclass', ).fit(multiclass_data[0]) results = performance_calculator.calculate( @@ -87,6 +88,7 @@ def no_timestamp_metrics(performance_calculator, multiclass_data) -> pd.DataFram ('recall', ProblemType.CLASSIFICATION_MULTICLASS, MulticlassClassificationRecall), ('specificity', ProblemType.CLASSIFICATION_MULTICLASS, MulticlassClassificationSpecificity), ('accuracy', ProblemType.CLASSIFICATION_MULTICLASS, MulticlassClassificationAccuracy), + ('confusion_matrix', ProblemType.CLASSIFICATION_MULTICLASS, MulticlassClassificationConfusionMatrix), ], ) def test_metric_factory_returns_correct_metric_given_key_and_problem_type(key, problem_type, metric): # noqa: D103 @@ -120,6 +122,15 @@ def test_metric_factory_returns_correct_metric_given_key_and_problem_type(key, p ('recall', [0.75103, 0.76315, 0.75848, 0.75899, 0.75798, 0.55783, 0.56017, 0.56594, 0.56472, 0.56277]), ('specificity', [0.87555, 0.88151, 0.87937, 0.87963, 0.87899, 0.77991, 0.78068, 0.78422, 0.78342, 0.78243]), ('accuracy', [0.75117, 0.763, 0.75867, 0.75917, 0.758, 0.56083, 0.56233, 0.56983, 0.56783, 0.566]), + ('true_upmarket_card_pred_upmarket_card', [1490, 1524, 1544, 1528, 1567, 1165, 1120, 1147, 1166, 1173]), + ('true_upmarket_card_pred_prepaid_card', [245, 234, 224, 201, 229, 372, 382, 382, 377, 371]), + ('true_upmarket_card_pred_highstreet_card', [245, 252, 251, 262, 242, 525, 508, 535, 513, 513]), + ('true_prepaid_card_pred_upmarket_card', [204, 201, 200, 246, 194, 448, 419, 416, 440, 436]), + ('true_prepaid_card_pred_prepaid_card', [1560, 1518, 1557, 1577, 1493, 878, 908, 875, 888, 869]), + ('true_prepaid_card_pred_highstreet_card', [274, 237, 264, 237, 270, 539, 595, 550, 539, 568]), + ('true_highstreet_card_pred_upmarket_card', [250, 237, 259, 251, 277, 330, 318, 302, 312, 326]), + ('true_highstreet_card_pred_prepaid_card', [275, 261, 250, 248, 240, 421, 404, 396, 412, 390]), + ('true_highstreet_card_pred_highstreet_card', [1457, 1536, 1451, 1450, 1488, 1322, 1346, 1397, 1353, 1354]), ], ) def test_metric_values_are_calculated_correctly(realized_performance_metrics, metric, expected): @@ -136,6 +147,15 @@ def test_metric_values_are_calculated_correctly(realized_performance_metrics, me ('recall', [0.75103, 0.76315, 0.75848, 0.75899, 0.75798, 0.55783, 0.56017, 0.56594, 0.56472, 0.56277]), ('specificity', [0.87555, 0.88151, 0.87937, 0.87963, 0.87899, 0.77991, 0.78068, 0.78422, 0.78342, 0.78243]), ('accuracy', [0.75117, 0.763, 0.75867, 0.75917, 0.758, 0.56083, 0.56233, 0.56983, 0.56783, 0.566]), + ('true_upmarket_card_pred_upmarket_card', [1490, 1524, 1544, 1528, 1567, 1165, 1120, 1147, 1166, 1173]), + ('true_upmarket_card_pred_prepaid_card', [245, 234, 224, 201, 229, 372, 382, 382, 377, 371]), + ('true_upmarket_card_pred_highstreet_card', [245, 252, 251, 262, 242, 525, 508, 535, 513, 513]), + ('true_prepaid_card_pred_upmarket_card', [204, 201, 200, 246, 194, 448, 419, 416, 440, 436]), + ('true_prepaid_card_pred_prepaid_card', [1560, 1518, 1557, 1577, 1493, 878, 908, 875, 888, 869]), + ('true_prepaid_card_pred_highstreet_card', [274, 237, 264, 237, 270, 539, 595, 550, 539, 568]), + ('true_highstreet_card_pred_upmarket_card', [250, 237, 259, 251, 277, 330, 318, 302, 312, 326]), + ('true_highstreet_card_pred_prepaid_card', [275, 261, 250, 248, 240, 421, 404, 396, 412, 390]), + ('true_highstreet_card_pred_highstreet_card', [1457, 1536, 1451, 1450, 1488, 1322, 1346, 1397, 1353, 1354]), ], ) def test_metric_values_without_timestamps_are_calculated_correctly(no_timestamp_metrics, metric, expected): diff --git a/tests/performance_estimation/CBPE/test_cbpe_metrics.py b/tests/performance_estimation/CBPE/test_cbpe_metrics.py index fe82c4309..5c6210ab9 100644 --- a/tests/performance_estimation/CBPE/test_cbpe_metrics.py +++ b/tests/performance_estimation/CBPE/test_cbpe_metrics.py @@ -2453,11 +2453,56 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 'estimated_recall': [0.7564129287764665, 0.6934788458355289, 0.6319310599943714], 'estimated_specificity': [0.8782068281303994, 0.8469556750949159, 0.8172644220189141], 'estimated_accuracy': [0.7564451493123628, 0.6946947603445697, 0.6378557309960986], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 4976.829215997277, + 5148.649186425118, + 5412.348045797111, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 878.1877379091701, + 1038.3533241561252, + 1250.9260097761653, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 831.7702766018707, + 993.7691398029524, + 1109.9706655490413, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 806.1451187447954, + 1140.1932616586546, + 1451.431964364007, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 5180.838942632071, + 4134.524656135082, + 3326.8467648553315, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 755.9948957802203, + 998.509495865855, + 1200.1095251814281, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 812.0256652579275, + 1062.1575519162266, + 1263.219989838882, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 786.9733194587595, + 873.1220197087925, + 967.2272253685034, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 4971.234827617909, + 4610.7213643311925, + 4017.9198092695306, + ], } ), ), ( - {'chunk_size': 20000, 'timestamp_column_name': 'timestamp'}, + {'chunk_size': 20000, 'timestamp_column_name': 'timestamp', 'normalize_confusion_matrix': 'true'}, pd.DataFrame( { 'key': ['[0:19999]', '[20000:39999]', '[40000:59999]'], @@ -2467,11 +2512,56 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 'estimated_recall': [0.7564129287764665, 0.6934788458355289, 0.6319310599943714], 'estimated_specificity': [0.8782068281303994, 0.8469556750949159, 0.8172644220189141], 'estimated_accuracy': [0.7564451493123628, 0.6946947603445697, 0.6378557309960986], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 0.7442780881812128, + 0.7170050012869645, + 0.6962791266676683, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 0.1313317902358936, + 0.14460191393226796, + 0.16092713592008898, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 0.12439012158289371, + 0.1383930847807676, + 0.1427937374122426, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 0.11955326034187638, + 0.18175544842770236, + 0.24277980997563847, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 0.7683308780213619, + 0.6590745693568182, + 0.5564788741190233, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 0.1121158616367618, + 0.15916998221547937, + 0.20074131590533828, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 0.1235915933057778, + 0.16226052551901615, + 0.20216802004274595, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 0.1197785865673972, + 0.13338250761817996, + 0.15479680076083163, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 0.756629820126825, + 0.7043569668628038, + 0.6430351791964225, + ], } ), ), ( - {'chunk_number': 4}, + {'chunk_number': 4, 'normalize_confusion_matrix': 'pred'}, pd.DataFrame( { 'key': ['[0:14999]', '[15000:29999]', '[30000:44999]', '[45000:59999]'], @@ -2501,11 +2591,65 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 0.6364205304514962, 0.6375753072973162, ], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 0.7546260682147157, + 0.7511343683695074, + 0.6628383225865804, + 0.6651814251770874, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 0.12922483020709813, + 0.12720280190168412, + 0.22365956156664257, + 0.22578913179209303, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 0.12747696595643684, + 0.12776612448252053, + 0.17277613353669485, + 0.17660735301820177, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 0.12118073967907128, + 0.1249170750987652, + 0.18024418583692642, + 0.17798857692081155, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 0.7554502796336932, + 0.7576402255283115, + 0.5994574163887797, + 0.5998622938235557, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 0.11748464117810321, + 0.11221429055241054, + 0.1924554660281669, + 0.18783942082621902, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 0.12419319210621305, + 0.12394855653172744, + 0.15691749157649315, + 0.15682999790210106, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 0.11532489015920869, + 0.1151569725700045, + 0.17688302204457784, + 0.17434857438435108, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 0.7550383928654599, + 0.7600195849650688, + 0.6347684004351383, + 0.6355532261555792, + ], } ), ), ( - {'chunk_number': 4, 'timestamp_column_name': 'timestamp'}, + {'chunk_number': 4, 'timestamp_column_name': 'timestamp', 'normalize_confusion_matrix': 'all'}, pd.DataFrame( { 'key': ['[0:14999]', '[15000:29999]', '[30000:44999]', '[45000:59999]'], @@ -2535,6 +2679,60 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 0.6364205304514962, 0.6375753072973162, ], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 0.24922783612904678, + 0.24847524905663304, + 0.2702612787293017, + 0.2678907326329857, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 0.044125972021383776, + 0.04231613209929359, + 0.06202825174114887, + 0.06269411559427118, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 0.04184643869129968, + 0.04299755975918424, + 0.05441296365515643, + 0.05644371002461729, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 0.04002195895800795, + 0.04132256844267153, + 0.0734915627052428, + 0.07168193287857484, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 0.25796108881891844, + 0.25204164835908494, + 0.16624952347848823, + 0.16656176358500735, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 0.03856629154406535, + 0.03776384924723789, + 0.0606106414344707, + 0.060033478896059596, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 0.041016871579611966, + 0.041002182500695435, + 0.06398049189878882, + 0.06316066782177283, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 0.03937960582636446, + 0.038308886208288165, + 0.04905555811369625, + 0.04841078748738816, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 0.24785393643130169, + 0.25577192432691115, + 0.1999097282437062, + 0.2031228110793231, + ], } ), ), @@ -2549,6 +2747,15 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 'estimated_recall': [0.6957620347508907, 0.6272720458900231], 'estimated_specificity': [0.8480220572478717, 0.8145095377877009], 'estimated_accuracy': [0.6967957612985849, 0.6305270354546132], + 'estimated_true_highstreet_card_pred_highstreet_card': [15431.207920621628, 106.61852759787631], + 'estimated_true_highstreet_card_pred_prepaid_card': [3140.1950482057946, 27.27202363566655], + 'estimated_true_highstreet_card_pred_upmarket_card': [2911.0243109194275, 24.485771034437157], + 'estimated_true_prepaid_card_pred_highstreet_card': [3369.0309742564546, 28.73937051100256], + 'estimated_true_prepaid_card_pred_prepaid_card': [12568.980575116106, 73.22978850637857], + 'estimated_true_prepaid_card_pred_upmarket_card': [2927.072726648623, 27.541190178880235], + 'estimated_true_upmarket_card_pred_highstreet_card': [3111.761105121915, 25.642101891121143], + 'estimated_true_upmarket_card_pred_prepaid_card': [2605.8243766781006, 21.49818785795489], + 'estimated_true_upmarket_card_pred_upmarket_card': [13514.90296243195, 84.97303878668261], } ), ), @@ -2640,6 +2847,114 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 0.6365172577468735, 0.6393273094601863, ], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 1483.745037516118, + 1536.2546154566053, + 1486.1512390473335, + 1455.1117469508827, + 1504.2836388142573, + 1581.5006283773678, + 1619.4221852490452, + 1653.916785154108, + 1596.0668735461204, + 1621.3736981076686, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 271.9744616336458, + 263.3288788018858, + 255.36687592730394, + 249.4490435511216, + 256.51230189620316, + 373.19858748671334, + 376.5016769489258, + 379.92027913064413, + 383.2996831232405, + 357.9152833417768, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 249.77244098451774, + 249.25341402994002, + 256.5894445268087, + 248.14404310496627, + 268.90063411102597, + 336.40636999479034, + 321.1640326687466, + 329.70510362377274, + 338.73618885526093, + 336.8384100540352, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 249.18645665281267, + 234.635939041771, + 241.34496258349301, + 243.99437206824282, + 251.00618066387275, + 426.16695555097976, + 444.34831788068584, + 437.82379051045245, + 436.76196316220245, + 432.5014066529442, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 1570.046457210577, + 1517.6502407889316, + 1541.0740605507428, + 1544.8307224347122, + 1476.439576685086, + 1000.6602945021018, + 1015.9002964606179, + 990.2540537898107, + 1008.1898500978498, + 977.1648111020529, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 227.67692529471515, + 228.16728611276778, + 224.39810820574257, + 236.66544157504953, + 228.04435068127367, + 373.80025334213235, + 351.54340476695654, + 359.16491614682934, + 364.41698830746793, + 360.7362423945684, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 243.06850583106933, + 254.1094455016238, + 238.50379836917327, + 249.89388098087457, + 244.71018052186997, + 378.3324160716525, + 385.22949687026846, + 390.2594243354395, + 372.1711632916771, + 381.12489523938723, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 237.97908115577718, + 232.0208804091826, + 234.55906352195305, + 231.72023401416573, + 229.04812141871074, + 297.14111801118486, + 301.59802659045613, + 282.82566707954516, + 285.5104667789098, + 294.91990555617025, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 1466.550633720767, + 1484.5792998572922, + 1522.0124472674486, + 1540.1905153199841, + 1541.0550152077003, + 1232.7933766630772, + 1184.292562564297, + 1176.129980229398, + 1214.8468228372712, + 1237.4253475513965, + ], } ), ), @@ -2731,6 +3046,114 @@ def test_cbpe_for_binary_classification_with_timestamps(calculator_opts, expecte 0.6365172577468735, 0.6393273094601863, ], + 'estimated_true_highstreet_card_pred_highstreet_card': [ + 1483.745037516118, + 1536.2546154566053, + 1486.1512390473335, + 1455.1117469508827, + 1504.2836388142573, + 1581.5006283773678, + 1619.4221852490452, + 1653.916785154108, + 1596.0668735461204, + 1621.3736981076686, + ], + 'estimated_true_highstreet_card_pred_prepaid_card': [ + 271.9744616336458, + 263.3288788018858, + 255.36687592730394, + 249.4490435511216, + 256.51230189620316, + 373.19858748671334, + 376.5016769489258, + 379.92027913064413, + 383.2996831232405, + 357.9152833417768, + ], + 'estimated_true_highstreet_card_pred_upmarket_card': [ + 249.77244098451774, + 249.25341402994002, + 256.5894445268087, + 248.14404310496627, + 268.90063411102597, + 336.40636999479034, + 321.1640326687466, + 329.70510362377274, + 338.73618885526093, + 336.8384100540352, + ], + 'estimated_true_prepaid_card_pred_highstreet_card': [ + 249.18645665281267, + 234.635939041771, + 241.34496258349301, + 243.99437206824282, + 251.00618066387275, + 426.16695555097976, + 444.34831788068584, + 437.82379051045245, + 436.76196316220245, + 432.5014066529442, + ], + 'estimated_true_prepaid_card_pred_prepaid_card': [ + 1570.046457210577, + 1517.6502407889316, + 1541.0740605507428, + 1544.8307224347122, + 1476.439576685086, + 1000.6602945021018, + 1015.9002964606179, + 990.2540537898107, + 1008.1898500978498, + 977.1648111020529, + ], + 'estimated_true_prepaid_card_pred_upmarket_card': [ + 227.67692529471515, + 228.16728611276778, + 224.39810820574257, + 236.66544157504953, + 228.04435068127367, + 373.80025334213235, + 351.54340476695654, + 359.16491614682934, + 364.41698830746793, + 360.7362423945684, + ], + 'estimated_true_upmarket_card_pred_highstreet_card': [ + 243.06850583106933, + 254.1094455016238, + 238.50379836917327, + 249.89388098087457, + 244.71018052186997, + 378.3324160716525, + 385.22949687026846, + 390.2594243354395, + 372.1711632916771, + 381.12489523938723, + ], + 'estimated_true_upmarket_card_pred_prepaid_card': [ + 237.97908115577718, + 232.0208804091826, + 234.55906352195305, + 231.72023401416573, + 229.04812141871074, + 297.14111801118486, + 301.59802659045613, + 282.82566707954516, + 285.5104667789098, + 294.91990555617025, + ], + 'estimated_true_upmarket_card_pred_upmarket_card': [ + 1466.550633720767, + 1484.5792998572922, + 1522.0124472674486, + 1540.1905153199841, + 1541.0550152077003, + 1232.7933766630772, + 1184.292562564297, + 1176.129980229398, + 1214.8468228372712, + 1237.4253475513965, + ], } ), ), @@ -2756,11 +3179,24 @@ def test_cbpe_for_multiclass_classification_with_timestamps(calculator_opts, exp y_pred='y_pred', y_true='y_true', problem_type='classification_multiclass', - metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy'], + metrics=['roc_auc', 'f1', 'precision', 'recall', 'specificity', 'accuracy', 'confusion_matrix'], **calculator_opts, ).fit(ref_df) result = cbpe.estimate(ana_df) - sut = result.filter(period='analysis').to_df()[[('chunk', 'key')] + [(m.name, 'value') for m in result.metrics]] + column_names = [(m.name, 'value') for m in result.metrics] + column_names = [c for c in column_names if c[0] != 'confusion_matrix'] + column_names += [ + ('true_highstreet_card_pred_highstreet_card', 'value'), + ('true_highstreet_card_pred_prepaid_card', 'value'), + ('true_highstreet_card_pred_upmarket_card', 'value'), + ('true_prepaid_card_pred_highstreet_card', 'value'), + ('true_prepaid_card_pred_prepaid_card', 'value'), + ('true_prepaid_card_pred_upmarket_card', 'value'), + ('true_upmarket_card_pred_highstreet_card', 'value'), + ('true_upmarket_card_pred_prepaid_card', 'value'), + ('true_upmarket_card_pred_upmarket_card', 'value'), + ] + sut = result.filter(period='analysis').to_df()[[('chunk', 'key')] + column_names] sut.columns = [ 'key', 'estimated_roc_auc', @@ -2769,6 +3205,15 @@ def test_cbpe_for_multiclass_classification_with_timestamps(calculator_opts, exp 'estimated_recall', 'estimated_specificity', 'estimated_accuracy', + 'estimated_true_highstreet_card_pred_highstreet_card', + 'estimated_true_highstreet_card_pred_prepaid_card', + 'estimated_true_highstreet_card_pred_upmarket_card', + 'estimated_true_prepaid_card_pred_highstreet_card', + 'estimated_true_prepaid_card_pred_prepaid_card', + 'estimated_true_prepaid_card_pred_upmarket_card', + 'estimated_true_upmarket_card_pred_highstreet_card', + 'estimated_true_upmarket_card_pred_prepaid_card', + 'estimated_true_upmarket_card_pred_upmarket_card', ] pd.testing.assert_frame_equal(expected, sut) diff --git a/tests/test_thresholds.py b/tests/test_thresholds.py index dcd50d894..c3f429ee3 100644 --- a/tests/test_thresholds.py +++ b/tests/test_thresholds.py @@ -72,7 +72,7 @@ def test_standard_deviation_threshold_init_sets_default_instance_attributes(): assert sut.std_lower_multiplier == 3 assert sut.std_upper_multiplier == 3 - assert sut.offset_from == np.mean + assert sut.offset_from == np.nanmean @pytest.mark.parametrize( @@ -153,3 +153,11 @@ def test_standard_deviation_threshold_raises_threshold_exception_when_negative_l def test_standard_deviation_threshold_raises_threshold_exception_when_negative_upper_multiplier_given(): with pytest.raises(ThresholdException, match="'std_upper_multiplier' should be greater than 0 but got value -1"): _ = StandardDeviationThreshold(0, -1) + + +def test_standard_deviation_threshold_deals_with_nan_values(): + t = StandardDeviationThreshold() + upper, lower = t.thresholds(np.asarray([-1, 1, np.nan, 1, np.nan])) + + assert not np.isnan(upper) + assert not np.isnan(lower)