diff --git a/Rdmp.Core.Tests/DataQualityEngine/DQEPartialUpdateTests.cs b/Rdmp.Core.Tests/DataQualityEngine/DQEPartialUpdateTests.cs index 6b2952f7af..71737a6881 100644 --- a/Rdmp.Core.Tests/DataQualityEngine/DQEPartialUpdateTests.cs +++ b/Rdmp.Core.Tests/DataQualityEngine/DQEPartialUpdateTests.cs @@ -1,8 +1,5 @@ -using NLog; -using NPOI.POIFS.Properties; -using NPOI.SS.Formula.Functions; +using NPOI.SS.Formula.Functions; using NUnit.Framework; -using Org.BouncyCastle.Tls; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; @@ -20,33 +17,25 @@ using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.DataQualityEngine.Data; using Rdmp.Core.DataQualityEngine.Reports; -using Rdmp.Core.Logging; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; using Rdmp.Core.Tests.DataLoad.Engine.Integration; -using System; using System.Collections.Generic; using System.Data; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; using System.Threading; -using System.Threading.Tasks; -using Tests.Common; -using static Org.BouncyCastle.Math.EC.ECCurve; namespace Rdmp.Core.Tests.DataQualityEngine { internal class DQEPartialUpdateTests : DataLoadEngineTestsBase { - string validatorXML = "\r\n\r\n \r\n \r\n \r\n Wrong\r\n \r\n chi\r\n \r\n \r\n \r\n time\r\n \r\n \r\n \r\n"; - string fileLocation = Path.GetTempPath(); - string fileName = "SteppedDQEPartialUpdates.csv"; + readonly string validatorXML = "\r\n\r\n \r\n \r\n \r\n Wrong\r\n \r\n chi\r\n \r\n \r\n \r\n time\r\n \r\n \r\n \r\n"; + readonly string fileLocation = Path.GetTempPath(); + readonly string fileName = "SteppedDQEPartialUpdates.csv"; [Test] public void SteppedDQEPartialUpdates() @@ -63,7 +52,7 @@ public void SteppedDQEPartialUpdates() var table = server.CreateTable("PartialToaDQE", dt); table.CreatePrimaryKey(table.DiscoverColumns().Where(c => c.GetRuntimeName() == "chi").ToArray()); - + dt.Dispose(); var catalogue = new Catalogue(CatalogueRepository, "PartialToaDQE"); var importer = new TableInfoImporter(CatalogueRepository, table); importer.DoImport(out var _tableInfo, out var _columnInfos); @@ -93,7 +82,7 @@ public void SteppedDQEPartialUpdates() var listener = new ToMemoryDataLoadEventListener(false); report.GenerateReport(catalogue, listener, source.Token); - + source.Dispose(); var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); lmd.LocationOfForLoadingDirectory = Path.GetTempPath(); lmd.LocationOfForArchivingDirectory = Path.GetTempPath(); @@ -144,7 +133,6 @@ public void SteppedDQEPartialUpdates() dt.Rows.Add(new string[] { "1111111112", "A", "2024-11-01" }); dt.Rows.Add(new string[] { "1111111113", "B", "2024-10-01" }); SetupFile(dt); - PerformLoad(lmd, logManager); //end of first load report = new CatalogueConstraintReport(catalogue, SpecialFieldNames.DataLoadRunID) @@ -296,10 +284,6 @@ public void SteppedDQEPartialUpdates() evaluations = dqeRepository.GetAllObjectsWhere("CatalogueID", catalogue.ID).ToList();//.Where(e => e.CatalogueID == catalogue.ID).ToList(); Assert.That(evaluations.Count, Is.EqualTo(13)); CompareEvaluations(evaluations[12], evaluations[11]); - - - Assert.That(true, Is.EqualTo(true)); - } private void SetupFile(DataTable dt) @@ -318,6 +302,7 @@ private void SetupFile(DataTable dt) lines.Add(string.Join(',', row.ItemArray.Select(i => i.ToString()))); } File.AppendAllLines(Path.Combine(fileLocation, fileName), lines); + dt.Dispose(); } private void CompareEvaluations(Evaluation e1, Evaluation e2) diff --git a/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs b/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs index f8a65c9a82..4c46c07d52 100644 --- a/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs +++ b/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs @@ -215,10 +215,8 @@ public void UpdateReport(ICatalogue c, int dataLoadID, int? commandTimeout, IDat var reportBuilder = new ReportBuilder(c, _validator, _queryBuilder, _dataLoadRunFieldName, _containsDataLoadID, _timePeriodicityField, _pivotCategory, rDT); reportBuilder.BuildReportInternals(cancellationToken, forker, dqeRepository); var newByPivotRowStatesOverDataLoadRunId = reportBuilder.GetByPivotRowStatesOverDataLoadRunId(); - var newByPivotCategoryCubesOverTime = reportBuilder.GetByPivotCategoryCubesOverTime(); var pivotColumn = c.PivotCategory_ExtractionInformation.ColumnInfo.GetRuntimeName(); - var timeColumn = c.TimeCoverage_ExtractionInformation.ColumnInfo.GetRuntimeName(); var incomingPivotCategories = rDT.AsEnumerable().Select(r => r[pivotColumn].ToString()).ToList().Distinct(); @@ -402,9 +400,9 @@ public void UpdateReport(ICatalogue c, int dataLoadID, int? commandTimeout, IDat //* Periodicity States *// //Unchanged - newByPivotCategoryCubesOverTime = [];//reset + Dictionary newByPivotCategoryCubesOverTime = [];//reset - var unchangedPivotCategories = previousRowSates.Where(rs => rs.PivotCategory != "ALL" && !existingIncomingPivotCategories.Contains(rs.PivotCategory) && !replacedPivotCategories.Contains(rs.PivotCategory)).Select(rs => rs.PivotCategory).Distinct(); foreach (var previousRowState in previousRowSates.Where(rs => rs.PivotCategory != "ALL" && !existingIncomingPivotCategories.Contains(rs.PivotCategory) && !replacedPivotCategories.Contains(rs.PivotCategory))) ; + var unchangedPivotCategories = previousRowSates.Where(rs => rs.PivotCategory != "ALL" && !existingIncomingPivotCategories.Contains(rs.PivotCategory) && !replacedPivotCategories.Contains(rs.PivotCategory)).Select(rs => rs.PivotCategory).Distinct(); newByPivotCategoryCubesOverTime.TryGetValue("ALL", out var value); if (value is null) { diff --git a/Rdmp.Core/DataQualityEngine/Reports/ReportBuilder.cs b/Rdmp.Core/DataQualityEngine/Reports/ReportBuilder.cs index 6fafeae879..d63016b12a 100644 --- a/Rdmp.Core/DataQualityEngine/Reports/ReportBuilder.cs +++ b/Rdmp.Core/DataQualityEngine/Reports/ReportBuilder.cs @@ -32,10 +32,10 @@ public class ReportBuilder private readonly Validator _validator; private readonly bool _containsDataLoadID; - public static int MaximumPivotValues = 5000; + private static readonly int MaximumPivotValues = 5000; - private Dictionary byPivotRowStatesOverDataLoadRunId = new(); - private Dictionary byPivotCategoryCubesOverTime = new(); + private readonly Dictionary byPivotRowStatesOverDataLoadRunId = []; + private readonly Dictionary byPivotCategoryCubesOverTime = []; private readonly string _timePeriodicityField; private readonly string _pivotCategory;