Task/RDMP-248 data load allow specials (#1992)

* add sql changes

* add toggle

* add missing files

* add changelog

* fix typo

* update imports

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Re-Instantiate connection to FTP server on FTP Downloader cleanup
 - Add AWS S3 Bucket Release component for flat files
 - Add UI linkage for projects and cohort builder configurations
+- Add ability to allow data loads to import columns with reserved prefixes
 - Add goto for Cohort Identification Configuration from External Cohort
 
 ## [8.2.3] - 2024-08-05

Documentation/DataLoadEngine/ReservedColumnPrefixes.md

@@ -0,0 +1,25 @@
+# Reserved Column Prefixes
+By default RDMP uses the prefix "hic_" for internally important columns.
+We advise that you typically don't use the prefix in your table schemas.
+
+## Data Loads
+By default, RDMP will not import a column stating with "hic" in the RAW stage of a data load.
+If you want to bypass this for whatever reason, you can allow this type of column on a Dataload by Dataload basis.
+
+Via the CLI:
+```
+./rdmp.exe ToggleAllowReservedPrefixForLoadMetadata LoadMedata:<id>
+```
+
+
+Via the UI:
+Right Click on a Data Load.
+
+There will be a menu option that says "Allow Reserved Prefix Columns" or "Drop Reserved Prefix Columns" depending on the current configuration.
+
+![Attachers Location](./Images/AllowReservedPrefixColumns.PNG)
+
+
+Selecting "Allow..." will enable "hic_" prefixed columns to be imported during a data load.
+
+Selecting "Drop..." will prevent "hic_" prefixed columns from being imported during a data load

Rdmp.Core.Tests/DataLoad/Engine/Integration/DatabaseOperationTests.cs

@@ -75,7 +75,7 @@ public void CloneDatabaseAndTable()
             //now clone the catalogue data structures to MachineName
             foreach (TableInfo tableInfo in cata.GetTableInfoList(false))
                 cloner.CreateTablesInDatabaseFromCatalogueInfo(ThrowImmediatelyDataLoadEventListener.Quiet, tableInfo,
-                    LoadBubble.Raw);
+                    LoadBubble.Raw,false);
 
             Assert.Multiple(() =>
             {

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -1,4 +1,4 @@
-// Copyright (c) The University of Dundee 2018-2019
+// Copyright (c) The University of Dundee 2018-2024
 // This file is part of the Research Data Management Platform (RDMP).
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
@@ -506,7 +506,11 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
 
             yield return new ExecuteCommandOverrideRawServer(_activator, lmd);
             yield return new ExecuteCommandCreateNewLoadMetadata(_activator);
-
+            var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow";
+            yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd)
+            {
+                OverrideCommandName=$"{reservedTest} Reserved Prefix Columns"
+            };
 
             yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced };
             yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced };

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleAllowReservedPrefixForLoadMetadata.cs

@@ -0,0 +1,21 @@
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Curation.Data.DataLoad;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands;
+
+public class ExecuteCommandToggleAllowReservedPrefixForLoadMetadata : BasicCommandExecution
+{
+    private LoadMetadata _loadMetadata;
+    public ExecuteCommandToggleAllowReservedPrefixForLoadMetadata([DemandsInitialization("The LoadMetadata to update")] LoadMetadata loadMetadata)
+    {
+
+        _loadMetadata = loadMetadata;
+    }
+
+    public override void Execute()
+    {
+        base.Execute();
+        _loadMetadata.AllowReservedPrefix = !_loadMetadata.AllowReservedPrefix;
+        _loadMetadata.SaveToDatabase();
+    }
+}

Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs

@@ -56,6 +56,11 @@ public interface ILoadMetadata : INamed, ILoggedActivityRootObject
     /// </summary>
     ExternalDatabaseServer OverrideRAWServer { get; }
 
+    /// <summary>
+    /// Optional - Allows for columns with the reserved prefix 'hic_' to be imported when doing a data load
+    /// </summary>
+    bool AllowReservedPrefix { get; }
+
     /// <summary>
     /// List of all the user configured steps in a data load.  For example you could have 2 ProcessTasks, one that downloads files from an FTP server and one that loads RAW.
     /// </summary>

Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs

@@ -60,7 +60,7 @@ public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHa
     private bool _ignoreTrigger;
     private string _folder;
     private DateTime? _lastLoadTime;
-
+    private bool _allowReservedPrefix;
 
     public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading");
     public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving");
@@ -83,6 +83,14 @@ public DirectoryInfo GetRootDirectory()
         return null;
     }
 
+
+    ///  <inheritdoc/>
+    public bool AllowReservedPrefix
+    {
+        get => _allowReservedPrefix;
+        set => SetField(ref _allowReservedPrefix, value);
+    }
+
     ///  <inheritdoc/>
     public string LocationOfForLoadingDirectory
     {
@@ -252,6 +260,7 @@ internal LoadMetadata(ICatalogueRepository repository, DbDataReader r)
         IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false;
         Folder = r["Folder"] as string ?? FolderHelper.Root;
         LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ? null : DateTime.Parse(r["LastLoadTime"].ToString());
+        AllowReservedPrefix = ObjectToNullableBool(r["AllowReservedPrefix"]) ?? false;
     }
 
     internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base()

Rdmp.Core/DataLoad/Engine/DatabaseManagement/Operations/DatabaseCloner.cs

@@ -46,7 +46,7 @@ public DiscoveredDatabase CreateDatabaseForStage(LoadBubble stageToCreateDatabas
     }
 
     public void CreateTablesInDatabaseFromCatalogueInfo(IDataLoadEventListener listener, TableInfo tableInfo,
-        LoadBubble copyToStage)
+        LoadBubble copyToStage, bool allowReservedPrefixColumns)
     {
         if (copyToStage == LoadBubble.Live)
             throw new Exception("Please don't try to create tables in the live database");
@@ -55,9 +55,7 @@ public void CreateTablesInDatabaseFromCatalogueInfo(IDataLoadEventListener liste
 
         var cloneOperation = new TableInfoCloneOperation(_hicDatabaseConfiguration, tableInfo, copyToStage, listener)
         {
-            DropHICColumns =
-                copyToStage ==
-                LoadBubble.Raw, //don't drop columns like hic_sourceID, these are optional for population (and don't get Diff'ed) but should still be there
+            DropHICColumns = !allowReservedPrefixColumns,
             AllowNulls = copyToStage == LoadBubble.Raw
         };
 

Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

@@ -151,11 +151,12 @@ public void LogWarning(string senderName, string message)
 
     public void CreateTablesInStage(DatabaseCloner cloner, LoadBubble stage)
     {
+        bool allowReservedPrefixColumns = stage == LoadBubble.Raw ? LoadMetadata.AllowReservedPrefix : true;
         foreach (TableInfo regularTableInfo in RegularTablesToLoad)
-            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage);
+            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, regularTableInfo, stage, allowReservedPrefixColumns);
 
         foreach (TableInfo lookupTableInfo in LookupTablesToLoad)
-            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage);
+            cloner.CreateTablesInDatabaseFromCatalogueInfo(_listener, lookupTableInfo, stage, allowReservedPrefixColumns);
 
         PushForDisposal(cloner);
     }

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/PopulateRAW.cs

@@ -88,7 +88,6 @@ private void CreateRawDatabaseIfRequired(IDataLoadJob job)
 
         if (!job.PersistentRaw) cloner.CreateDatabaseForStage(LoadBubble.Raw);
 
-
         job.CreateTablesInStage(cloner, LoadBubble.Raw);
     }
 

Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql

@@ -598,6 +598,7 @@ CREATE TABLE [dbo].[LoadMetadata](
 	[CacheFilenameDateFormat] [varchar](20) NOT NULL,
 	[CacheArchiveType] [int] NOT NULL,
 	[SoftwareVersion] [nvarchar](50) NOT NULL,
+	[AllowReservedPrefix] [bit] NOT NULL default 0,
  CONSTRAINT [PK_LoadMetadata] PRIMARY KEY CLUSTERED 
 (
 	[ID] ASC

Rdmp.Core/Databases/CatalogueDatabase/up/086_AddDataLoadPrefixOverride.sql

@@ -0,0 +1,8 @@
+--Version: 8.2.4
+--Description: Add ability to allow data loads to import columns with the reserved column _hic
+
+ if not exists (select 1 from sys.columns where name = 'AllowReservedPrefix' and OBJECT_NAME(object_id) = 'LoadMetadata')
+ BEGIN
+ ALTER TABLE [dbo].[LoadMetadata]
+ ADD AllowReservedPrefix [bit] NOT NULL DEFAULT 0 WITH VALUES
+ END

Rdmp.Core/Rdmp.Core.csproj

@@ -128,6 +128,7 @@
 		<None Remove="Databases\CatalogueDatabase\up\083_AddGroupBy.sql" />
 		<None Remove="Databases\CatalogueDatabase\up\084_AddLoadDirectorySplit.sql" />
 		<None Remove="Databases\CatalogueDatabase\up\085_AddTicketingReleaseStatuses.sql" />
+		<None Remove="Databases\CatalogueDatabase\up\086_AddDataLoadPrefixOverride.sql" />
 		<None Remove="Databases\DataExportDatabase\runAfterCreateDatabase\CreateDataExportManager.sql" />
 		<None Remove="Databases\DataExportDatabase\up\001_AddUsernamePasswordFieldsToExternalCohortTable.sql" />
 		<None Remove="Databases\DataExportDatabase\up\002_FixServerAndDatabaseNameOnExternalCohort.sql" />
@@ -253,6 +254,7 @@
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\078_AddLastLoadTimeToLoadMetadata.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\084_AddLoadDirectorySplit.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\085_AddTicketingReleaseStatuses.sql" />
+		<EmbeddedResource Include="Databases\CatalogueDatabase\up\086_AddDataLoadPrefixOverride.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\079_AddProcessTaskConfiguration.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\082_AddCohortVersioning.sql" />
 		<EmbeddedResource Include="Databases\CatalogueDatabase\up\083_AddGroupBy.sql" />

SharedAssemblyInfo.cs

@@ -10,6 +10,6 @@
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
-[assembly: AssemblyVersion("8.2.3")]
-[assembly: AssemblyFileVersion("8.2.3")]
-[assembly: AssemblyInformationalVersion("8.2.3")]
+[assembly: AssemblyVersion("8.2.4")]
+[assembly: AssemblyFileVersion("8.2.4")]
+[assembly: AssemblyInformationalVersion("8.2.4")]