From 8a6e0870e1326100191b617c8c9139b407e47397 Mon Sep 17 00:00:00 2001 From: Simon Thompson Date: Thu, 7 Mar 2024 11:08:51 +0000 Subject: [PATCH] Addi new docker containers (#589) --- .github/workflows/tre-sql-pg-container.yaml | 39 +++++++ .../workflows/tre-sql-trino-container.yaml | 39 +++++++ DARE-Control.sln | 14 +++ src/Tre-Hasura/Program.cs | 2 +- src/Tre-SQL-PG/Dockerfile | 20 ++++ src/Tre-SQL-PG/Program.cs | 89 ++++++++++++++ src/Tre-SQL-PG/Properties/launchSettings.json | 10 ++ src/Tre-SQL-PG/Tre-SQL-PG.csproj | 18 +++ src/Tre-SQL-Trino/Dockerfile | 20 ++++ src/Tre-SQL-Trino/Program.cs | 110 ++++++++++++++++++ .../Properties/launchSettings.json | 10 ++ src/Tre-SQL-Trino/Tre-SQL-Trino.csproj | 19 +++ 12 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tre-sql-pg-container.yaml create mode 100644 .github/workflows/tre-sql-trino-container.yaml create mode 100644 src/Tre-SQL-PG/Dockerfile create mode 100644 src/Tre-SQL-PG/Program.cs create mode 100644 src/Tre-SQL-PG/Properties/launchSettings.json create mode 100644 src/Tre-SQL-PG/Tre-SQL-PG.csproj create mode 100644 src/Tre-SQL-Trino/Dockerfile create mode 100644 src/Tre-SQL-Trino/Program.cs create mode 100644 src/Tre-SQL-Trino/Properties/launchSettings.json create mode 100644 src/Tre-SQL-Trino/Tre-SQL-Trino.csproj diff --git a/.github/workflows/tre-sql-pg-container.yaml b/.github/workflows/tre-sql-pg-container.yaml new file mode 100644 index 000000000..1b4b0f1d7 --- /dev/null +++ b/.github/workflows/tre-sql-pg-container.yaml @@ -0,0 +1,39 @@ +name: TRE SQL PG Container + +on: + pull_request: + paths: + - 'src/Tre-SQL-PG/**' + push: + branches: + - main + paths: + - 'src/Tre-SQL-PG/**' + +permissions: + contents: write + pull-requests: read + actions: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + tre-hasura-container: + uses: SwanseaUniversityMedical/workflows/.github/workflows/pr-and-release-container.yaml@v1.4.2-containers + with: + job-name: tre-sqlpg-container + registry: ${{ vars.HARBOR_REGISTRY }} + registry-user: ${{ vars.HARBOR_USER }} + registry-repo: ${{ vars.HARBOR_PROJECT }}/${{ vars.HARBOR_REPO }}-tre-sqlpg + release-tag-format: 'DARE-TRE-SQLPG-Container-${version}' + cosign-public-key: ${{ vars.COSIGN_PUBLIC_KEY }} + slack-channel: ${{ vars.SLACK_CHANNEL }} + build-file: src/Tre-SQL-PG/Dockerfile + build-context: src + secrets: + cosign-private-key: ${{ secrets.COSIGN_PRIVATE_KEY }} + cosign-password: ${{ secrets.COSIGN_PASSWORD }} + registry-token: ${{ secrets.HARBOR_TOKEN }} + slack-token: ${{ secrets.SLACK_TOKEN }} diff --git a/.github/workflows/tre-sql-trino-container.yaml b/.github/workflows/tre-sql-trino-container.yaml new file mode 100644 index 000000000..344a1f571 --- /dev/null +++ b/.github/workflows/tre-sql-trino-container.yaml @@ -0,0 +1,39 @@ +name: TRE SQL Trino Container + +on: + pull_request: + paths: + - 'src/Tre-SQL-Trino/**' + push: + branches: + - main + paths: + - 'src/Tre-SQL-Trino/**' + +permissions: + contents: write + pull-requests: read + actions: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + tre-hasura-container: + uses: SwanseaUniversityMedical/workflows/.github/workflows/pr-and-release-container.yaml@v1.4.2-containers + with: + job-name: tre-sqltrino-container + registry: ${{ vars.HARBOR_REGISTRY }} + registry-user: ${{ vars.HARBOR_USER }} + registry-repo: ${{ vars.HARBOR_PROJECT }}/${{ vars.HARBOR_REPO }}-tre-sqltrino + release-tag-format: 'DARE-TRE-SQLTRINO-Container-${version}' + cosign-public-key: ${{ vars.COSIGN_PUBLIC_KEY }} + slack-channel: ${{ vars.SLACK_CHANNEL }} + build-file: src/Tre-SQL-Trino/Dockerfile + build-context: src + secrets: + cosign-private-key: ${{ secrets.COSIGN_PRIVATE_KEY }} + cosign-password: ${{ secrets.COSIGN_PASSWORD }} + registry-token: ${{ secrets.HARBOR_TOKEN }} + slack-token: ${{ secrets.SLACK_TOKEN }} diff --git a/DARE-Control.sln b/DARE-Control.sln index eeed214f3..c44c66b97 100644 --- a/DARE-Control.sln +++ b/DARE-Control.sln @@ -28,6 +28,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Data-Egress-UI", "src\Data- EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tre-Hasura", "src\Tre-Hasura\Tre-Hasura.csproj", "{3BAD3B69-2B08-4E09-BB8A-2611A0990062}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tre-SQL-PG", "src\Tre-SQL-PG\Tre-SQL-PG.csproj", "{C51CB601-57E5-445D-975E-62041FCF65A9}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tre-SQL-Trino", "src\Tre-SQL-Trino\Tre-SQL-Trino.csproj", "{B47DC0A4-0691-456F-BBB2-7901E2398526}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -70,6 +74,14 @@ Global {3BAD3B69-2B08-4E09-BB8A-2611A0990062}.Debug|Any CPU.Build.0 = Debug|Any CPU {3BAD3B69-2B08-4E09-BB8A-2611A0990062}.Release|Any CPU.ActiveCfg = Release|Any CPU {3BAD3B69-2B08-4E09-BB8A-2611A0990062}.Release|Any CPU.Build.0 = Release|Any CPU + {C51CB601-57E5-445D-975E-62041FCF65A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C51CB601-57E5-445D-975E-62041FCF65A9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C51CB601-57E5-445D-975E-62041FCF65A9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C51CB601-57E5-445D-975E-62041FCF65A9}.Release|Any CPU.Build.0 = Release|Any CPU + {B47DC0A4-0691-456F-BBB2-7901E2398526}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B47DC0A4-0691-456F-BBB2-7901E2398526}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B47DC0A4-0691-456F-BBB2-7901E2398526}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B47DC0A4-0691-456F-BBB2-7901E2398526}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -85,6 +97,8 @@ Global {68369D1D-B5AE-4CED-944C-3FEF0EB23414} = {BCA7B4D1-726B-46EA-90D1-2BD3ACD5154A} {80C9C407-D56A-4FD1-9FE3-58B046729ADC} = {ED66814A-91F3-4E06-AE48-B7A9F666659E} {3BAD3B69-2B08-4E09-BB8A-2611A0990062} = {505AD4FB-0822-4989-9A01-1F56C3820D0D} + {C51CB601-57E5-445D-975E-62041FCF65A9} = {505AD4FB-0822-4989-9A01-1F56C3820D0D} + {B47DC0A4-0691-456F-BBB2-7901E2398526} = {505AD4FB-0822-4989-9A01-1F56C3820D0D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {29E246A8-202C-4544-A30D-33E0AB416005} diff --git a/src/Tre-Hasura/Program.cs b/src/Tre-Hasura/Program.cs index b0ff50a25..b2e1405df 100644 --- a/src/Tre-Hasura/Program.cs +++ b/src/Tre-Hasura/Program.cs @@ -8,7 +8,7 @@ using Tre_Hasura.Models; -Console.WriteLine("Hello, World!"); +Console.WriteLine("TREFX Hashura Runner"); var configuration = GetConfiguration(); using IHost host = Host.CreateDefaultBuilder(args).ConfigureServices diff --git a/src/Tre-SQL-PG/Dockerfile b/src/Tre-SQL-PG/Dockerfile new file mode 100644 index 000000000..c677c19fc --- /dev/null +++ b/src/Tre-SQL-PG/Dockerfile @@ -0,0 +1,20 @@ +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/runtime:7.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build +WORKDIR /src +COPY ["src/Tre-SQL-PG/Tre-SQL-PG.csproj", "src/Tre-SQL-PG/"] +RUN dotnet restore "src/Tre-SQL-PG/Tre-SQL-PG.csproj" +COPY . . +WORKDIR "/src/src/Tre-SQL-PG" +RUN dotnet build "Tre-SQL-PG.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "Tre-SQL-PG.csproj" -c Release -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Tre-SQL-PG.dll"] \ No newline at end of file diff --git a/src/Tre-SQL-PG/Program.cs b/src/Tre-SQL-PG/Program.cs new file mode 100644 index 000000000..9d01a51be --- /dev/null +++ b/src/Tre-SQL-PG/Program.cs @@ -0,0 +1,89 @@ +using System; +using System.Data; +using System.IO; +using Npgsql; + +Console.WriteLine("TREFX SQL Runner Module : Postgress"); + +// Tre-SQL-PG.exe "--Query=select * from \"profileForm\"" --Output=data.csv + +var Token = ""; +var OutputFilename = "data.csv"; +var Query = "SELECT * FROM \"profileForm\""; +string connectionString = "Host=localhost:5432;Username=admin;Password=admin;Database=assets3"; + +foreach (var arg in args) +{ + Console.WriteLine("Arg = " + arg); + + if (arg.StartsWith("--Output")) + { + OutputFilename = arg.Replace("--Output=", ""); + } + + if (arg.StartsWith("--Token_")) + { + Token = arg.Replace("--Token_", ""); + } + + if (arg.StartsWith("--Connection")) + { + connectionString = arg.Replace("--Connection=", ""); + } + + if (arg.StartsWith("--Query")) + { + Query = arg.Replace("--Query=", ""); + } + +} + +Console.WriteLine("Query > " + Query); +Console.WriteLine("Token > " + Token); +Console.WriteLine("Output > " + OutputFilename); + +ExportToCsv(connectionString, Query, OutputFilename); + +Console.WriteLine("Completed"); + + + +static void ExportToCsv(string connectionString, string sqlQuery, string csvFilePath) +{ + using (var connection = new NpgsqlConnection(connectionString)) + { + connection.Open(); + using (var command = new NpgsqlCommand(sqlQuery, connection)) + { + using (var reader = command.ExecuteReader()) + { + if (reader.HasRows) + { + using (var streamWriter = new StreamWriter(csvFilePath)) + { + // Write column headers + for (int i = 0; i < reader.FieldCount; i++) + { + if (i > 0) + streamWriter.Write(","); + streamWriter.Write(reader.GetName(i)); + } + streamWriter.WriteLine(); + + // Write data rows + while (reader.Read()) + { + for (int i = 0; i < reader.FieldCount; i++) + { + if (i > 0) + streamWriter.Write(","); + streamWriter.Write(reader[i].ToString()); + } + streamWriter.WriteLine(); + } + } + } + } + } + } +} diff --git a/src/Tre-SQL-PG/Properties/launchSettings.json b/src/Tre-SQL-PG/Properties/launchSettings.json new file mode 100644 index 000000000..8016a40c5 --- /dev/null +++ b/src/Tre-SQL-PG/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "Tre-SQL-PG": { + "commandName": "Project" + }, + "Docker": { + "commandName": "Docker" + } + } +} \ No newline at end of file diff --git a/src/Tre-SQL-PG/Tre-SQL-PG.csproj b/src/Tre-SQL-PG/Tre-SQL-PG.csproj new file mode 100644 index 000000000..55467efa1 --- /dev/null +++ b/src/Tre-SQL-PG/Tre-SQL-PG.csproj @@ -0,0 +1,18 @@ + + + + Exe + net7.0 + Tre_SQL_PG + enable + enable + Linux + ..\.. + + + + + + + + diff --git a/src/Tre-SQL-Trino/Dockerfile b/src/Tre-SQL-Trino/Dockerfile new file mode 100644 index 000000000..3976404f5 --- /dev/null +++ b/src/Tre-SQL-Trino/Dockerfile @@ -0,0 +1,20 @@ +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/runtime:7.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build +WORKDIR /src +COPY ["src/Tre-SQL-Trino/Tre-SQL-Trino.csproj", "src/Tre-SQL-Trino/"] +RUN dotnet restore "src/Tre-SQL-Trino/Tre-SQL-Trino.csproj" +COPY . . +WORKDIR "/src/src/Tre-SQL-Trino" +RUN dotnet build "Tre-SQL-Trino.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "Tre-SQL-Trino.csproj" -c Release -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Tre-SQL-Trino.dll"] \ No newline at end of file diff --git a/src/Tre-SQL-Trino/Program.cs b/src/Tre-SQL-Trino/Program.cs new file mode 100644 index 000000000..5bb78ccb8 --- /dev/null +++ b/src/Tre-SQL-Trino/Program.cs @@ -0,0 +1,110 @@ +using System; +using System.Data; +using System.IO; +using System.Net.Http; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Threading.Tasks; + +Console.WriteLine("TREFX SQL Runner Module : Trino"); + +// Tre-SQL-Trino.exe "--Query=select * from \"profileForm\"" --Output=data.csv + +string trinoUrl = "http://trino-server:8080/v1/statement"; +var Token = ""; +var OutputFilename = "data.csv"; +var Query = "SELECT * FROM \"profileForm\""; + +foreach (var arg in args) +{ + Console.WriteLine("Arg = " + arg); + + if (arg.StartsWith("--Output")) + { + OutputFilename = arg.Replace("--Output=", ""); + } + + if (arg.StartsWith("--Token_")) + { + Token = arg.Replace("--Token_", ""); + } + + if (arg.StartsWith("--Connection")) + { + trinoUrl = arg.Replace("--Connection=", ""); + } + + if (arg.StartsWith("--Query")) + { + Query = arg.Replace("--Query=", ""); + } + +} + +Console.WriteLine("Query > " + Query); +Console.WriteLine("Token > " + Token); +Console.WriteLine("Output > " + OutputFilename); + +await ExportToCsv(trinoUrl, Query, OutputFilename); + +Console.WriteLine("Completed"); + + +static async Task ExportToCsv(string trinoUrl, string sqlQuery, string csvFilePath) +{ + using (var httpClient = new HttpClient()) + { + var requestBody = new + { + query = sqlQuery + }; + + var requestContent = new StringContent(JsonConvert.SerializeObject(requestBody)); + requestContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); + + using (var response = await httpClient.PostAsync(trinoUrl, requestContent)) + { + response.EnsureSuccessStatusCode(); + + using (var responseStream = await response.Content.ReadAsStreamAsync()) + using (var streamReader = new StreamReader(responseStream)) + using (var jsonReader = new JsonTextReader(streamReader)) + { + var jsonSerializer = new JsonSerializer(); + + using (var streamWriter = new StreamWriter(csvFilePath)) + using (var csvWriter = new CsvHelper.CsvWriter(streamWriter, System.Globalization.CultureInfo.InvariantCulture)) + { + while (jsonReader.Read()) + { + if (jsonReader.TokenType == JsonToken.StartObject) + { + // Deserialize JSON response into a dynamic object + var jsonObject = jsonSerializer.Deserialize(jsonReader); + + // Write column headers + var columns = jsonObject["columns"]; + foreach (var column in columns) + { + csvWriter.WriteField(column["name"].ToString()); + } + csvWriter.NextRecord(); + + // Write data rows + var data = jsonObject["data"]; + foreach (var rowData in data) + { + foreach (var value in rowData) + { + csvWriter.WriteField(value.ToString()); + } + csvWriter.NextRecord(); + } + } + } + } + } + } + } +} + diff --git a/src/Tre-SQL-Trino/Properties/launchSettings.json b/src/Tre-SQL-Trino/Properties/launchSettings.json new file mode 100644 index 000000000..3c254e39c --- /dev/null +++ b/src/Tre-SQL-Trino/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "Tre-SQL-Trino": { + "commandName": "Project" + }, + "Docker": { + "commandName": "Docker" + } + } +} \ No newline at end of file diff --git a/src/Tre-SQL-Trino/Tre-SQL-Trino.csproj b/src/Tre-SQL-Trino/Tre-SQL-Trino.csproj new file mode 100644 index 000000000..e6f64ee23 --- /dev/null +++ b/src/Tre-SQL-Trino/Tre-SQL-Trino.csproj @@ -0,0 +1,19 @@ + + + + Exe + net7.0 + Tre_SQL_Trino + enable + enable + Linux + ..\.. + + + + + + + + +