From 0984759dcaa9842c47b40755921d490378583e38 Mon Sep 17 00:00:00 2001 From: mahadi99xy <118971382+mahadi99xy@users.noreply.github.com> Date: Mon, 9 Oct 2023 10:46:14 +0100 Subject: [PATCH] fix(api, tre-api): copy from an external and internal bucket (#194) Co-authored-by: jaybeeelsdon --- .../Services/ConsumeInternalMessageService.cs | 49 +++++++++++++++---- src/TRE-API/DoAgentWork.cs | 33 +++++++++++-- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/DARE-API/Services/ConsumeInternalMessageService.cs b/src/DARE-API/Services/ConsumeInternalMessageService.cs index 0bc9af857..88f0fb676 100644 --- a/src/DARE-API/Services/ConsumeInternalMessageService.cs +++ b/src/DARE-API/Services/ConsumeInternalMessageService.cs @@ -8,6 +8,12 @@ using BL.Models.Enums; using DARE_API.Repositories.DbContexts; using BL.Models.Tes; +using BL.Models.ViewModels; +using BL.Services; +using EasyNetQ.Management.Client.Model; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory; +using System.Text.Json.Nodes; namespace DARE_API.Services { @@ -15,13 +21,19 @@ public class ConsumeInternalMessageService : BackgroundService { private readonly IBus _bus; private readonly ApplicationDbContext _dbContext; + private readonly MinioSettings _minioSettings; + private readonly IMinioHelper _minioHelper; + private readonly IDareClientHelper _clientHelper; - public ConsumeInternalMessageService(IBus bus , IServiceProvider serviceProvider) + public ConsumeInternalMessageService(IBus bus, IServiceProvider serviceProvider) { _bus = bus; - _dbContext = serviceProvider.CreateScope().ServiceProvider.GetRequiredService(); ; - + _dbContext = serviceProvider.CreateScope().ServiceProvider.GetRequiredService(); + _minioSettings = serviceProvider.CreateScope().ServiceProvider.GetRequiredService(); + _minioHelper = serviceProvider.CreateScope().ServiceProvider.GetRequiredService(); + _clientHelper = serviceProvider.CreateScope().ServiceProvider.GetRequiredService(); + } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -34,8 +46,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } catch (Exception e) { - Log.Error("{Function} ConsumeProcessForm:- Failed to subscribe due to error: {e}","ExecuteAsync", e.Message); - + Log.Error("{Function} ConsumeProcessForm:- Failed to subscribe due to error: {e}", "ExecuteAsync", e.Message); + } } @@ -47,6 +59,23 @@ private void Process(IMessage message, MessageReceivedInfo info) //TODO: Mahadi copy crate from external to local submission (if in external) + Uri uri = new Uri(sub.DockerInputLocation); + string fileName = Path.GetFileName(uri.LocalPath); + + var messageMQ = new FetchFileMQ(); + messageMQ.Url = sub.SourceCrate; + messageMQ.BucketName = sub.Project.SubmissionBucket; + messageMQ.Key = fileName; + + if (uri.Host + ":" + uri.Port != _minioSettings.Url) + { + _minioHelper.RabbitExternalObject(messageMQ); + + var minioEndpoint = _clientHelper.CallAPIWithoutModel("/api/Project/GetMinioEndPoint").Result; + + messageMQ.Url = "http://" + minioEndpoint.Url + "/browser/" + messageMQ.BucketName + "/" + messageMQ.Key; + } + //TODO: Validate format of Crate var dbproj = sub.Project; @@ -59,7 +88,7 @@ private void Process(IMessage message, MessageReceivedInfo info) tres = trestr.Split('|').Select(x => x.ToLower()).ToList(); } - + var dbtres = new List(); @@ -75,12 +104,12 @@ private void Process(IMessage message, MessageReceivedInfo info) } } UpdateSubmissionStatus.UpdateStatus(sub, StatusType.WaitingForChildSubsToComplete, ""); - + foreach (var tre in dbtres) { _dbContext.Add(new Submission() { - DockerInputLocation = tesTask.Executors.First().Image, + DockerInputLocation = messageMQ.Url, Project = dbproj, StartTime = DateTime.Now.ToUniversalTime(), Status = StatusType.WaitingForAgentToTransfer, @@ -99,7 +128,7 @@ private void Process(IMessage message, MessageReceivedInfo info) _dbContext.SaveChanges(); Log.Information("{Function} Processed sub for {id}", "Process", message.Body); - + } catch (Exception ex) { @@ -108,7 +137,7 @@ private void Process(IMessage message, MessageReceivedInfo info) } } - + private T ConvertByteArrayToType(byte[] byteArray) { diff --git a/src/TRE-API/DoAgentWork.cs b/src/TRE-API/DoAgentWork.cs index 8e02b13e2..0be8e63e6 100644 --- a/src/TRE-API/DoAgentWork.cs +++ b/src/TRE-API/DoAgentWork.cs @@ -20,6 +20,8 @@ using System.Threading.Tasks; using TRE_API.Repositories.DbContexts; using TRE_API.Services; +using BL.Services; +using Microsoft.AspNetCore.SignalR; namespace TRE_API { @@ -39,11 +41,15 @@ public class DoAgentWork : IDoAgentWork private readonly IServiceProvider _serviceProvider; private readonly ApplicationDbContext _dbContext; private readonly ISubmissionHelper _subHelper; - public DoAgentWork(IServiceProvider serviceProvider, ApplicationDbContext dbContext, ISubmissionHelper subHelper) + private readonly MinioSettings _minioSettings; + private readonly IMinioHelper _minioHelper; + public DoAgentWork(IServiceProvider serviceProvider, ApplicationDbContext dbContext, ISubmissionHelper subHelper, MinioSettings minioSettings, IMinioHelper minioHelper) { _serviceProvider = serviceProvider; _dbContext = dbContext; _subHelper = subHelper; + _minioSettings = minioSettings; + _minioHelper = minioHelper; } public void testing() @@ -236,10 +242,10 @@ public void Execute() { Log.Information("Submission: {submission}", aSubmission); - - + //TODO: Put user approval check + // Check user is allowed ont he project - if ( ! _subHelper.IsUserApprovedOnProject(aSubmission.Project.Id, aSubmission.SubmittedBy.Id)) + if (false && ! _subHelper.IsUserApprovedOnProject(aSubmission.Project.Id, aSubmission.SubmittedBy.Id)) { Log.Error("User {UserID}/project {ProjectId} is not value for this submission {submission}", aSubmission.SubmittedBy.Id, aSubmission.Project.Id, aSubmission); // record error with submission layer @@ -248,7 +254,26 @@ public void Execute() else { + //TODO: Mahadi copy from submission input bucket to tre input bucket + try + { + Uri uri = new Uri(aSubmission.DockerInputLocation); + string fileName = Path.GetFileName(uri.LocalPath); + var sourceBucket = aSubmission.Project.SubmissionBucket; + var subProj = _dbContext.Projects.Where(x => x.SubmissionProjectId == aSubmission.Project.Id); + foreach (var proj in subProj) + { + var destinationBucket = proj.SubmissionBucketTre; + var copyResult = _minioHelper.CopyObject(_minioSettings, sourceBucket, destinationBucket, fileName, fileName); + } + } + catch (Exception ex) + { + + throw; + } + // The TES message var tesMessage = JsonConvert.DeserializeObject(aSubmission.TesJson); var processedOK = true;