From fcd5e509086c6e48905d5f61fd5d71dd32079550 Mon Sep 17 00:00:00 2001 From: "Ya.komarov" Date: Mon, 11 Nov 2024 16:37:47 +0500 Subject: [PATCH] ddcore-9261: new shelf access methods added --- src/ComDiadocApi.cs | 19 +++++ src/DiadocApi.Async.cs | 23 +++++ src/DiadocApi.cs | 23 +++++ src/DiadocHttpApi.Shelf.cs | 142 ++++++++++++++++++++++++++++++- src/DiadocHttpApi.ShelfAsync.cs | 145 +++++++++++++++++++++++++++++++- src/IDiadocApi.cs | 10 +++ 6 files changed, 355 insertions(+), 7 deletions(-) diff --git a/src/ComDiadocApi.cs b/src/ComDiadocApi.cs index d8010353..5cbb341f 100644 --- a/src/ComDiadocApi.cs +++ b/src/ComDiadocApi.cs @@ -2052,16 +2052,35 @@ public BoxCounteragentEventList GetCounteragentEvents( #region Shelf + [Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")] public string UploadFileToShelf(string authToken, string fileName) { return diadoc.UploadFileToShelf(authToken, File.ReadAllBytes(fileName)); } + public string UploadFileToShelfV2(string authToken, string fileName) + { + var fileExtension = Path.GetExtension(fileName); + return diadoc.UploadFileToShelfV2(authToken, File.ReadAllBytes(fileName), fileExtension); + } + + public string UploadLargeFileToShelf(string authToken, string fileName) + { + var fileExtension = Path.GetExtension(fileName); + return diadoc.UploadLargeFileToShelf(authToken, File.ReadAllBytes(fileName), fileExtension); + } + + [Obsolete("Use GetFileFromShelfV2")] public void GetFileFromShelf(string authToken, string nameOnShelf, string fileName) { File.WriteAllBytes(fileName, diadoc.GetFileFromShelf(authToken, nameOnShelf)); } + public void GetFileFromShelfV2(string authToken, string nameOnShelf, string fileName) + { + File.WriteAllBytes(fileName, diadoc.GetFileFromShelfV2(authToken, nameOnShelf)); + } + #endregion #region Parse... diff --git a/src/DiadocApi.Async.cs b/src/DiadocApi.Async.cs index 5a0d536f..20b69d2e 100644 --- a/src/DiadocApi.Async.cs +++ b/src/DiadocApi.Async.cs @@ -862,6 +862,7 @@ public Task GetCounteragentEventsAsync( return diadocHttpApi.GetCounteragentEventsAsync(authToken, boxId, afterIndexKey, timestampFromTicks, timestampToTicks, limit); } + [Obsolete("Use UploadFileToShelfV2Async or UploadLargeFileToShelfAsync")] public Task UploadFileToShelfAsync(string authToken, byte[] data) { if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); @@ -869,6 +870,21 @@ public Task UploadFileToShelfAsync(string authToken, byte[] data) return diadocHttpApi.UploadFileToShelfAsync(authToken, data); } + public Task UploadFileToShelfV2Async(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (content == null) throw new ArgumentNullException("content"); + return diadocHttpApi.UploadFileToShelfV2Async(authToken, content, fileExtension); + } + + public Task UploadLargeFileToShelfAsync(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (content == null) throw new ArgumentNullException("content"); + return diadocHttpApi.UploadLargeFileToShelfAsync(authToken, content, fileExtension); + } + + [Obsolete("Use GetFileFromShelfV2Async")] public Task GetFileFromShelfAsync(string authToken, string nameOnShelf) { if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); @@ -876,6 +892,13 @@ public Task GetFileFromShelfAsync(string authToken, string nameOnShelf) return diadocHttpApi.GetFileFromShelfAsync(authToken, nameOnShelf); } + public Task GetFileFromShelfV2Async(string authToken, string fileName) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("fileName"); + return diadocHttpApi.GetFileFromShelfV2Async(authToken, fileName); + } + public Task ParseRussianAddressAsync(string address) { return diadocHttpApi.ParseRussianAddressAsync(address); diff --git a/src/DiadocApi.cs b/src/DiadocApi.cs index 6a5f5590..b962dc12 100644 --- a/src/DiadocApi.cs +++ b/src/DiadocApi.cs @@ -998,6 +998,7 @@ public BoxCounteragentEventList GetCounteragentEvents( return diadocHttpApi.GetCounteragentEvents(authToken, boxId, afterIndexKey, timestampFromTicks, timestampToTicks, limit); } + [Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")] public string UploadFileToShelf(string authToken, byte[] data) { if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); @@ -1005,6 +1006,21 @@ public string UploadFileToShelf(string authToken, byte[] data) return diadocHttpApi.UploadFileToShelf(authToken, data); } + public string UploadFileToShelfV2(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (content == null) throw new ArgumentNullException("content"); + return diadocHttpApi.UploadFileToShelfV2(authToken, content, fileExtension); + } + + public string UploadLargeFileToShelf(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (content == null) throw new ArgumentNullException("content"); + return diadocHttpApi.UploadLargeFileToShelf(authToken, content, fileExtension); + } + + [Obsolete("Use GetFileFromShelfV2")] public byte[] GetFileFromShelf(string authToken, string nameOnShelf) { if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); @@ -1012,6 +1028,13 @@ public byte[] GetFileFromShelf(string authToken, string nameOnShelf) return diadocHttpApi.GetFileFromShelf(authToken, nameOnShelf); } + public byte[] GetFileFromShelfV2(string authToken, string fileName) + { + if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken"); + if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("fileName"); + return diadocHttpApi.GetFileFromShelfV2(authToken, fileName); + } + public RussianAddress ParseRussianAddress(string address) { return diadocHttpApi.ParseRussianAddress(address); diff --git a/src/DiadocHttpApi.Shelf.cs b/src/DiadocHttpApi.Shelf.cs index 2216a24d..d1eee9d0 100644 --- a/src/DiadocHttpApi.Shelf.cs +++ b/src/DiadocHttpApi.Shelf.cs @@ -12,6 +12,7 @@ public partial class DiadocHttpApi { private const int partLength = 512 * 1024; private const int maxAttempts = 3; + private readonly HttpStatusCode[] nonRetriableStatusCodes = { HttpStatusCode.OK, @@ -20,9 +21,10 @@ public partial class DiadocHttpApi HttpStatusCode.PaymentRequired }; - public int ShelfUploadChunkSize { get { return partLength; } } - public int ShelfUploadMaxAttemptsCount { get { return maxAttempts; } } + public int ShelfUploadChunkSize => partLength; + public int ShelfUploadMaxAttemptsCount => maxAttempts; + [Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")] public string UploadFileToShelf(string authToken, byte[] data) { var nameOnShelf = string.Format("api-{0}", Guid.NewGuid()); @@ -31,7 +33,7 @@ public string UploadFileToShelf(string authToken, byte[] data) var httpErrors = new List(); var attempts = 0; var missingParts = Enumerable.Range(0, parts.Count).ToArray(); - while(missingParts.Length > 0) + while (missingParts.Length > 0) { if (++attempts > maxAttempts) throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); @@ -41,6 +43,130 @@ public string UploadFileToShelf(string authToken, byte[] data) return nameOnShelf; } + public string UploadFileToShelfV2(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + var httpErrors = new List(); + var queryString = string.Format("V2/ShelfUpload?fileExtension={0}", fileExtension); + + for (var i = 0; i < maxAttempts; i++) + { + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(content)); + var response = HttpClient.PerformHttpRequest(request); + return Encoding.UTF8.GetString(response.Content); + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + } + } + + throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); + } + + public string UploadLargeFileToShelf(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + var parts = SplitDataIntoParts(content).ToList(); + + var httpErrors = new List(); + var attempts = 0; + var missingParts = Enumerable.Range(0, parts.Count).ToList(); + string fileName = null; + + while (missingParts.Count > 0) + { + if (++attempts > maxAttempts) + { + throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); + } + + if (fileName == null) + { + fileName = ShelfUploadPartInit(authToken, parts[0], missingParts, httpErrors, fileExtension); + } + + if (fileName != null) + { + missingParts = ShelfUploadParts(authToken, fileName, parts, missingParts, httpErrors); + } + } + + return fileName; + } + + [CanBeNull] + private string ShelfUploadPartInit(string authToken, ArraySegment firstPart, [NotNull] IList missingParts, [NotNull] ICollection httpErrors, [CanBeNull] string fileExtension) + { + var queryString = string.Format("ShelfUploadPartInit?fileExtension={0}", fileExtension); + if (missingParts.Count == 1) + { + queryString += "&isLastPart=true"; + } + + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(firstPart)); + var response = HttpClient.PerformHttpRequest(request); + missingParts.Remove(0); + + return Encoding.UTF8.GetString(response.Content); + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + } + + return null; + } + + [NotNull] + private List ShelfUploadParts(string authToken, string fileName, [NotNull] IList> parts, [NotNull] IList missingParts, [NotNull] ICollection httpErrors) + { + var currentMissingParts = missingParts.Count == 0 ? new List() : null; + for (var i = 0; i < missingParts.Count; ++i) + { + var partIndex = missingParts[i]; + currentMissingParts = UploadPart(authToken, fileName, parts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors); + } + + if (currentMissingParts == null) + { + throw new Exception("UploadPart did not return missing parts"); + } + + return currentMissingParts; + } + + [CanBeNull] + private List UploadPart(string authToken, string fileName, ArraySegment part, int partIndex, bool isLast, ICollection httpErrors) + { + var queryString = string.Format("ShelfUploadPart?fileName={0}&partIndex={1}&isLastPart={2}", fileName, partIndex, isLast); + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(part)); + var response = HttpClient.PerformHttpRequest(request); + if (isLast) + { + var responseString = Encoding.UTF8.GetString(response.Content); + return responseString + .Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries) + .Select(int.Parse) + .ToList(); + } + + return null; + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + return isLast ? new List { partIndex } : null; + } + } + [NotNull] private int[] PutMissingParts(string authToken, string nameOnShelf, [NotNull] IList> allParts, [NotNull] IList missingParts, [NotNull] ICollection httpErrors) { @@ -50,11 +176,13 @@ private int[] PutMissingParts(string authToken, string nameOnShelf, [NotNull] IL var partIndex = missingParts[i]; currentMissingParts = PutPart(authToken, nameOnShelf, allParts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors); } + if (currentMissingParts == null) throw new Exception("ShelfUpload did not return missing parts"); return currentMissingParts; } + [Obsolete("Use GetFileFromShelfV2")] public byte[] GetFileFromShelf(string authToken, string nameOnShelf) { if (!nameOnShelf.Contains("__userId__")) @@ -63,6 +191,12 @@ public byte[] GetFileFromShelf(string authToken, string nameOnShelf) return PerformHttpRequest(authToken, "GET", queryString); } + public byte[] GetFileFromShelfV2(string authToken, string fileName) + { + var queryString = $"V2/ShelfDownload?fileName={fileName}"; + return PerformHttpRequest(authToken, "GET", queryString); + } + [CanBeNull] private int[] PutPart(string authToken, string nameOnShelf, ArraySegment part, int partIndex, bool isLast, ICollection httpErrors) { @@ -76,7 +210,7 @@ private int[] PutPart(string authToken, string nameOnShelf, ArraySegment p if (isLast) { var responseString = Encoding.UTF8.GetString(response.Content); - var result = responseString.Split(new[] {',', '[', ']'}, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray(); + var result = responseString.Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray(); return result; } return null; diff --git a/src/DiadocHttpApi.ShelfAsync.cs b/src/DiadocHttpApi.ShelfAsync.cs index 20c9dc59..8dfd3938 100644 --- a/src/DiadocHttpApi.ShelfAsync.cs +++ b/src/DiadocHttpApi.ShelfAsync.cs @@ -10,6 +10,7 @@ namespace Diadoc.Api { public partial class DiadocHttpApi { + [Obsolete("Use UploadFileToShelfV2Async or UploadLargeFileToShelfAsync")] public async Task UploadFileToShelfAsync(string authToken, byte[] data) { var nameOnShelf = $"api-{Guid.NewGuid()}"; @@ -18,7 +19,7 @@ public async Task UploadFileToShelfAsync(string authToken, byte[] data) var httpErrors = new List(); var attempts = 0; var missingParts = Enumerable.Range(0, parts.Count).ToArray(); - while(missingParts.Length > 0) + while (missingParts.Length > 0) { if (++attempts > maxAttempts) throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); @@ -28,6 +29,132 @@ public async Task UploadFileToShelfAsync(string authToken, byte[] data) return nameOnShelf; } + public async Task UploadFileToShelfV2Async(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + var httpErrors = new List(); + var queryString = string.Format("V2/ShelfUpload?fileExtension={0}", fileExtension); + + for (var i = 0; i < maxAttempts; i++) + { + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(content)); + var response = await HttpClient.PerformHttpRequestAsync(request).ConfigureAwait(false); + return Encoding.UTF8.GetString(response.Content); + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + } + } + + throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); + } + + public async Task UploadLargeFileToShelfAsync(string authToken, byte[] content, [CanBeNull] string fileExtension) + { + var parts = SplitDataIntoParts(content).ToList(); + + var httpErrors = new List(); + var attempts = 0; + var missingParts = Enumerable.Range(0, parts.Count).ToList(); + string fileName = null; + + while (missingParts.Count > 0) + { + if (++attempts > maxAttempts) + { + throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray()); + } + + if (fileName == null) + { + fileName = await ShelfUploadPartInitAsync(authToken, parts[0], missingParts, httpErrors, fileExtension).ConfigureAwait(false); + } + + if (fileName != null) + { + missingParts = await ShelfUploadPartsAsync(authToken, fileName, parts, missingParts, httpErrors) + .ConfigureAwait(false); + } + } + + return fileName; + } + + [ItemCanBeNull] + private async Task ShelfUploadPartInitAsync(string authToken, ArraySegment firstPart, [NotNull] IList missingParts, [NotNull] ICollection httpErrors, [CanBeNull] string fileExtension) + { + var queryString = string.Format("ShelfUploadPartInit?fileExtension={0}", fileExtension); + if (missingParts.Count == 1) + { + queryString += "&isLastPart=true"; + } + + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(firstPart)); + var response = await HttpClient.PerformHttpRequestAsync(request).ConfigureAwait(false); + missingParts.Remove(0); + + return Encoding.UTF8.GetString(response.Content); + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + } + + return null; + } + + [ItemNotNull] + private async Task> ShelfUploadPartsAsync(string authToken, string fileName, [NotNull] IList> parts, [NotNull] IList missingParts, [NotNull] ICollection httpErrors) + { + var currentMissingParts = missingParts.Count == 0 ? new List() : null; + for (var i = 0; i < missingParts.Count; ++i) + { + var partIndex = missingParts[i]; + currentMissingParts = await UploadPartAsync(authToken, fileName, parts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors) + .ConfigureAwait(false); + } + + if (currentMissingParts == null) + { + throw new Exception("UploadPartAsync did not return missing parts"); + } + + return currentMissingParts; + } + + [ItemCanBeNull] + private async Task> UploadPartAsync(string authToken, string fileName, ArraySegment part, int partIndex, bool isLast, ICollection httpErrors) + { + var queryString = string.Format("ShelfUploadPart?fileName={0}&partIndex={1}&isLastPart={2}", fileName, partIndex, isLast); + try + { + var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(part)); + var response = await HttpClient.PerformHttpRequestAsync(request).ConfigureAwait(false); + if (isLast) + { + var responseString = Encoding.UTF8.GetString(response.Content); + return responseString + .Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries) + .Select(int.Parse) + .ToList(); + } + + return null; + } + catch (HttpClientException e) + { + if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw; + httpErrors.Add(e); + return isLast ? new List { partIndex } : null; + } + } + [ItemNotNull] private async Task PutMissingPartsAsync(string authToken, string nameOnShelf, [NotNull] IList> allParts, [NotNull] IList missingParts, [NotNull] ICollection httpErrors) { @@ -37,11 +164,16 @@ private async Task PutMissingPartsAsync(string authToken, string nameOnSh var partIndex = missingParts[i]; currentMissingParts = await PutPartAsync(authToken, nameOnShelf, allParts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors).ConfigureAwait(false); } + if (currentMissingParts == null) - throw new Exception("ShelfUpload did not return missing parts"); + { + throw new Exception("ShelfUpload did not return missing parts"); + } + return currentMissingParts; } + [Obsolete("Use GetFileFromShelfV2Async")] public Task GetFileFromShelfAsync(string authToken, string nameOnShelf) { if (!nameOnShelf.Contains("__userId__")) @@ -50,6 +182,12 @@ public Task GetFileFromShelfAsync(string authToken, string nameOnShelf) return PerformHttpRequestAsync(authToken, "GET", queryString); } + public Task GetFileFromShelfV2Async(string authToken, string fileName) + { + var queryString = $"V2/ShelfDownload?fileName={fileName}"; + return PerformHttpRequestAsync(authToken, "GET", queryString); + } + [ItemCanBeNull] private async Task PutPartAsync(string authToken, string nameOnShelf, ArraySegment part, int partIndex, bool isLast, ICollection httpErrors) { @@ -63,9 +201,10 @@ private async Task PutPartAsync(string authToken, string nameOnShelf, Arr if (isLast) { var responseString = Encoding.UTF8.GetString(response.Content); - var result = responseString.Split(new[] {',', '[', ']'}, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray(); + var result = responseString.Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray(); return result; } + return null; } catch (HttpClientException e) diff --git a/src/IDiadocApi.cs b/src/IDiadocApi.cs index 6e6e4c81..8f3f615d 100644 --- a/src/IDiadocApi.cs +++ b/src/IDiadocApi.cs @@ -198,8 +198,13 @@ BoxCounteragentEventList GetCounteragentEvents( long? timestampFromTicks = null, long? timestampToTicks = null, int? limit = null); + [Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")] string UploadFileToShelf(string authToken, byte[] data); + string UploadFileToShelfV2(string authToken, byte[] content, [CanBeNull] string fileExtension); + string UploadLargeFileToShelf(string authToken, byte[] content, [CanBeNull] string fileExtension); + [Obsolete("Use GetFileFromShelfV2")] byte[] GetFileFromShelf(string authToken, string nameOnShelf); + byte[] GetFileFromShelfV2(string authToken, string fileName); [Obsolete(ObsoleteReasons.UseAuthTokenOverload)] RussianAddress ParseRussianAddress(string address); RussianAddress ParseRussianAddress(string authToken, string address); @@ -572,8 +577,13 @@ Task GetCounteragentEventsAsync( long? timestampFromTicks = null, long? timestampToTicks = null, int? limit = null); + [Obsolete("Use UploadFileToShelfV2Async or UploadLargeFileToShelfAsync")] Task UploadFileToShelfAsync(string authToken, byte[] data); + Task UploadFileToShelfV2Async(string authToken, byte[] content, [CanBeNull] string fileExtension); + Task UploadLargeFileToShelfAsync(string authToken, byte[] content, [CanBeNull] string fileExtension); + [Obsolete("Use GetFileFromShelfV2Async")] Task GetFileFromShelfAsync(string authToken, string nameOnShelf); + Task GetFileFromShelfV2Async(string authToken, string fileName); [Obsolete(ObsoleteReasons.UseAuthTokenOverload)] Task ParseRussianAddressAsync(string address); Task ParseRussianAddressAsync(string authToken, string address);