Skip to content

Commit

Permalink
ddcore-9261: new shelf access methods added
Browse files Browse the repository at this point in the history
  • Loading branch information
Ya.komarov committed Nov 15, 2024
1 parent 1eb647b commit fcd5e50
Show file tree
Hide file tree
Showing 6 changed files with 355 additions and 7 deletions.
19 changes: 19 additions & 0 deletions src/ComDiadocApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2052,16 +2052,35 @@ public BoxCounteragentEventList GetCounteragentEvents(

#region Shelf

[Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")]
public string UploadFileToShelf(string authToken, string fileName)
{
return diadoc.UploadFileToShelf(authToken, File.ReadAllBytes(fileName));
}

public string UploadFileToShelfV2(string authToken, string fileName)
{
var fileExtension = Path.GetExtension(fileName);
return diadoc.UploadFileToShelfV2(authToken, File.ReadAllBytes(fileName), fileExtension);
}

public string UploadLargeFileToShelf(string authToken, string fileName)
{
var fileExtension = Path.GetExtension(fileName);
return diadoc.UploadLargeFileToShelf(authToken, File.ReadAllBytes(fileName), fileExtension);
}

[Obsolete("Use GetFileFromShelfV2")]
public void GetFileFromShelf(string authToken, string nameOnShelf, string fileName)
{
File.WriteAllBytes(fileName, diadoc.GetFileFromShelf(authToken, nameOnShelf));
}

public void GetFileFromShelfV2(string authToken, string nameOnShelf, string fileName)
{
File.WriteAllBytes(fileName, diadoc.GetFileFromShelfV2(authToken, nameOnShelf));
}

#endregion

#region Parse...
Expand Down
23 changes: 23 additions & 0 deletions src/DiadocApi.Async.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -862,20 +862,43 @@ public Task<BoxCounteragentEventList> GetCounteragentEventsAsync(
return diadocHttpApi.GetCounteragentEventsAsync(authToken, boxId, afterIndexKey, timestampFromTicks, timestampToTicks, limit);
}

[Obsolete("Use UploadFileToShelfV2Async or UploadLargeFileToShelfAsync")]
public Task<string> UploadFileToShelfAsync(string authToken, byte[] data)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (data == null) throw new ArgumentNullException("data");
return diadocHttpApi.UploadFileToShelfAsync(authToken, data);
}

public Task<string> UploadFileToShelfV2Async(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (content == null) throw new ArgumentNullException("content");
return diadocHttpApi.UploadFileToShelfV2Async(authToken, content, fileExtension);
}

public Task<string> UploadLargeFileToShelfAsync(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (content == null) throw new ArgumentNullException("content");
return diadocHttpApi.UploadLargeFileToShelfAsync(authToken, content, fileExtension);
}

[Obsolete("Use GetFileFromShelfV2Async")]
public Task<byte[]> GetFileFromShelfAsync(string authToken, string nameOnShelf)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (string.IsNullOrEmpty(nameOnShelf)) throw new ArgumentNullException("nameOnShelf");
return diadocHttpApi.GetFileFromShelfAsync(authToken, nameOnShelf);
}

public Task<byte[]> GetFileFromShelfV2Async(string authToken, string fileName)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("fileName");
return diadocHttpApi.GetFileFromShelfV2Async(authToken, fileName);
}

public Task<RussianAddress> ParseRussianAddressAsync(string address)
{
return diadocHttpApi.ParseRussianAddressAsync(address);

Check warning on line 904 in src/DiadocApi.Async.cs

View workflow job for this annotation

GitHub Actions / build_and_test 

'DiadocHttpApi.ParseRussianAddressAsync(string)' is obsolete: 'Use overload with authToken argument'
Expand Down
23 changes: 23 additions & 0 deletions src/DiadocApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -998,20 +998,43 @@ public BoxCounteragentEventList GetCounteragentEvents(
return diadocHttpApi.GetCounteragentEvents(authToken, boxId, afterIndexKey, timestampFromTicks, timestampToTicks, limit);
}

[Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")]
public string UploadFileToShelf(string authToken, byte[] data)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (data == null) throw new ArgumentNullException("data");
return diadocHttpApi.UploadFileToShelf(authToken, data);
}

public string UploadFileToShelfV2(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (content == null) throw new ArgumentNullException("content");
return diadocHttpApi.UploadFileToShelfV2(authToken, content, fileExtension);
}

public string UploadLargeFileToShelf(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (content == null) throw new ArgumentNullException("content");
return diadocHttpApi.UploadLargeFileToShelf(authToken, content, fileExtension);
}

[Obsolete("Use GetFileFromShelfV2")]
public byte[] GetFileFromShelf(string authToken, string nameOnShelf)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (string.IsNullOrEmpty(nameOnShelf)) throw new ArgumentNullException("nameOnShelf");
return diadocHttpApi.GetFileFromShelf(authToken, nameOnShelf);
}

public byte[] GetFileFromShelfV2(string authToken, string fileName)
{
if (string.IsNullOrEmpty(authToken)) throw new ArgumentNullException("authToken");
if (string.IsNullOrEmpty(fileName)) throw new ArgumentNullException("fileName");
return diadocHttpApi.GetFileFromShelfV2(authToken, fileName);
}

public RussianAddress ParseRussianAddress(string address)
{
return diadocHttpApi.ParseRussianAddress(address);
Expand Down
142 changes: 138 additions & 4 deletions src/DiadocHttpApi.Shelf.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public partial class DiadocHttpApi
{
private const int partLength = 512 * 1024;
private const int maxAttempts = 3;

private readonly HttpStatusCode[] nonRetriableStatusCodes =
{
HttpStatusCode.OK,
Expand All @@ -20,9 +21,10 @@ public partial class DiadocHttpApi
HttpStatusCode.PaymentRequired
};

public int ShelfUploadChunkSize { get { return partLength; } }
public int ShelfUploadMaxAttemptsCount { get { return maxAttempts; } }
public int ShelfUploadChunkSize => partLength;
public int ShelfUploadMaxAttemptsCount => maxAttempts;

[Obsolete("Use UploadFileToShelfV2 or UploadLargeFileToShelf")]
public string UploadFileToShelf(string authToken, byte[] data)
{
var nameOnShelf = string.Format("api-{0}", Guid.NewGuid());
Expand All @@ -31,7 +33,7 @@ public string UploadFileToShelf(string authToken, byte[] data)
var httpErrors = new List<HttpClientException>();
var attempts = 0;
var missingParts = Enumerable.Range(0, parts.Count).ToArray();
while(missingParts.Length > 0)
while (missingParts.Length > 0)
{
if (++attempts > maxAttempts)
throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray());
Expand All @@ -41,6 +43,130 @@ public string UploadFileToShelf(string authToken, byte[] data)
return nameOnShelf;
}

public string UploadFileToShelfV2(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
var httpErrors = new List<HttpClientException>();
var queryString = string.Format("V2/ShelfUpload?fileExtension={0}", fileExtension);

for (var i = 0; i < maxAttempts; i++)
{
try
{
var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(content));
var response = HttpClient.PerformHttpRequest(request);
return Encoding.UTF8.GetString(response.Content);
}
catch (HttpClientException e)
{
if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw;
httpErrors.Add(e);
}
}

throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray());
}

public string UploadLargeFileToShelf(string authToken, byte[] content, [CanBeNull] string fileExtension)
{
var parts = SplitDataIntoParts(content).ToList();

var httpErrors = new List<HttpClientException>();
var attempts = 0;
var missingParts = Enumerable.Range(0, parts.Count).ToList();
string fileName = null;

while (missingParts.Count > 0)
{
if (++attempts > maxAttempts)
{
throw new AggregateException("Reached the limit of attempts to send a file", httpErrors.ToArray());
}

if (fileName == null)
{
fileName = ShelfUploadPartInit(authToken, parts[0], missingParts, httpErrors, fileExtension);
}

if (fileName != null)
{
missingParts = ShelfUploadParts(authToken, fileName, parts, missingParts, httpErrors);
}
}

return fileName;
}

[CanBeNull]
private string ShelfUploadPartInit(string authToken, ArraySegment<byte> firstPart, [NotNull] IList<int> missingParts, [NotNull] ICollection<HttpClientException> httpErrors, [CanBeNull] string fileExtension)
{
var queryString = string.Format("ShelfUploadPartInit?fileExtension={0}", fileExtension);
if (missingParts.Count == 1)
{
queryString += "&isLastPart=true";
}

try
{
var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(firstPart));
var response = HttpClient.PerformHttpRequest(request);
missingParts.Remove(0);

return Encoding.UTF8.GetString(response.Content);
}
catch (HttpClientException e)
{
if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw;
httpErrors.Add(e);
}

return null;
}

[NotNull]
private List<int> ShelfUploadParts(string authToken, string fileName, [NotNull] IList<ArraySegment<byte>> parts, [NotNull] IList<int> missingParts, [NotNull] ICollection<HttpClientException> httpErrors)
{
var currentMissingParts = missingParts.Count == 0 ? new List<int>() : null;
for (var i = 0; i < missingParts.Count; ++i)
{
var partIndex = missingParts[i];
currentMissingParts = UploadPart(authToken, fileName, parts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors);
}

if (currentMissingParts == null)
{
throw new Exception("UploadPart did not return missing parts");
}

return currentMissingParts;
}

[CanBeNull]
private List<int> UploadPart(string authToken, string fileName, ArraySegment<byte> part, int partIndex, bool isLast, ICollection<HttpClientException> httpErrors)
{
var queryString = string.Format("ShelfUploadPart?fileName={0}&partIndex={1}&isLastPart={2}", fileName, partIndex, isLast);
try
{
var request = BuildRequest(authToken, "POST", queryString, new HttpRequestBody(part));
var response = HttpClient.PerformHttpRequest(request);
if (isLast)
{
var responseString = Encoding.UTF8.GetString(response.Content);
return responseString
.Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries)
.Select(int.Parse)
.ToList();
}

return null;
}
catch (HttpClientException e)
{
if (e.ResponseStatusCode.HasValue && nonRetriableStatusCodes.Contains(e.ResponseStatusCode.Value)) throw;
httpErrors.Add(e);
return isLast ? new List<int> { partIndex } : null;
}
}

[NotNull]
private int[] PutMissingParts(string authToken, string nameOnShelf, [NotNull] IList<ArraySegment<byte>> allParts, [NotNull] IList<int> missingParts, [NotNull] ICollection<HttpClientException> httpErrors)
{
Expand All @@ -50,11 +176,13 @@ private int[] PutMissingParts(string authToken, string nameOnShelf, [NotNull] IL
var partIndex = missingParts[i];
currentMissingParts = PutPart(authToken, nameOnShelf, allParts[partIndex], partIndex, i == missingParts.Count - 1, httpErrors);
}

if (currentMissingParts == null)
throw new Exception("ShelfUpload did not return missing parts");
return currentMissingParts;
}

[Obsolete("Use GetFileFromShelfV2")]
public byte[] GetFileFromShelf(string authToken, string nameOnShelf)
{
if (!nameOnShelf.Contains("__userId__"))
Expand All @@ -63,6 +191,12 @@ public byte[] GetFileFromShelf(string authToken, string nameOnShelf)
return PerformHttpRequest(authToken, "GET", queryString);
}

public byte[] GetFileFromShelfV2(string authToken, string fileName)
{
var queryString = $"V2/ShelfDownload?fileName={fileName}";
return PerformHttpRequest(authToken, "GET", queryString);
}

[CanBeNull]
private int[] PutPart(string authToken, string nameOnShelf, ArraySegment<byte> part, int partIndex, bool isLast, ICollection<HttpClientException> httpErrors)
{
Expand All @@ -76,7 +210,7 @@ private int[] PutPart(string authToken, string nameOnShelf, ArraySegment<byte> p
if (isLast)
{
var responseString = Encoding.UTF8.GetString(response.Content);
var result = responseString.Split(new[] {',', '[', ']'}, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray();
var result = responseString.Split(new[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToArray();
return result;
}
return null;
Expand Down
Loading

0 comments on commit fcd5e50

Please sign in to comment.