Skip to content

Commit

Permalink
Update implementation from production code
Browse files Browse the repository at this point in the history
  • Loading branch information
alanta committed Feb 28, 2024
1 parent 6d9eeec commit 395a8b4
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 49 deletions.
32 changes: 16 additions & 16 deletions Kontent.Statiq/KontentDownloadImages.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,44 +32,44 @@ protected override async Task<IEnumerable<IDocument>> ExecuteContextAsync(IExecu
context.GetContentProvider(_cached[asset.LocalPath.ToString()].Data, _cached[asset.LocalPath.ToString()].MediaType)
)).ToList();

if (newAssets.Length != assets.Length)
if( newAssets.Length == 0 )
{
context.LogInformation(null, $"Downloading {newAssets.Length} files, skipping {assets.Length - newAssets.Length} already downloaded.");
context.LogInformation(null, $"Skipping image download because there are no new images.");
}

if (newAssets.Length > 0)
else if( newAssets.Length != assets.Length )
{
await DownloadNewAssets(context, newAssets, downloadsWithDestination);
context.LogInformation(null, $"Downloading {newAssets.Length} files, skipping {assets.Length-newAssets.Length} already downloaded.");
}

return downloadsWithDestination;
}
var childModules = newAssets.Select(a => a.OriginalUrl).Chunk(20).Select( x => new ReadWeb(x.ToArray()) );


private async Task DownloadNewAssets(IExecutionContext context, KontentImageDownload[] assets, List<IDocument> downloadsWithDestination)
{
var downloads = new List<IDocument>();

// Workaround for unlimited concurrency in ReadWeb. By fetching chunks of 20 images we prevent timeouts
// caused by flooding the Kontent Delivery API with 100s of concurrent requests.
foreach (var batch in assets.Select(a => a.OriginalUrl).Chunk(20))
foreach (var module in childModules)
{
var module = new ReadWeb(batch);
var documents = await module.ExecuteAsync(context);
var documents = await module!.ExecuteAsync(context);
downloads.AddRange(documents);
}

foreach (var download in downloads)
{
var downloadedUrl = download.Get<string>(Keys.SourceUri);
var asset = assets.FirstOrDefault(a => a.OriginalUrl == downloadedUrl);

Check warning on line 60 in Kontent.Statiq/KontentDownloadImages.cs

View workflow job for this annotation

GitHub Actions / build

"Find" method should be used instead of the "FirstOrDefault" extension method. (https://rules.sonarsource.com/csharp/RSPEC-6602)
if (asset == null)
if (asset != null)
{
_cached[asset.LocalPath.ToString()]=new CachedImage(Data: await download.GetContentBytesAsync(), MediaType: download.ContentProvider.MediaType);
downloadsWithDestination.Add(download.Clone(destination: asset.LocalPath.ToString().ToLower().TrimStart('/')));
}
else
{
throw new InvalidOperationException($"No asset found for url {downloadedUrl}");
}

_cached[asset.LocalPath.ToString()] = new CachedImage(Data: await download.GetContentBytesAsync(), MediaType: download.ContentProvider.MediaType);
downloadsWithDestination.Add(download.Clone(destination: asset.LocalPath.ToString().ToLower().TrimStart('/')));
}

return downloadsWithDestination;
}
}
}
33 changes: 0 additions & 33 deletions Kontent.Statiq/LinqExtensions.cs

This file was deleted.

0 comments on commit 395a8b4

Please sign in to comment.