Object storage v2 beta controller

.vscode/settings.json

@@ -2,6 +2,7 @@
     "cSpell.words": [
         "Finalizer",
         "maxiops",
+        "objectstorages",
         "Peerings"
     ]
 }

README.md

@@ -11,7 +11,7 @@ Always refer to the official UpCloud documentation and support channels for auth
 - [ ] Implement UpCloud API objects as custom resource types.
 - [ ] To keep up with UpCloud API object states by pulling changes from the UpCloud API.
 - [ ] To be able to display the current UpCloud API object states via Kubernetes API. 
-- [ ] To be able to maintain the desired state of the UpCloud API objects.
+- [ ] To be able to maintain the desired state of the UpCloud API objects (manipulate UpCloud API objects via K8s entity spec).
 
 These features can be implemented one by one, and the features can be implemented only for selected UpCloud API objects.
 

src/UpcloudApiKubernetesOperator/AutoLoader/AutoLoaderService.cs

@@ -0,0 +1,69 @@
+using System.Collections.ObjectModel;
+using Microsoft.Extensions.Options;
+using UpcloudApiKubernetesOperator.AutoLoader.Loaders;
+using UpcloudApiKubernetesOperator.AutoLoader.Options;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader;
+
+internal class AutoLoaderService : IHostedService
+{
+    private readonly ReadOnlyCollection<ILoader> Loaders;
+    private readonly ILogger Logger;
+    private readonly AutoLoaderOptions Options;
+    private Task? Runner;
+    private readonly CancellationTokenSource CancellationTokenSource = new();
+
+    public AutoLoaderService(
+        ObjectStorageV2Loader objectStorageV2Loader,
+        IOptions<AutoLoaderOptions> options,
+        ILogger<AutoLoaderService> logger
+    ) {
+        Loaders = new (
+            new ILoader[] {
+                objectStorageV2Loader
+            }
+        );
+        Options = options.Value;
+        Logger  = logger;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        Runner = RunLoaders();
+        Logger.LogInformation("UpCloud entity auto loader enabled (interval: {refreshInterval})", Options.RefreshInterval);
+        return Task.CompletedTask;
+    }
+
+    private async Task RunLoaders()
+    {
+        var interval = TimeSpan.FromSeconds(Options.RefreshInterval);
+
+        while (CancellationTokenSource.IsCancellationRequested is false) {
+            await Task.Delay(interval, cancellationToken: CancellationTokenSource.Token);
+
+            foreach (var loader in Loaders) {
+                try {
+                    await loader.Run(CancellationTokenSource.Token);
+                }
+                catch (Exception ex) {
+                    Logger.LogError(ex, "Executing loader failed (type: {loaderType})", loader.GetType().Name);
+                }
+            }
+        }
+    }
+
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        CancellationTokenSource.Cancel();
+
+        if (Runner is not null && Runner.IsCompleted is false) {
+            try {
+                await Runner;
+            }
+            catch (TaskCanceledException) { }
+            Runner = null;
+        }
+
+        CancellationTokenSource.Dispose();
+    }
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Extensions/DependencyInjection/IServiceCollectionExtensions.cs

@@ -0,0 +1,21 @@
+using Microsoft.Extensions.Options;
+
+using UpcloudApiKubernetesOperator.AutoLoader.Options;
+using UpcloudApiKubernetesOperator.AutoLoader.Loaders.Extensions.DependencyInjection;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader.Extensions.DependencyInjection;
+
+public static class IServiceCollectionExtensions
+{
+    public static IServiceCollection AddAutoLoaderService(this IServiceCollection services, IConfiguration configuration) =>
+        configuration.GetRequiredSection(key: AutoLoaderOptions.DEFAULT_SECTION_NAME) switch {
+            var autoLoaderConfig => autoLoaderConfig.GetValue("Enabled", defaultValue: false) switch {
+                false => services,
+                true  => services
+                    .Configure<AutoLoaderOptions>(config: autoLoaderConfig)
+                    .AddSingleton<IValidateOptions<AutoLoaderOptions>, AutoLoaderOptions.Validator>()
+                    .AddLoaders()
+                    .AddHostedService<AutoLoaderService>()
+            }
+        };
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Loaders/BaseLoader.cs

@@ -0,0 +1,17 @@
+using KubeOps.KubernetesClient;
+using Microsoft.Extensions.Options;
+using UpcloudApiKubernetesOperator.AutoLoader.Options;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader.Loaders;
+
+internal abstract class BaseLoader : ILoader
+{
+    protected readonly AutoLoaderOptions Options;
+    protected readonly IKubernetesClient KubernetesClient;
+    protected readonly ILogger Logger;
+
+    public BaseLoader(IOptions<AutoLoaderOptions> options, IKubernetesClient kubernetesClient, ILogger<AutoLoaderService> logger) =>
+        (Options, KubernetesClient, Logger) = (options.Value, kubernetesClient, logger);
+
+    public abstract Task Run(CancellationToken cancellationToken);
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Loaders/Extensions/DependencyInjection/IServiceCollectionExtensions.cs

@@ -0,0 +1,10 @@
+using Microsoft.Extensions.Options;
+using UpcloudApiKubernetesOperator.AutoLoader.Options;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader.Loaders.Extensions.DependencyInjection;
+
+public static class IServiceCollectionExtensions
+{
+    public static IServiceCollection AddLoaders(this IServiceCollection services) => services
+        .AddSingleton<ObjectStorageV2Loader>();
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Loaders/ILoader.cs

@@ -0,0 +1,6 @@
+namespace UpcloudApiKubernetesOperator.AutoLoader.Loaders;
+
+internal interface ILoader
+{
+    abstract Task Run(CancellationToken cancellationToken);
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Loaders/ObjectStorageV2Loader.cs

@@ -0,0 +1,134 @@
+using System.Collections.ObjectModel;
+
+using Microsoft.Extensions.Options;
+
+using k8s.Models;
+using k8s.Autorest;
+using KubeOps.KubernetesClient;
+
+using UpcloudApiKubernetesOperator.AutoLoader.Options;
+using UpcloudApiKubernetesOperator.Entities;
+using UpcloudApiKubernetesOperator.UpCloudApi.ObjectStorageV2;
+using UpcloudApiKubernetesOperator.UpCloudApi.ObjectStorageV2.Models.Responses;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader.Loaders;
+
+internal class ObjectStorageV2Loader : BaseLoader
+{
+    protected readonly IObjectStorageV2Client UpCApiClient;
+
+    public ObjectStorageV2Loader(IOptions<AutoLoaderOptions> options, IObjectStorageV2Client upcApiClient, IKubernetesClient kubernetesClient, ILogger<AutoLoaderService> logger)
+        : base(options, kubernetesClient, logger)
+        => UpCApiClient = upcApiClient;
+
+    public override async Task Run(CancellationToken cancellationToken)
+    {
+        Logger.LogDebug("Loading object storage instances from upc api");
+
+        var instancesFromApi = await UpCApiClient.ListInstances(cancellationToken: cancellationToken);
+
+        if (instancesFromApi is null || instancesFromApi.Count == 0) {
+            Logger.LogDebug("Loading object storage instances from upc api completed, no instances found");
+            return;
+        }
+
+        var instancesFromK8s = await KubernetesClient.List<V1Alpha1ObjectStorage2>(@namespace: Options.Namespace, labelSelector: null);
+
+        Collection<InstanceDetailsResponse>? instancesToCreate = null;
+        foreach (var instanceFromApi in instancesFromApi) {
+            if (DoesInstanceExistInK8s(in instancesFromK8s, in instanceFromApi)) {
+                Logger.LogDebug("Loading object storage instance list from upc api, already existing instance skipped (uuid: {instanceUuid})",
+                    instanceFromApi.UUID
+                );
+
+                continue;
+            }
+
+            (instancesToCreate ??= new ()).Add(instanceFromApi);
+        }
+
+        if (instancesToCreate is null || instancesToCreate.Count == 0) {
+            Logger.LogDebug("Loading object storage instances from upc api completed, no new instances found");
+            return;
+        }
+
+        var created = 0;
+        foreach (var newInstanceDetails in instancesToCreate) {
+            var newResource = CreateNewResource(in newInstanceDetails);
+            Logger.LogInformation("Loading object storage instances from upc api completed, creating new instance (uuid: {newInstanceUuid})",
+                newResource.Spec.Id
+            );
+
+            try {
+                _ = await KubernetesClient.Create(resource: newResource);
+
+                Logger.LogInformation("Loading object storage instances from upc api completed, new instance created (uuid: {newInstanceUuid})",
+                    newResource.Spec.Id
+                );
+                created += 1;
+            }
+            catch (HttpOperationException hex) {
+                Logger.LogError(hex, "Loading object storage instances from upc api completed, creating new instance failed (uuid: {newInstanceUuid}, error: {errorContent})",
+                    newResource.Spec.Id,
+                    hex.Response.Content.AsSpan().ToString()
+                );
+            }
+            catch (Exception ex) {
+                Logger.LogError(ex, "Loading object storage instances from upc api completed, creating new instance failed (uuid: {newInstanceUuid})",
+                    newResource.Spec.Id
+                );
+            }
+        }
+
+        Logger.LogDebug("Loading object storage instances from upc api completed (created: {createdCount})",
+            created
+        );
+    }
+
+    private static bool DoesInstanceExistInK8s(in IList<V1Alpha1ObjectStorage2> instancesFromK8s, in InstanceDetailsResponse instancesFromApi)
+    {
+        for (var idx = 0; idx < instancesFromK8s.Count; idx++) {
+            if (instancesFromK8s[idx].Status.Id == instancesFromApi.UUID) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private V1Alpha1ObjectStorage2 CreateNewResource(in InstanceDetailsResponse newInstanceDetails) => new()
+        {
+            Spec       = CreateNewInstanceSpec(in newInstanceDetails),
+            Status     = new (),
+            Metadata   = new V1ObjectMeta {
+                Name              = string.Format("object-storage-v2-{0}", newInstanceDetails.UUID),
+                NamespaceProperty = Options.Namespace
+            }
+        };
+
+    private static V1Alpha1ObjectStorage2.V1Alpha1ObjectStorageSpec CreateNewInstanceSpec(in InstanceDetailsResponse newInstanceDetails) => new()
+        {
+            Id               = newInstanceDetails.UUID,
+            Region           = newInstanceDetails.Region,
+            ConfiguredStatus = newInstanceDetails.ConfiguredStatus,
+            Networks         = new (
+                newInstanceDetails.Networks.Select(x => new V1Alpha1ObjectStorage2.Network {
+                    Name   = x.Name,
+                    Family = x.Family,
+                    Type   = x.Type,
+                    Id     = x.UUID
+                }).ToList()
+            ),
+            Users            = new (
+                newInstanceDetails.Users.Select(x => new V1Alpha1ObjectStorage2.User {
+                    Username = x.Username
+                }).ToList()
+            ),
+            Labels           = new (
+                newInstanceDetails.Labels.Select(x => new V1Alpha1ObjectStorage2.Label {
+                    Name  = x.Key,
+                    Value = x.Value
+                }).ToList()
+            )
+        };
+}

src/UpcloudApiKubernetesOperator/AutoLoader/Options/AutoLoaderOptions.cs

@@ -0,0 +1,37 @@
+using System.Collections.ObjectModel;
+
+using Microsoft.Extensions.Options;
+
+namespace UpcloudApiKubernetesOperator.AutoLoader.Options;
+
+internal sealed class AutoLoaderOptions
+{
+    public const string DEFAULT_SECTION_NAME = "AutoLoader";
+    public bool Enabled        { get; init; } = false;
+    public int RefreshInterval { get; init; } = 120;
+    public string Namespace    { get; init; } = string.Empty;
+
+    public AutoLoaderOptions() {}
+
+    public class Validator : IValidateOptions<AutoLoaderOptions>
+    {
+        public ValidateOptionsResult Validate(string? name, AutoLoaderOptions options)
+        {
+            Collection<string>? errors = null;
+
+            if (options.RefreshInterval < 1) {
+                (errors ??= new ()).Add($"'{nameof(options.RefreshInterval)}' has to be positive integer, auto loader refresh interval in seconds");
+            }
+
+            if (string.IsNullOrEmpty(options.Namespace)) {
+                (errors ??= new ()).Add($"'{nameof(options.Namespace)}' cannot be null or empty");
+            }
+
+            if (errors is not null) {
+                return ValidateOptionsResult.Fail(errors);
+            }
+
+            return ValidateOptionsResult.Success;
+        }
+    }
+}