Part 42: Velero Restore Into a Throwaway Cluster📋

# What the CronJob does, in shell form
LATEST=$(velero backup get -o name | grep daily-acme-prod | sort | tail -1)
velero restore create test-restore-$(date +%Y%m%d) \
    --from-backup $LATEST \
    --namespace-mappings acme-prod:acme-restore-test \
    --wait
kubectl wait --for=condition=Ready pod -l app=acme-api -n acme-restore-test --timeout=300s
velero restore delete test-restore-$(date +%Y%m%d) --confirm
kubectl delete namespace acme-restore-test

[Injectable(ServiceLifetime.Singleton)]
[VerbGroup("k8s")]
public sealed class K8sRestoreTestCommand : IHomeLabVerbCommand
{
    public Command Build()
    {
        var sourceCluster = new Argument<string>("source");
        var backupId = new Option<string?>("--backup-id");
        var topology = new Option<string>("--topology", () => "k8s-multi");

        var cmd = new Command("restore-test", "Spin up a fresh cluster, restore the latest backup, verify, destroy");
        cmd.AddArgument(sourceCluster);
        cmd.AddOption(backupId);
        cmd.AddOption(topology);

        cmd.SetHandler(async (string src, string? id, string topo) =>
        {
            var result = await _mediator.SendAsync(new K8sRestoreTestRequest(src, id, topo), default);
            _console.Render(result);
            Environment.ExitCode = result.IsSuccess ? 0 : 1;
        }, sourceCluster, backupId, topology);

        return cmd;
    }
}

[Injectable(ServiceLifetime.Singleton)]
public sealed class K8sRestoreTestRequestHandler : IRequestHandler<K8sRestoreTestRequest, Result<K8sRestoreTestResponse>>
{
    public async Task<Result<K8sRestoreTestResponse>> HandleAsync(K8sRestoreTestRequest req, CancellationToken ct)
    {
        var ephemeralName = $"restore-test-{_clock.UtcNow:yyyyMMddHHmmss}";

        // 1. Acquire a fresh HomeLab instance
        var scope = await _registry.AcquireAsync(ephemeralName, ct);
        if (scope.IsFailure) return scope.Map<K8sRestoreTestResponse>();

        try
        {
            // 2. Stand up a fresh cluster in that instance
            var createResult = await _mediator.SendAsync(
                new K8sCreateRequest(scope.Value.Name, _config.K8s!.Distribution, req.Topology, _config.K8s.Version), ct);
            if (createResult.IsFailure) return createResult.Map<K8sRestoreTestResponse>();

            // 3. Install Velero in the fresh cluster, pointing at the SAME MinIO bucket
            await _veleroInstaller.InstallAsync(scope.Value, _config.K8s.Backup!.MinioEndpoint, ct);

            // 4. Find the latest backup of the source cluster
            var latestBackup = req.BackupId ?? await _velero.GetLatestBackupAsync(req.SourceCluster, ct);

            // 5. Restore it
            var restoreResult = await _velero.RestoreAsync(latestBackup, ct);
            if (restoreResult.IsFailure) return restoreResult.Map<K8sRestoreTestResponse>();

            // 6. Verify a known healthcheck endpoint
            await Task.Delay(TimeSpan.FromMinutes(2), ct);   // give workloads time to come up
            var healthResult = await _http.GetAsync($"https://gitlab.{scope.Value.TldPrefix}.lab/-/health", ct);
            if (healthResult.IsFailure || !healthResult.Value.Contains("ok"))
                return Result.Failure<K8sRestoreTestResponse>("restore test failed: workload did not become healthy");

            await _events.PublishAsync(new RestoreTestPassed(req.SourceCluster, latestBackup, _clock.UtcNow), ct);
            return Result.Success(new K8sRestoreTestResponse(req.SourceCluster, latestBackup, "passed"));
        }
        finally
        {
            // 7. Destroy the ephemeral instance regardless of outcome
            await _mediator.SendAsync(new K8sDestroyRequest(ephemeralName), ct);
            await _registry.ReleaseAsync(ephemeralName, ct);
        }
    }
}