Part 40: kubeadm Upgrades — The Painful One📋

[Injectable(ServiceLifetime.Singleton)]
public sealed class KubernetesUpgradePathResolver
{
    // Kubernetes supports skipping at most one minor version per upgrade.
    // 1.30 → 1.31 → 1.32 is allowed. 1.30 → 1.32 is not.
    public IReadOnlyList<string> ResolvePath(string from, string to)
    {
        var fromV = ParseSemver(from);
        var toV = ParseSemver(to);

        if (fromV.Minor == toV.Minor)
            return new[] { to };   // patch upgrade only

        var path = new List<string>();
        for (var minor = fromV.Minor + 1; minor <= toV.Minor; minor++)
        {
            // Last patch of the target minor (in practice, this is queried from a release feed)
            var version = LatestPatchOf(toV.Major, minor);
            path.Add(version);
        }

        return path;
    }
}

[Saga]
public sealed class KubeadmUpgradeSaga
{
    [SagaStep(Order = 1, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> BackupBeforeUpgrade(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        var result = await _velero.RunBackupAsync($"pre-upgrade-{ctx.TargetVersion}", new[] { "kube-system" }, ct);
        if (result.IsSuccess) ctx.PreUpgradeBackupId = result.Value;
        return result.Map();
    }

    [SagaStep(Order = 2, Compensation = nameof(RestoreFromBackup))]
    public async Task<Result> CheckUpgradePlan(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        var planResult = await _vagrant.SshCommandAsync(
            ctx.FirstControlPlaneNode,
            $"sudo kubeadm upgrade plan {ctx.TargetVersion}",
            ct);

        // The plan output tells us if the upgrade is even valid
        if (planResult.IsFailure) return planResult.Map();
        if (!planResult.Value.StdOut.Contains("[upgrade/versions] Latest version"))
            return Result.Failure("kubeadm upgrade plan returned unexpected output");
        return Result.Success();
    }

    [SagaStep(Order = 3, Compensation = nameof(RestoreFromBackup))]
    public async Task<Result> UpgradeFirstControlPlane(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        // 1. Apt-pin the new kubeadm version on the first cp node
        var pkgResult = await _vagrant.SshCommandAsync(
            ctx.FirstControlPlaneNode,
            $"sudo apk add --no-cache kubeadm={ctx.TargetVersion.TrimStart('v')}-r0",
            ct);
        if (pkgResult.IsFailure) return pkgResult.Map();

        // 2. Run kubeadm upgrade apply
        var applyResult = await _vagrant.SshCommandAsync(
            ctx.FirstControlPlaneNode,
            $"sudo kubeadm upgrade apply -y {ctx.TargetVersion}",
            ct);
        return applyResult.Map();
    }

    [SagaStep(Order = 4, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> DrainControlPlaneNode(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        return await _kubectl.DrainAsync(
            ctx.FirstControlPlaneNode,
            ignoreDaemonsets: true,
            deleteEmptyDirData: true,
            force: true,
            ct).Map();
    }

    [SagaStep(Order = 5, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> UpgradeKubeletAndKubectl(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        return await _vagrant.SshCommandAsync(
            ctx.FirstControlPlaneNode,
            $"sudo apk add --no-cache kubelet={ctx.TargetVersion.TrimStart('v')}-r0 kubectl={ctx.TargetVersion.TrimStart('v')}-r0 && " +
            $"sudo service kubelet restart",
            ct).Map();
    }

    [SagaStep(Order = 6, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> UncordonControlPlaneNode(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        return await _kubectl.UncordonAsync(ctx.FirstControlPlaneNode, ct).Map();
    }

    // Steps 7..9 repeat for each additional control plane (in HA topology)
    // Steps 10..12 repeat for each worker

    public async Task<Result> RestoreFromBackup(KubeadmUpgradeContext ctx, CancellationToken ct)
    {
        if (ctx.PreUpgradeBackupId is null) return Result.Success();
        return await _velero.RestoreAsync(ctx.PreUpgradeBackupId, ct).Map();
    }

    public Task<Result> NothingToCompensate(KubeadmUpgradeContext ctx, CancellationToken ct)
        => Task.FromResult(Result.Success());
}

$ homelab k8s upgrade --to v1.32.0
Current version: v1.30.5
Target version:  v1.32.0
Upgrade path:
  v1.31.4 (intermediate stop)
  v1.32.0 (target)

This is a 2-step upgrade across 6 nodes. Estimated time: ~45 minutes.
A pre-upgrade backup will be taken to MinIO before each step.

Continue? [y/N] y

[1/2] Upgrading to v1.31.4
  ✓ pre-upgrade backup (acme-pre-upgrade-v1.31.4, 1.4 GB)
  ✓ kubeadm upgrade plan v1.31.4
  ✓ kubeadm upgrade apply v1.31.4 (cp-1) — 4m22s
  ✓ drain cp-1
  ✓ kubelet upgrade on cp-1
  ✓ uncordon cp-1
  ✓ drain cp-2
  ✓ kubelet upgrade on cp-2
  ✓ uncordon cp-2
  ✓ drain cp-3
  ✓ kubelet upgrade on cp-3
  ✓ uncordon cp-3
  ✓ drain w-1
  ✓ kubelet upgrade on w-1
  ✓ uncordon w-1
  ✓ drain w-2
  ✓ kubelet upgrade on w-2
  ✓ uncordon w-2
  ✓ drain w-3
  ✓ kubelet upgrade on w-3
  ✓ uncordon w-3
  ✓ all nodes at v1.31.4

[2/2] Upgrading to v1.32.0
  ... same flow ...

✓ cluster upgrade complete: all nodes at v1.32.0
  total elapsed: 41m18s
  pre-upgrade backups: acme-pre-upgrade-v1.31.4, acme-pre-upgrade-v1.32.0