Part 41: k3s Upgrades — The Easy Mode📋

[Saga]
public sealed class K3sUpgradeSaga
{
    [SagaStep(Order = 1, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> BackupBeforeUpgrade(K3sUpgradeContext ctx, CancellationToken ct)
    {
        var result = await _velero.RunBackupAsync($"pre-upgrade-{ctx.TargetVersion}", new[] { "kube-system" }, ct);
        if (result.IsSuccess) ctx.PreUpgradeBackupId = result.Value;
        return result.Map();
    }

    [SagaStep(Order = 2, Compensation = nameof(RestoreFromBackup))]
    public async Task<Result> UpgradeServerNodes(K3sUpgradeContext ctx, CancellationToken ct)
    {
        // For HA k3s (with --cluster-init), upgrade server nodes one at a time
        foreach (var node in ctx.ServerNodes)
        {
            var result = await UpgradeOneNode(node, ctx.TargetVersion, isServer: true, ct);
            if (result.IsFailure) return result;

            // Wait for the API to come back
            var waitResult = await WaitForApiHealthy(node, ct);
            if (waitResult.IsFailure) return waitResult;
        }
        return Result.Success();
    }

    [SagaStep(Order = 3, Compensation = nameof(NothingToCompensate))]
    public async Task<Result> UpgradeAgentNodes(K3sUpgradeContext ctx, CancellationToken ct)
    {
        // Agents (workers) can be upgraded in parallel because they are stateless
        var tasks = ctx.AgentNodes.Select(async node =>
        {
            await _kubectl.DrainAsync(node, ct: ct);
            var upgradeResult = await UpgradeOneNode(node, ctx.TargetVersion, isServer: false, ct);
            await _kubectl.UncordonAsync(node, ct);
            return upgradeResult;
        });
        var results = await Task.WhenAll(tasks);
        return results.All(r => r.IsSuccess) ? Result.Success() : Result.Failure("agent upgrade had failures");
    }

    private async Task<Result> UpgradeOneNode(string nodeName, string targetVersion, bool isServer, CancellationToken ct)
    {
        var role = isServer ? "server" : "agent";
        var installCmd =
            $"curl -sfL https://get.k3s.io | " +
            $"INSTALL_K3S_VERSION={targetVersion} " +
            $"INSTALL_K3S_SKIP_START=true " +
            $"sh -s - {role}";

        return await _vagrant.SshCommandAsync(nodeName, installCmd, ct).Map();
        // The k3s installer replaces the binary; the daemon picks up the new version on restart
    }

    public async Task<Result> RestoreFromBackup(K3sUpgradeContext ctx, CancellationToken ct)
        => ctx.PreUpgradeBackupId is null ? Result.Success() : await _velero.RestoreAsync(ctx.PreUpgradeBackupId, ct).Map();

    public Task<Result> NothingToCompensate(K3sUpgradeContext ctx, CancellationToken ct)
        => Task.FromResult(Result.Success());
}

apiVersion: upgrade.cattle.io/v1
kind: Plan
metadata:
  name: k3s-server
  namespace: system-upgrade
spec:
  concurrency: 1
  cordon: true
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/control-plane: "true"
  serviceAccountName: system-upgrade
  upgrade:
    image: rancher/k3s-upgrade
  version: v1.32.0+k3s1
---
apiVersion: upgrade.cattle.io/v1
kind: Plan
metadata:
  name: k3s-agent
  namespace: system-upgrade
spec:
  concurrency: 2
  cordon: true
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/control-plane: "false"
  prepare:
    args: ["prepare", "k3s-server"]
    image: rancher/k3s-upgrade
  serviceAccountName: system-upgrade
  upgrade:
    image: rancher/k3s-upgrade
  version: v1.32.0+k3s1

[Injectable(ServiceLifetime.Singleton)]
public sealed class K3sUpgradePlanContributor : IK8sManifestContributor
{
    public bool ShouldContribute() =>
        _config.K8s?.Distribution == "k3s" && _config.K8s?.K3s?.SystemUpgradeController?.Enabled == true;

    public void Contribute(KubernetesBundle bundle)
    {
        // Generate the Plan CRDs above
    }
}