Part XI: Schema Version Merging -- 32 to 1📋

// Naive: 32 separate type systems
// Which one do you use? What if your Docker Compose is 2.5.0?
ComposeFile_v2_10_1 file = new();
// Fails at runtime if the user's Compose spec is older
file.Services["web"].Provider = new ComposeServiceProvider();

// Merged: one type system, version metadata on properties
ComposeFile file = new();
file.Services["web"].Provider = new ComposeServiceProvider();
// The IDE shows: [SinceVersion("2.5.0")]
// The validator warns at build time if you target < 2.5.0

public static class SchemaVersionMerger
{
    public static UnifiedSchema Merge(
        IReadOnlyList<(ComposeSchemaVersion Version, SchemaModel Schema)> versioned)
    {
        var sorted = versioned.OrderBy(v => v.Version).ToList();
        var oldest = sorted[0].Version;
        var newest = sorted[^1].Version;

        // Step 2: Union all definition names across all versions
        var allDefinitions = sorted
            .SelectMany(v => v.Schema.Definitions.Keys)
            .Distinct()
            .ToList();

        var unified = new UnifiedSchema();

        foreach (var defName in allDefinitions)
        {
            var firstSeen = sorted
                .First(v => v.Schema.Definitions.ContainsKey(defName))
                .Version;
            var lastSeen = sorted
                .Last(v => v.Schema.Definitions.ContainsKey(defName))
                .Version;

            // Step 5: Take the latest version's definition shape
            var latestDef = sorted
                .Last(v => v.Schema.Definitions.ContainsKey(defName))
                .Schema.Definitions[defName];

            var unifiedDef = new UnifiedDefinition
            {
                Name = defName,
                Schema = latestDef,
                SinceVersion = firstSeen == oldest ? null : firstSeen,
                UntilVersion = lastSeen == newest ? null : lastSeen,
                Properties = MergeProperties(
                    defName, sorted, oldest, newest),
            };

            unified.Definitions.Add(defName, unifiedDef);
        }

        return unified;
    }

    private static List<UnifiedProperty> MergeProperties(
        string defName,
        List<(ComposeSchemaVersion Version, SchemaModel Schema)> sorted,
        ComposeSchemaVersion oldest,
        ComposeSchemaVersion newest)
    {
        // Step 3: Union all property names for this definition
        var allProps = sorted
            .Where(v => v.Schema.Definitions.ContainsKey(defName))
            .SelectMany(v => v.Schema.Definitions[defName]
                .Properties.Select(p => p.Name))
            .Distinct()
            .ToList();

        return allProps.Select(propName =>
        {
            // Step 4: Find version bounds
            var firstSeen = sorted
                .First(v =>
                    v.Schema.Definitions.ContainsKey(defName) &&
                    v.Schema.Definitions[defName].Properties
                        .Any(p => p.Name == propName))
                .Version;

            var lastSeen = sorted
                .Last(v =>
                    v.Schema.Definitions.ContainsKey(defName) &&
                    v.Schema.Definitions[defName].Properties
                        .Any(p => p.Name == propName))
                .Version;

            // Step 5: Take latest property shape
            var latestProp = sorted
                .Last(v =>
                    v.Schema.Definitions.ContainsKey(defName) &&
                    v.Schema.Definitions[defName].Properties
                        .Any(p => p.Name == propName))
                .Schema.Definitions[defName].Properties
                .First(p => p.Name == propName);

            return new UnifiedProperty
            {
                Name = propName,
                Schema = latestProp.Schema,
                Description = latestProp.Description,
                SinceVersion = firstSeen == oldest
                    ? null : firstSeen,
                UntilVersion = lastSeen == newest
                    ? null : lastSeen,
            };
        }).ToList();
    }
}

{
  "build": {
    "type": "string",
    "description": "Build context path"
  }
}

{
  "build": {
    "oneOf": [
      { "type": "string" },
      {
        "type": "object",
        "properties": {
          "context": { "type": "string" },
          "dockerfile": { "type": "string" },
          "args": { "$ref": "#/definitions/list_or_dict" },
          "ssh": { "$ref": "#/definitions/list_or_dict" },
          "cache_from": { "type": "array" },
          "cache_to": { "type": "array" },
          "extra_hosts": { "$ref": "#/definitions/list_or_dict" },
          "isolation": { "type": "string" },
          "labels": { "$ref": "#/definitions/list_or_dict" },
          "no_cache": { "type": "boolean" },
          "pull": { "type": "boolean" },
          "shm_size": { "type": ["integer", "string"] },
          "target": { "type": "string" },
          "secrets": { "$ref": "#/definitions/service_config_or_secret" },
          "tags": { "type": "array" },
          "platforms": { "type": "array" },
          "privileged": { "type": "boolean" },
          "network": { "type": "string" },
          "entitlements": { "type": "array" },
          "ulimits": { "$ref": "#/definitions/ulimits" },
          "additional_contexts": { "$ref": "#/definitions/list_or_dict" },
          "dockerfile_inline": { "type": "string" }
        }
      }
    ]
  }
}

public partial class ComposeServiceBuildConfig
{
    /// <summary>
    /// Build context path. Also accepts the string shorthand form.
    /// </summary>
    public string? Context { get; set; }

    public string? Dockerfile { get; set; }

    public ComposeListOrDict? Args { get; set; }

    public ComposeListOrDict? Ssh { get; set; }

    public List<string>? CacheFrom { get; set; }

    public List<string>? CacheTo { get; set; }

    public ComposeListOrDict? ExtraHosts { get; set; }

    public string? Isolation { get; set; }

    public ComposeListOrDict? Labels { get; set; }

    public bool? NoCache { get; set; }

    public bool? Pull { get; set; }

    public StringOrInt? ShmSize { get; set; }

    public string? Target { get; set; }

    [SinceVersion("1.4.0")]
    public List<ComposeServiceConfigOrSecret>? Secrets { get; set; }

    [SinceVersion("1.8.0")]
    public List<string>? Tags { get; set; }

    [SinceVersion("1.12.0")]
    public List<string>? Platforms { get; set; }

    [SinceVersion("1.15.0")]
    public bool? Privileged { get; set; }

    [SinceVersion("1.15.0")]
    public string? Network { get; set; }

    [SinceVersion("1.19.0")]
    public List<string>? Entitlements { get; set; }

    [SinceVersion("1.19.0")]
    public ComposeUlimits? Ulimits { get; set; }

    [SinceVersion("2.0.0")]
    public ComposeListOrDict? AdditionalContexts { get; set; }

    [SinceVersion("2.1.0")]
    public string? DockerfileInline { get; set; }
}

# Short syntax (string)
ports:
  - "8080:80"
  - "443:443/tcp"

# Long syntax (object)
ports:
  - target: 80
    published: "8080"
    protocol: tcp
    mode: host

{
  "ports": {
    "type": "array",
    "items": {
      "oneOf": [
        { "type": "number" },
        {
          "type": "string",
          "format": "ports"
        },
        {
          "type": "object",
          "properties": {
            "name": { "type": "string" },
            "mode": { "type": "string" },
            "host_ip": { "type": "string" },
            "target": { "type": "integer" },
            "published": { "type": ["string", "integer"] },
            "protocol": { "type": "string" },
            "app_protocol": { "type": "string" }
          }
        }
      ]
    }
  }
}

public partial class ComposeServicePortsConfig
{
    /// <summary>
    /// The short syntax form (e.g., "8080:80/tcp").
    /// Mutually exclusive with structured properties.
    /// </summary>
    public string? Value { get; set; }

    public int? Target { get; set; }

    public StringOrInt? Published { get; set; }

    public string? Protocol { get; set; }

    public string? Mode { get; set; }

    public string? HostIp { get; set; }

    [SinceVersion("1.16.0")]
    public string? Name { get; set; }

    [SinceVersion("2.6.0")]
    public string? AppProtocol { get; set; }
}

depends_on:
  - db
  - redis

depends_on:
  db:
    condition: service_healthy
  redis:
    condition: service_started
    restart: true
    required: true

public partial class ComposeServiceDependsOnConfig
{
    /// <summary>
    /// Simple form: just the service name.
    /// </summary>
    public string? Value { get; set; }

    public string? Condition { get; set; }

    [SinceVersion("2.3.0")]
    public bool? Restart { get; set; }

    [SinceVersion("2.9.0")]
    public bool? Required { get; set; }
}

volumes:
  # Bind mount
  - type: bind
    source: ./data
    target: /app/data
    bind:
      propagation: rprivate
      create_host_path: true

  # Named volume
  - type: volume
    source: db-data
    target: /var/lib/postgresql/data
    volume:
      nocopy: true
      subpath: pgdata

  # tmpfs mount
  - type: tmpfs
    target: /tmp
    tmpfs:
      size: 67108864
      mode: 1777

  # Image mount (newest addition)
  - type: image
    source: my-config-image:latest
    target: /config

public partial class ComposeServiceVolumeConfig
{
    /// <summary>
    /// Short syntax: "source:target:mode" or just "target"
    /// </summary>
    public string? Value { get; set; }

    public string? Type { get; set; }     // bind | volume | tmpfs | image
    public string? Source { get; set; }
    public string? Target { get; set; }
    public bool? ReadOnly { get; set; }

    public ComposeServiceVolumeBindConfig? Bind { get; set; }
    public ComposeServiceVolumeVolumeConfig? Volume { get; set; }
    public ComposeServiceVolumeTmpfsConfig? Tmpfs { get; set; }

    [SinceVersion("2.8.0")]
    public string? Image { get; set; }
}

public partial class ComposeServiceVolumeBindConfig
{
    public string? Propagation { get; set; }

    [SinceVersion("1.13.0")]
    public bool? CreateHostPath { get; set; }

    [SinceVersion("2.2.0")]
    public string? Selinux { get; set; }
}

public partial class ComposeServiceVolumeVolumeConfig
{
    public bool? Nocopy { get; set; }

    [SinceVersion("2.6.0")]
    public string? Subpath { get; set; }
}

public partial class ComposeServiceVolumeTmpfsConfig
{
    public long? Size { get; set; }

    [SinceVersion("1.10.0")]
    public int? Mode { get; set; }
}

1.0.9:  endpoint_mode, labels, mode, placement, replicas,
        resources, restart_policy, update_config
1.7.0:  + rollback_config
1.15.0: + extensions (x- prefix support)
2.4.0:  + placement.max_replicas_per_node
2.8.0:  + resources.reservations.devices (GPU support)

public partial class ComposeServiceDeployConfig
{
    public string? EndpointMode { get; set; }
    public ComposeListOrDict? Labels { get; set; }
    public string? Mode { get; set; }
    public ComposeServiceDeployPlacement? Placement { get; set; }
    public int? Replicas { get; set; }
    public ComposeServiceDeployResources? Resources { get; set; }
    public ComposeServiceDeployRestartPolicy? RestartPolicy { get; set; }
    public ComposeServiceDeployUpdateConfig? UpdateConfig { get; set; }

    [SinceVersion("1.7.0")]
    public ComposeServiceDeployUpdateConfig? RollbackConfig { get; set; }
}

public partial class ComposeServiceDeployResources
{
    public ComposeServiceDeployResourcesLimit? Limits { get; set; }
    public ComposeServiceDeployResourcesReservation? Reservations { get; set; }
}

public partial class ComposeServiceDeployResourcesReservation
{
    public string? Cpus { get; set; }
    public StringOrInt? Memory { get; set; }

    [SinceVersion("2.8.0")]
    public List<ComposeServiceDeployResourcesDevice>? Devices { get; set; }
}

public partial class ComposeServiceDeployPlacement
{
    public List<string>? Constraints { get; set; }
    public List<ComposeServiceDeployPlacementPreference>? Preferences { get; set; }

    [SinceVersion("2.4.0")]
    public int? MaxReplicasPerNode { get; set; }
}

// <auto-generated by FrenchExDev.Net.DockerCompose.Bundle.SourceGenerator/>
namespace FrenchExDev.Net.DockerCompose.Bundle;

public static class ComposeSchemaVersions
{
    public static IReadOnlyList<ComposeSchemaVersion> Available { get; } = new[]
    {
        ComposeSchemaVersion.Parse("1.0.9"),
        ComposeSchemaVersion.Parse("1.1.0"),
        ComposeSchemaVersion.Parse("1.2.0"),
        ComposeSchemaVersion.Parse("1.3.0"),
        ComposeSchemaVersion.Parse("1.4.0"),
        ComposeSchemaVersion.Parse("1.5.0"),
        ComposeSchemaVersion.Parse("1.6.0"),
        ComposeSchemaVersion.Parse("1.7.0"),
        ComposeSchemaVersion.Parse("1.8.0"),
        ComposeSchemaVersion.Parse("1.9.0"),
        ComposeSchemaVersion.Parse("1.10.0"),
        ComposeSchemaVersion.Parse("1.11.0"),
        ComposeSchemaVersion.Parse("1.12.0"),
        ComposeSchemaVersion.Parse("1.13.0"),
        ComposeSchemaVersion.Parse("1.14.0"),
        ComposeSchemaVersion.Parse("1.15.0"),
        ComposeSchemaVersion.Parse("1.16.0"),
        ComposeSchemaVersion.Parse("1.17.0"),
        ComposeSchemaVersion.Parse("1.18.0"),
        ComposeSchemaVersion.Parse("1.19.0"),
        ComposeSchemaVersion.Parse("2.0.0"),
        ComposeSchemaVersion.Parse("2.1.0"),
        ComposeSchemaVersion.Parse("2.2.0"),
        ComposeSchemaVersion.Parse("2.3.0"),
        ComposeSchemaVersion.Parse("2.4.0"),
        ComposeSchemaVersion.Parse("2.5.0"),
        ComposeSchemaVersion.Parse("2.6.0"),
        ComposeSchemaVersion.Parse("2.7.0"),
        ComposeSchemaVersion.Parse("2.7.1"),
        ComposeSchemaVersion.Parse("2.8.0"),
        ComposeSchemaVersion.Parse("2.9.0"),
        ComposeSchemaVersion.Parse("2.10.0"),
        ComposeSchemaVersion.Parse("2.10.1"),
    };

    public static ComposeSchemaVersion Oldest => Available[0];     // 1.0.9
    public static ComposeSchemaVersion Latest => Available[^1];    // 2.10.1
}

var target = ComposeSchemaVersion.Parse("2.5.0");
bool isSupported = target >= ComposeSchemaVersions.Oldest
    && target <= ComposeSchemaVersions.Latest;

// <auto-generated by FrenchExDev.Net.DockerCompose.Bundle.SourceGenerator/>
namespace FrenchExDev.Net.DockerCompose.Bundle;

/// <summary>
/// Indicates the compose specification version where this property
/// first appeared. Absence means the property has always been present.
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public sealed class SinceVersionAttribute : Attribute
{
    public string Version { get; }

    public SinceVersionAttribute(string version)
        => Version = version;
}

/// <summary>
/// Indicates the last compose specification version where this
/// property was present. Absence means the property is still current.
/// </summary>
[AttributeUsage(AttributeTargets.Property)]
public sealed class UntilVersionAttribute : Attribute
{
    public string Version { get; }

    public UntilVersionAttribute(string version)
        => Version = version;
}

private static void EmitProperty(
    IndentedTextWriter writer,
    UnifiedProperty prop)
{
    if (prop.Description is not null)
    {
        writer.WriteLine("/// <summary>");
        writer.WriteLine($"/// {prop.Description}");
        writer.WriteLine("/// </summary>");
    }

    if (prop.SinceVersion is not null)
        writer.WriteLine(
            $"[SinceVersion(\"{prop.SinceVersion}\")]");

    if (prop.UntilVersion is not null)
        writer.WriteLine(
            $"[UntilVersion(\"{prop.UntilVersion}\")]");

    writer.WriteLine(
        $"public {prop.ClrType} {prop.PascalName} {{ get; set; }}");
    writer.WriteLine();
}

// <auto-generated by FrenchExDev.Net.DockerCompose.Bundle.SourceGenerator/>
namespace FrenchExDev.Net.DockerCompose.Bundle;

public partial class ComposeService
{
    /// <summary>
    /// Container image to use.
    /// </summary>
    public string? Image { get; set; }

    /// <summary>
    /// Build configuration.
    /// </summary>
    public ComposeServiceBuildConfig? Build { get; set; }

    /// <summary>
    /// Command to run in the container.
    /// </summary>
    public ComposeStringOrList? Command { get; set; }

    // ... 50+ properties with no version annotation ...

    /// <summary>
    /// Development mode configuration for hot-reload
    /// and file watching.
    /// </summary>
    [SinceVersion("1.19.0")]
    public ComposeDevelopment? Develop { get; set; }

    /// <summary>
    /// Post-start lifecycle hooks.
    /// </summary>
    [SinceVersion("2.3.0")]
    public List<ComposeServiceHook>? PostStart { get; set; }

    /// <summary>
    /// Pre-stop lifecycle hooks.
    /// </summary>
    [SinceVersion("2.3.0")]
    public List<ComposeServiceHook>? PreStop { get; set; }

    /// <summary>
    /// External service provider configuration.
    /// </summary>
    [SinceVersion("2.5.0")]
    public ComposeServiceProvider? Provider { get; set; }

    /// <summary>
    /// GPU and AI model configurations.
    /// </summary>
    [SinceVersion("2.7.1")]
    public object? Models { get; set; }

    /// <summary>
    /// GPGPU configuration (deprecated, use deploy.resources).
    /// </summary>
    [UntilVersion("1.8.0")]
    public ComposeServiceGpuConfig? Gpus { get; set; }
}

public sealed class UnifiedSchema
{
    public Dictionary<string, UnifiedDefinition> Definitions { get; }
        = new();

    public ComposeSchemaVersion OldestVersion { get; init; }
    public ComposeSchemaVersion NewestVersion { get; init; }
    public int TotalVersions { get; init; }
}

public sealed class UnifiedDefinition
{
    public required string Name { get; init; }
    public required SchemaDefinition Schema { get; init; }
    public ComposeSchemaVersion? SinceVersion { get; init; }
    public ComposeSchemaVersion? UntilVersion { get; init; }
    public required List<UnifiedProperty> Properties { get; init; }

    public bool IsAlwaysPresent
        => SinceVersion is null && UntilVersion is null;

    public bool IsDeprecated
        => UntilVersion is not null;
}

public sealed class UnifiedProperty
{
    public required string Name { get; init; }
    public required SchemaPropertyType Schema { get; init; }
    public string? Description { get; init; }
    public ComposeSchemaVersion? SinceVersion { get; init; }
    public ComposeSchemaVersion? UntilVersion { get; init; }

    public string PascalName => Name.ToPascalCase();
    public string ClrType => Schema.ToClrType();

    public bool IsAlwaysPresent
        => SinceVersion is null && UntilVersion is null;

    public bool IsDeprecated
        => UntilVersion is not null;
}

// Generated output for a deprecated property
[UntilVersion("1.8.0")]
[Obsolete("Removed after compose-spec 1.8.0. " +
    "Use deploy.resources.reservations.devices instead.")]
public ComposeServiceGpuConfig? Gpus { get; set; }

// Inside MergeProperties, after finding firstSeen and lastSeen
var typeAtFirst = sorted
    .First(v => HasProperty(v, defName, propName))
    .Schema.Definitions[defName].Properties
    .First(p => p.Name == propName).Schema;

var typeAtLast = sorted
    .Last(v => HasProperty(v, defName, propName))
    .Schema.Definitions[defName].Properties
    .First(p => p.Name == propName).Schema;

if (!typeAtFirst.IsAssignableFrom(typeAtLast))
{
    diagnostics.Report(Diagnostic.Create(
        DiagnosticDescriptors.IncompatibleTypeChange,
        Location.None,
        defName, propName,
        typeAtFirst.ToClrType(),
        typeAtLast.ToClrType()));
}

public readonly record struct ComposeSchemaVersion(
    int Major, int Minor, int Patch)
    : IComparable<ComposeSchemaVersion>
{
    public int CompareTo(ComposeSchemaVersion other)
    {
        var major = Major.CompareTo(other.Major);
        if (major != 0) return major;

        var minor = Minor.CompareTo(other.Minor);
        if (minor != 0) return minor;

        return Patch.CompareTo(other.Patch);
    }

    public static ComposeSchemaVersion Parse(string version)
    {
        var parts = version.Split('.');
        return new(
            int.Parse(parts[0]),
            int.Parse(parts[1]),
            parts.Length > 2 ? int.Parse(parts[2]) : 0);
    }

    public override string ToString()
        => $"{Major}.{Minor}.{Patch}";

    public static bool operator >(ComposeSchemaVersion left,
        ComposeSchemaVersion right) => left.CompareTo(right) > 0;
    public static bool operator <(ComposeSchemaVersion left,
        ComposeSchemaVersion right) => left.CompareTo(right) < 0;
    public static bool operator >=(ComposeSchemaVersion left,
        ComposeSchemaVersion right) => left.CompareTo(right) >= 0;
    public static bool operator <=(ComposeSchemaVersion left,
        ComposeSchemaVersion right) => left.CompareTo(right) <= 0;
}

SchemaDownloader  →  SchemaReader  →  SchemaVersionMerger  →  CodeEmitter
(32 JSON files)     (32 SchemaModels)  (1 UnifiedSchema)     (~83 .g.cs files)

public void Initialize(IncrementalGeneratorInitializationContext context)
{
    var schemas = context.AdditionalTextsProvider
        .Where(t => t.Path.EndsWith(".schema.json"))
        .Select((text, ct) =>
        {
            var version = ExtractVersion(text.Path);
            var content = text.GetText(ct)!.ToString();
            var schema = SchemaReader.Read(content);
            return (Version: version, Schema: schema);
        })
        .Collect();

    context.RegisterSourceOutput(schemas, (ctx, versions) =>
    {
        var unified = SchemaVersionMerger.Merge(versions);
        CodeEmitter.Emit(ctx, unified);
    });
}

[Theory]
[InlineData("image", null, null)]
[InlineData("develop", "1.19.0", null)]
[InlineData("provider", "2.5.0", null)]
[InlineData("gpus", null, "1.8.0")]
[InlineData("post_start", "2.3.0", null)]
[InlineData("models", "2.7.1", null)]
public void ComposeService_PropertyVersionBounds(
    string propertyName,
    string? expectedSince,
    string? expectedUntil)
{
    var unified = SchemaVersionMerger.Merge(_allVersions);
    var service = unified.Definitions["service"];
    var prop = service.Properties
        .Single(p => p.Name == propertyName);

    Assert.Equal(
        expectedSince is null ? null
            : ComposeSchemaVersion.Parse(expectedSince),
        prop.SinceVersion);
    Assert.Equal(
        expectedUntil is null ? null
            : ComposeSchemaVersion.Parse(expectedUntil),
        prop.UntilVersion);
}

[Fact]
public void Properties_AreMonotonicallyPresent()
{
    var sorted = _allVersions.OrderBy(v => v.Version).ToList();

    foreach (var defName in _unified.Definitions.Keys)
    {
        foreach (var prop in _unified.Definitions[defName].Properties)
        {
            bool appeared = false;
            bool disappeared = false;

            foreach (var (version, schema) in sorted)
            {
                var hasProp = schema.Definitions
                    .TryGetValue(defName, out var def)
                    && def.Properties.Any(p => p.Name == prop.Name);

                if (hasProp && !appeared) appeared = true;
                if (!hasProp && appeared && !disappeared)
                    disappeared = true;
                if (hasProp && disappeared)
                    Assert.Fail(
                        $"{defName}.{prop.Name} reappeared " +
                        $"in {version} after disappearing");
            }
        }
    }
}

[Theory]
[MemberData(nameof(AllVersions))]
public void RoundTrip_PerVersion(ComposeSchemaVersion version)
{
    var yaml = LoadTestYaml(version);

    // Deserialize with per-version types (ground truth)
    var perVersion = PerVersionDeserializer.Deserialize(
        yaml, version);

    // Deserialize with unified types
    var unified = UnifiedDeserializer.Deserialize(yaml);

    // Compare all properties that exist in this version
    AssertEquivalent(perVersion, unified, version);
}

// What the developer writes -- no knowledge of 32 schemas needed
var compose = new ComposeFileBuilder()
    .WithService("api", s => s
        .WithImage("myapp:latest")
        .WithBuild(b => b
            .WithContext(".")
            .WithDockerfile("Dockerfile")
            .WithTarget("runtime"))
        .WithPorts(p => p.Add("8080:80"))
        .WithDevelop(d => d          // IDE: [SinceVersion("1.19.0")]
            .WithWatch(w => w
                .WithPath("./src")
                .WithAction("rebuild")))
        .WithDependsOn("db", dep => dep
            .WithCondition("service_healthy")
            .WithRestart(true)))       // IDE: [SinceVersion("2.3.0")]
    .Build();