Ops.Chaos -- Fault Injection Across Three Tiers📋

// =================================================================
// Ops.Chaos.Lib -- Chaos Engineering DSL Attributes
// =================================================================

/// The kind of fault to inject.
public enum FaultKind
{
    Timeout,              // delay exceeds timeout threshold
    Exception,            // throw a specific exception type
    Latency,              // add random delay (not necessarily timeout)
    PacketLoss,           // drop N% of network packets
    ProcessKill,          // kill a container/process
    CpuStress,            // consume CPU cycles
    MemoryStress,         // allocate memory until pressure
    DiskFull,             // fill disk to capacity
    DnsFailure,           // DNS resolution fails
    DependencyTimeout,    // downstream service times out
    ClockSkew,            // system clock drifts
    DataCorruption        // garble response payloads
}

/// Declares a chaos experiment.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)]
public sealed class ChaosExperimentAttribute : Attribute
{
    public string Name { get; }
    public OpsExecutionTier Tier { get; init; } = OpsExecutionTier.InProcess;
    public string Hypothesis { get; init; } = "";
    public string Owner { get; init; } = "";
    public string Schedule { get; init; } = "";

    public ChaosExperimentAttribute(string name) => Name = name;
}

/// Identifies the service or interface under attack.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class TargetServiceAttribute : Attribute
{
    public Type ServiceType { get; }
    public string Instance { get; init; } = "";

    public TargetServiceAttribute(Type serviceType) => ServiceType = serviceType;
}

/// Defines the fault to inject.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class FaultInjectionAttribute : Attribute
{
    public FaultKind Kind { get; }
    public double Probability { get; init; } = 1.0;
    public int DurationSeconds { get; init; } = 30;
    public int DelayMs { get; init; } = 0;
    public double BlastRadius { get; init; } = 1.0;
    public string ExceptionType { get; init; } = "";

    public FaultInjectionAttribute(FaultKind kind) => Kind = kind;
}

/// A metric that must remain within bounds during the experiment.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class SteadyStateProbeAttribute : Attribute
{
    public string Metric { get; }
    public string Operator { get; init; } = "<=";
    public double Expected { get; }
    public int ProbeIntervalSeconds { get; init; } = 5;

    public SteadyStateProbeAttribute(string metric, double expected)
    {
        Metric = metric;
        Expected = expected;
    }
}

/// Emergency stop: if this threshold is breached, abort immediately.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class AbortConditionAttribute : Attribute
{
    public string Metric { get; }
    public double Threshold { get; }
    public string Action { get; init; } = "abort";

    public AbortConditionAttribute(string metric, double threshold)
    {
        Metric = metric;
        Threshold = threshold;
    }
}

/// Container-tier: declare a Toxiproxy for network fault injection.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class ToxiProxyAttribute : Attribute
{
    public string Name { get; }
    public string Upstream { get; }
    public int ListenPort { get; init; }
    public int UpstreamPort { get; init; }

    public ToxiProxyAttribute(string name, string upstream)
    {
        Name = name;
        Upstream = upstream;
    }
}

/// Container-tier: declare a Docker container dependency.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class ContainerAttribute : Attribute
{
    public string Name { get; }
    public string Image { get; }
    public int Port { get; init; }
    public string[] Environment { get; init; } = [];

    public ContainerAttribute(string name, string image)
    {
        Name = name;
        Image = image;
    }
}

/// Cloud-tier: declare the cloud provider for experiment execution.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)]
public sealed class CloudProviderAttribute : Attribute
{
    public string Provider { get; }
    public string Region { get; init; } = "";
    public string SubscriptionId { get; init; } = "";

    public CloudProviderAttribute(string provider) => Provider = provider;
}

[ChaosExperiment("payment-timeout",
    Tier = OpsExecutionTier.InProcess,
    Hypothesis = "When IPaymentGateway.ChargeAsync times out, the circuit breaker " +
                 "trips within 10 seconds and OrderService returns a 503 with retry-after header")]
[TargetService(typeof(IPaymentGateway))]
[FaultInjection(FaultKind.Timeout,
    Probability = 1.0,
    DurationSeconds = 30,
    DelayMs = 35_000)]  // 35s delay against a 30s timeout
[SteadyStateProbe("order_error_rate", 0.05,
    Operator = "<=", ProbeIntervalSeconds = 2)]
[SteadyStateProbe("circuit_breaker_state", 1.0,
    Operator = "==")]  // 1.0 = Open
[AbortCondition("order_error_rate", 0.50)]
public class PaymentTimeoutChaos { }

// <auto-generated by Ops.Chaos.Generator />
// Experiment: payment-timeout
// Target: IPaymentGateway
// Fault: Timeout (P=1.0, Delay=35000ms)

using System.Diagnostics;

public sealed class PaymentGatewayChaosDecorator : IPaymentGateway
{
    private readonly IPaymentGateway _inner;
    private readonly IChaosConfiguration _config;
    private readonly IMeterFactory _meterFactory;
    private readonly ILogger<PaymentGatewayChaosDecorator> _logger;

    private static readonly Meter s_meter = new("Ops.Chaos.PaymentTimeout");
    private static readonly Counter<long> s_faultsInjected =
        s_meter.CreateCounter<long>("chaos.faults_injected");
    private static readonly Counter<long> s_faultsSkipped =
        s_meter.CreateCounter<long>("chaos.faults_skipped");

    public PaymentGatewayChaosDecorator(
        IPaymentGateway inner,
        IChaosConfiguration config,
        IMeterFactory meterFactory,
        ILogger<PaymentGatewayChaosDecorator> logger)
    {
        _inner = inner;
        _config = config;
        _meterFactory = meterFactory;
        _logger = logger;
    }

    public async Task<ChargeResult> ChargeAsync(
        PaymentRequest request, CancellationToken ct = default)
    {
        if (!_config.IsExperimentEnabled("payment-timeout"))
            return await _inner.ChargeAsync(request, ct);

        if (Random.Shared.NextDouble() > 1.0) // Probability = 1.0
        {
            s_faultsSkipped.Add(1);
            return await _inner.ChargeAsync(request, ct);
        }

        // Inject fault: Timeout after 35000ms
        _logger.LogWarning(
            "Chaos: injecting Timeout fault on IPaymentGateway.ChargeAsync " +
            "(experiment=payment-timeout, delay=35000ms)");
        s_faultsInjected.Add(1);

        await Task.Delay(TimeSpan.FromMilliseconds(35_000), ct);

        // The caller's CancellationToken or Polly timeout should cancel
        // before this line is reached. If it does not, that is a finding.
        return await _inner.ChargeAsync(request, ct);
    }

    public async Task<RefundResult> RefundAsync(
        RefundRequest request, CancellationToken ct = default)
    {
        // Same fault injection logic for all interface methods
        if (!_config.IsExperimentEnabled("payment-timeout"))
            return await _inner.RefundAsync(request, ct);

        if (Random.Shared.NextDouble() > 1.0)
        {
            s_faultsSkipped.Add(1);
            return await _inner.RefundAsync(request, ct);
        }

        _logger.LogWarning(
            "Chaos: injecting Timeout fault on IPaymentGateway.RefundAsync " +
            "(experiment=payment-timeout, delay=35000ms)");
        s_faultsInjected.Add(1);

        await Task.Delay(TimeSpan.FromMilliseconds(35_000), ct);
        return await _inner.RefundAsync(request, ct);
    }
}

/// Runtime configuration for chaos experiments.
public interface IChaosConfiguration
{
    bool IsExperimentEnabled(string experimentName);
    void EnableExperiment(string experimentName);
    void DisableExperiment(string experimentName);
    void DisableAll();
}

// <auto-generated by Ops.Chaos.Generator />
public static class PaymentTimeoutChaosRegistration
{
    public static IServiceCollection AddPaymentTimeoutChaos(this IServiceCollection services)
    {
        services.AddSingleton<IChaosConfiguration, InMemoryChaosConfiguration>();
        services.Decorate<IPaymentGateway, PaymentGatewayChaosDecorator>();
        return services;
    }
}

[Fact]
public async Task PaymentTimeout_CircuitBreaker_Trips_Within_10_Seconds()
{
    // Arrange
    var services = new ServiceCollection();
    services.AddSingleton<IPaymentGateway, RealPaymentGateway>();
    services.AddPaymentTimeoutChaos();
    services.AddPollyCircuitBreaker<IPaymentGateway>(options =>
    {
        options.TimeoutSeconds = 30;
        options.FailureThreshold = 3;
        options.BreakDurationSeconds = 60;
    });

    var provider = services.BuildServiceProvider();
    var config = provider.GetRequiredService<IChaosConfiguration>();
    var gateway = provider.GetRequiredService<IPaymentGateway>();

    // Act: enable chaos and send requests
    config.EnableExperiment("payment-timeout");
    var sw = Stopwatch.StartNew();

    var tasks = Enumerable.Range(0, 5).Select(async _ =>
    {
        try
        {
            await gateway.ChargeAsync(new PaymentRequest { Amount = 100 });
        }
        catch (BrokenCircuitException)
        {
            // Expected after the circuit breaker trips
        }
        catch (TimeoutRejectedException)
        {
            // Expected during the timeout phase
        }
    });

    await Task.WhenAll(tasks);
    sw.Stop();

    // Assert: circuit breaker tripped (steady-state probe: circuit_breaker_state == Open)
    var breaker = provider.GetRequiredService<ICircuitBreakerStateProvider>();
    Assert.Equal(CircuitState.Open, breaker.GetState<IPaymentGateway>());

    // Assert: it happened within 10 seconds (not 35s * 5 = 175s)
    Assert.True(sw.Elapsed < TimeSpan.FromSeconds(10),
        $"Circuit breaker took {sw.Elapsed.TotalSeconds:F1}s to trip, expected < 10s");

    config.DisableAll();
}

[ChaosExperiment("database-partition",
    Tier = OpsExecutionTier.Container,
    Hypothesis = "When the database connection is severed, read replicas serve " +
                 "stale data and writes queue for retry. No data loss occurs.")]
[TargetService(typeof(IOrderRepository))]
[FaultInjection(FaultKind.PacketLoss,
    Probability = 1.0,
    DurationSeconds = 120)]
[SteadyStateProbe("orders_created_count", 0,
    Operator = ">=")]
[SteadyStateProbe("data_loss_events", 0,
    Operator = "==")]
[AbortCondition("unhandled_exception_count", 10)]

// Container infrastructure
[ToxiProxy("postgres-proxy", "postgres",
    ListenPort = 5433, UpstreamPort = 5432)]
[Container("postgres", "postgres:16-alpine", Port = 5432,
    Environment = ["POSTGRES_DB=orders", "POSTGRES_PASSWORD=test"])]
[Container("order-api", "order-service:latest", Port = 5000,
    Environment = ["ConnectionStrings__Orders=Host=toxiproxy;Port=5433;Database=orders"])]

public class DatabasePartitionChaos { }

# Auto-generated by Ops.Chaos.Generator
# Experiment: database-partition

services:
  postgres:
    image: postgres:16-alpine
    ports:
      - "5432:5432"
    environment:
      POSTGRES_DB: orders
      POSTGRES_PASSWORD: test
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      timeout: 3s
      retries: 5

  toxiproxy:
    image: ghcr.io/shopify/toxiproxy:2.9.0
    ports:
      - "5433:5433"
      - "8474:8474"  # Toxiproxy API
    depends_on:
      postgres:
        condition: service_healthy

  order-api:
    image: order-service:latest
    ports:
      - "5000:5000"
    environment:
      ConnectionStrings__Orders: "Host=toxiproxy;Port=5433;Database=orders;Username=postgres;Password=test"
    depends_on:
      toxiproxy:
        condition: service_started
      postgres:
        condition: service_healthy

// <auto-generated by Ops.Chaos.Generator />
// Experiment: database-partition

public sealed class DatabasePartitionToxiProxyClient : IAsyncDisposable
{
    private readonly HttpClient _http;
    private readonly string _proxyName;

    public DatabasePartitionToxiProxyClient(string toxiproxyHost = "localhost", int port = 8474)
    {
        _http = new HttpClient { BaseAddress = new Uri($"http://{toxiproxyHost}:{port}") };
        _proxyName = "postgres-proxy";
    }

    public async Task CreateProxyAsync()
    {
        var proxy = new
        {
            name = _proxyName,
            listen = "0.0.0.0:5433",
            upstream = "postgres:5432",
            enabled = true,
        };

        var response = await _http.PostAsJsonAsync("/proxies", proxy);
        response.EnsureSuccessStatusCode();
    }

    /// Inject 100% packet loss — complete network partition.
    public async Task InjectPacketLossAsync(double toxicity = 1.0)
    {
        var toxic = new
        {
            name = "packet_loss_downstream",
            type = "limit_data",
            stream = "downstream",
            toxicity,
            attributes = new { bytes = 0 },
        };

        await _http.PostAsJsonAsync($"/proxies/{_proxyName}/toxics", toxic);
    }

    /// Remove all toxics — restore normal connectivity.
    public async Task RemoveAllToxicsAsync()
    {
        var response = await _http.GetAsync($"/proxies/{_proxyName}/toxics");
        var toxics = await response.Content.ReadFromJsonAsync<JsonArray>();

        if (toxics is not null)
        {
            foreach (var toxic in toxics)
            {
                var name = toxic!["name"]!.GetValue<string>();
                await _http.DeleteAsync($"/proxies/{_proxyName}/toxics/{name}");
            }
        }
    }

    /// Disable the proxy entirely — hard cut.
    public async Task DisableProxyAsync()
    {
        var payload = new { enabled = false };
        await _http.PostAsJsonAsync($"/proxies/{_proxyName}", payload);
    }

    public async Task EnableProxyAsync()
    {
        var payload = new { enabled = true };
        await _http.PostAsJsonAsync($"/proxies/{_proxyName}", payload);
    }

    public async ValueTask DisposeAsync()
    {
        try { await RemoveAllToxicsAsync(); } catch { /* cleanup best effort */ }
        _http.Dispose();
    }
}

// <auto-generated by Ops.Chaos.Generator />
// Experiment: database-partition

public sealed class DatabasePartitionChaosFixture : IAsyncLifetime
{
    private readonly string _composeFile;
    public DatabasePartitionToxiProxyClient ToxiProxy { get; private set; } = null!;
    public HttpClient OrderApi { get; private set; } = null!;

    public DatabasePartitionChaosFixture()
    {
        _composeFile = Path.Combine(
            AppContext.BaseDirectory, "chaos", "docker-compose.chaos.yaml");
    }

    public async Task InitializeAsync()
    {
        // Start infrastructure
        await RunAsync("docker", $"compose -f {_composeFile} up -d --wait");

        // Wait for services
        await WaitForHealthy("http://localhost:5000/health", timeout: TimeSpan.FromSeconds(30));

        // Create ToxiProxy proxy
        ToxiProxy = new DatabasePartitionToxiProxyClient();
        await ToxiProxy.CreateProxyAsync();

        // Create API client
        OrderApi = new HttpClient { BaseAddress = new Uri("http://localhost:5000") };
    }

    public async Task DisposeAsync()
    {
        await ToxiProxy.DisposeAsync();
        OrderApi.Dispose();
        await RunAsync("docker", $"compose -f {_composeFile} down -v");
    }

    private static async Task RunAsync(string command, string args)
    {
        var psi = new ProcessStartInfo(command, args)
        {
            RedirectStandardOutput = true,
            RedirectStandardError = true,
        };
        var process = Process.Start(psi)!;
        await process.WaitForExitAsync();
        if (process.ExitCode != 0)
            throw new InvalidOperationException(
                $"{command} {args} exited with code {process.ExitCode}");
    }

    private static async Task WaitForHealthy(string url, TimeSpan timeout)
    {
        using var http = new HttpClient();
        var deadline = DateTime.UtcNow + timeout;
        while (DateTime.UtcNow < deadline)
        {
            try
            {
                var response = await http.GetAsync(url);
                if (response.IsSuccessStatusCode) return;
            }
            catch { /* retry */ }
            await Task.Delay(1000);
        }
        throw new TimeoutException($"{url} did not become healthy within {timeout}");
    }
}

[CollectionDefinition("DatabasePartitionChaos")]
public class DatabasePartitionChaosCollection
    : ICollectionFixture<DatabasePartitionChaosFixture> { }

[Collection("DatabasePartitionChaos")]
public class DatabasePartitionChaosTests
{
    private readonly DatabasePartitionChaosFixture _fixture;

    public DatabasePartitionChaosTests(DatabasePartitionChaosFixture fixture)
        => _fixture = fixture;

    [Fact]
    public async Task Database_Partition_Does_Not_Cause_Data_Loss()
    {
        // 1. Create an order (baseline)
        var createResponse = await _fixture.OrderApi.PostAsJsonAsync("/api/orders", new
        {
            customerId = "cust-001",
            items = new[] { new { productId = "prod-001", quantity = 1 } },
        });
        Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode);
        var orderId = (await createResponse.Content.ReadFromJsonAsync<JsonObject>())!["id"]!.ToString();

        // 2. Inject network partition
        await _fixture.ToxiProxy.InjectPacketLossAsync(toxicity: 1.0);

        // 3. Attempt writes during partition — should queue or return 503
        var writeResponse = await _fixture.OrderApi.PostAsJsonAsync("/api/orders", new
        {
            customerId = "cust-002",
            items = new[] { new { productId = "prod-002", quantity = 1 } },
        });
        // Accept 503 (Service Unavailable) or 202 (Accepted, queued for retry)
        Assert.True(
            writeResponse.StatusCode is HttpStatusCode.ServiceUnavailable
                or HttpStatusCode.Accepted,
            $"Expected 503 or 202, got {writeResponse.StatusCode}");

        // 4. Reads should still work (read replica or cache)
        var readResponse = await _fixture.OrderApi.GetAsync($"/api/orders/{orderId}");
        Assert.Equal(HttpStatusCode.OK, readResponse.StatusCode);

        // 5. Restore connectivity
        await _fixture.ToxiProxy.RemoveAllToxicsAsync();
        await Task.Delay(5000); // allow retry queue to drain

        // 6. Verify no data loss — the queued write should have completed
        if (writeResponse.StatusCode == HttpStatusCode.Accepted)
        {
            var queuedOrderId = writeResponse.Headers.Location?.Segments.Last();
            var verifyResponse = await _fixture.OrderApi.GetAsync($"/api/orders/{queuedOrderId}");
            Assert.Equal(HttpStatusCode.OK, verifyResponse.StatusCode);
        }
    }
}

[ChaosExperiment("azure-region-failover",
    Tier = OpsExecutionTier.Cloud,
    Hypothesis = "When the primary Azure region fails, traffic manager routes to " +
                 "secondary within 60 seconds. No requests are lost during failover.",
    Owner = "platform-team",
    Schedule = "0 3 * * 0")]  // 3 AM every Sunday
[TargetService(typeof(IOrderService))]
[FaultInjection(FaultKind.ProcessKill,
    DurationSeconds = 300,
    BlastRadius = 1.0)]  // kill the entire primary region
[SteadyStateProbe("http_success_rate", 0.95, Operator = ">=")]
[SteadyStateProbe("failover_duration_seconds", 60, Operator = "<=")]
[AbortCondition("http_success_rate", 0.50)]
[CloudProvider("azure", Region = "westeurope", SubscriptionId = "sub-prod-001")]

public class AzureRegionFailoverChaos { }

# Auto-generated by Ops.Chaos.Generator
# Experiment: azure-region-failover
# Schedule: Sundays at 03:00 UTC

variable "subscription_id" {
  type    = string
  default = "sub-prod-001"
}

variable "primary_region" {
  type    = string
  default = "westeurope"
}

variable "resource_group_name" {
  type = string
}

variable "app_service_name" {
  type = string
}

# Stop the primary region App Service to simulate region failure
resource "null_resource" "stop_primary" {
  triggers = {
    experiment = "azure-region-failover"
    timestamp  = timestamp()
  }

  provisioner "local-exec" {
    command = <<-EOT
      az webapp stop \
        --name ${var.app_service_name} \
        --resource-group ${var.resource_group_name} \
        --subscription ${var.subscription_id}
    EOT
  }
}

# Monitor: probe the traffic manager endpoint every 5 seconds
resource "null_resource" "steady_state_probe" {
  depends_on = [null_resource.stop_primary]

  provisioner "local-exec" {
    command = <<-EOT
      #!/bin/bash
      START=$(date +%s)
      FAILOVER_DETECTED=false
      ABORT=false

      for i in $(seq 1 60); do
        RESPONSE=$(curl -s -o /dev/null -w "%%{http_code}" \
          https://order-service.trafficmanager.net/health)

        if [ "$RESPONSE" = "200" ] && [ "$FAILOVER_DETECTED" = false ]; then
          FAILOVER_DETECTED=true
          NOW=$(date +%s)
          DURATION=$((NOW - START))
          echo "Failover detected after ${DURATION}s"

          if [ $DURATION -gt 60 ]; then
            echo "FAIL: failover took ${DURATION}s, budget is 60s"
            exit 1
          fi
        fi

        # Abort condition: success rate below 50% after 120s
        if [ $i -gt 24 ] && [ "$FAILOVER_DETECTED" = false ]; then
          echo "ABORT: no failover after 120s, success rate critical"
          ABORT=true
          break
        fi

        sleep 5
      done

      if [ "$FAILOVER_DETECTED" = false ]; then
        echo "FAIL: failover never completed"
        exit 1
      fi
    EOT
  }
}

# Restore: restart the primary region after experiment
resource "null_resource" "restore_primary" {
  depends_on = [null_resource.steady_state_probe]

  provisioner "local-exec" {
    command = <<-EOT
      az webapp start \
        --name ${var.app_service_name} \
        --resource-group ${var.resource_group_name} \
        --subscription ${var.subscription_id}
    EOT
  }
}

# Auto-generated by Ops.Chaos.Generator
# Experiment: azure-region-failover (Kubernetes variant)

apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: azure-region-failover
  namespace: order-service
  labels:
    ops.chaos/experiment: azure-region-failover
    ops.chaos/tier: cloud
    ops.chaos/owner: platform-team
spec:
  appinfo:
    appns: order-service
    applabel: app=order-service
    appkind: deployment
  engineState: active
  chaosServiceAccount: litmus-admin
  monitoring: true

  experiments:
    - name: pod-delete
      spec:
        probe:
          - name: http-success-rate
            type: httpProbe
            mode: Continuous
            runProperties:
              probeTimeout: 5
              retry: 60
              interval: 5
            httpProbe/inputs:
              url: https://order-service.trafficmanager.net/health
              method:
                get:
                  criteria: ==
                  responseCode: "200"
          - name: failover-duration
            type: cmdProbe
            mode: OnChaos
            runProperties:
              probeTimeout: 60
              retry: 1
            cmdProbe/inputs:
              command: ./scripts/measure-failover-duration.sh
              comparator:
                type: int
                criteria: "<="
                value: "60"

        components:
          env:
            - name: TOTAL_CHAOS_DURATION
              value: "300"
            - name: CHAOS_INTERVAL
              value: "10"
            - name: FORCE
              value: "true"
            - name: PODS_AFFECTED_PERC
              value: "100"  # BlastRadius = 1.0

  annotationCheck: "true"

---
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosSchedule
metadata:
  name: azure-region-failover-schedule
  namespace: order-service
spec:
  schedule:
    repeat:
      properties:
        minChaosInterval:
          hour:
            everyNthHour: 168  # weekly
        startTime:
          hour: 3
          weekDay: "Sun"
  engineTemplateSpec:
    engineState: active
    chaosServiceAccount: litmus-admin
    experiments:
      - name: pod-delete

// Resilience DSL declares the circuit breaker
[CircuitBreaker(typeof(IPaymentGateway),
    TimeoutSeconds = 30, FailureThreshold = 3, BreakDurationSeconds = 60)]

// Chaos DSL tests it — CHS001 fires if this experiment does not exist
[ChaosExperiment("payment-timeout", Tier = OpsExecutionTier.InProcess)]
[TargetService(typeof(IPaymentGateway))]
[FaultInjection(FaultKind.Timeout, DelayMs = 35_000)]

[SteadyStateProbe("http_success_rate", 0.95)]
// Requires: [OpsMetric("http_success_rate", MetricKind.Gauge)] in Observability DSL

[ChaosExperiment("payment-timeout")]
[OpsRequirementLink("FEATURE-456-AC3",
    "System must handle payment gateway timeouts without data loss")]

// Load test injects traffic
[LoadTest("order-spike", Tier = OpsExecutionTier.Container)]
[LoadProfile(ConcurrentUsers = 100)]
[TrafficPattern(TrafficShape.Spike, PeakMultiplier = 5.0)]

// Chaos experiment injects faults during the load test
[ChaosExperiment("payment-timeout-under-load", Tier = OpsExecutionTier.Container)]
[FaultInjection(FaultKind.Latency, DelayMs = 5000, Probability = 0.3)]