Ops.Observability -- Health, Metrics, Alerts, Dashboards📋

// =================================================================
// Ops.Observability.Lib -- Health, Metrics, Alerts, Dashboards DSL
// =================================================================

/// Declare a health check endpoint for a service.
/// The generator registers it with ASP.NET health checks (InProcess),
/// Docker HEALTHCHECK (Container), and monitoring probes (Cloud).
[AttributeUsage(AttributeTargets.Method)]
public sealed class HealthCheckAttribute : Attribute
{
    public string Name { get; }
    public string Endpoint { get; init; } = "/health";
    public string Timeout { get; init; } = "00:00:05";
    public int Retries { get; init; } = 3;
    public string[] Tags { get; init; } = [];
    public HealthCheckKind Kind { get; init; } = HealthCheckKind.Readiness;

    public HealthCheckAttribute(string name) => Name = name;
}

public enum HealthCheckKind
{
    Readiness,     // can accept traffic
    Liveness,      // process is alive (not deadlocked)
    Startup        // initialization complete
}

/// Declare a metric that the application exposes.
/// The generator creates OpenTelemetry meter registrations (InProcess),
/// Prometheus scrape targets (Container), and CloudWatch/Datadog
/// metric definitions (Cloud).
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = true)]
public sealed class MetricAttribute : Attribute
{
    public string Name { get; }
    public MetricKind Kind { get; init; } = MetricKind.Counter;
    public string Unit { get; init; } = "";
    public string Description { get; init; } = "";
    public string[] Labels { get; init; } = [];
    public double[] HistogramBuckets { get; init; } = [];

    public MetricAttribute(string name) => Name = name;
}

public enum MetricKind
{
    Counter,       // monotonically increasing (request count, error count)
    Gauge,         // current value (queue depth, active connections)
    Histogram,     // distribution (latency, request size)
    Summary        // quantiles (p50, p95, p99)
}

/// Declare an alert rule that fires when a metric crosses a threshold.
/// The generator emits Prometheus alerting rules (Container) and
/// CloudWatch alarms (Cloud).
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class AlertRuleAttribute : Attribute
{
    public string Name { get; }
    public string Metric { get; }
    public string Condition { get; }
    public AlertSeverity Severity { get; init; } = AlertSeverity.Warning;
    public string[] NotifyChannels { get; init; } = [];
    public string Runbook { get; init; } = "";
    public string For { get; init; } = "5m";           // duration before firing
    public string Summary { get; init; } = "";
    public string Description { get; init; } = "";

    public AlertRuleAttribute(string name, string metric, string condition)
    {
        Name = name;
        Metric = metric;
        Condition = condition;
    }
}

public enum AlertSeverity
{
    Info,          // informational, no page
    Warning,       // needs attention during business hours
    Critical,      // pages on-call immediately
    Fatal          // all-hands incident
}

/// Declare a Grafana dashboard. Panels are declared with [DashboardPanel].
[AttributeUsage(AttributeTargets.Class)]
public sealed class DashboardAttribute : Attribute
{
    public string Name { get; }
    public string Folder { get; init; } = "Generated";
    public string[] Tags { get; init; } = [];
    public string RefreshInterval { get; init; } = "30s";

    public DashboardAttribute(string name) => Name = name;
}

/// Declare a panel within a dashboard.
/// The generator produces the Grafana JSON panel definition.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class DashboardPanelAttribute : Attribute
{
    public string Title { get; }
    public string Metric { get; }
    public PanelVisualization Visualization { get; init; } = PanelVisualization.TimeSeries;
    public int GridX { get; init; } = 0;
    public int GridY { get; init; } = 0;
    public int Width { get; init; } = 12;
    public int Height { get; init; } = 8;
    public string Legend { get; init; } = "";
    public string[] GroupBy { get; init; } = [];

    public DashboardPanelAttribute(string title, string metric)
    {
        Title = title;
        Metric = metric;
    }
}

public enum PanelVisualization
{
    TimeSeries,    // line chart over time
    Gauge,         // single value with thresholds
    BarChart,      // categorical comparison
    Stat,          // big number
    Table,         // tabular data
    Heatmap        // 2D histogram
}

// -- OrderServiceObservability.cs -----------------------------------

[Dashboard("Order Service",
    Folder = "Services",
    Tags = ["orders", "payments"],
    RefreshInterval = "15s")]
[DashboardPanel("Request Latency (p95)",
    "order_api_request_duration_seconds",
    Visualization = PanelVisualization.TimeSeries,
    GridX = 0, GridY = 0, Width = 12, Height = 8,
    GroupBy = ["endpoint", "status_code"])]
[DashboardPanel("Error Rate",
    "order_api_http_errors_total",
    Visualization = PanelVisualization.TimeSeries,
    GridX = 12, GridY = 0, Width = 12, Height = 8)]
[DashboardPanel("Active Orders in Queue",
    "order_queue_depth",
    Visualization = PanelVisualization.Gauge,
    GridX = 0, GridY = 8, Width = 8, Height = 6)]
[DashboardPanel("Payment Success Rate",
    "order_payment_success_ratio",
    Visualization = PanelVisualization.Stat,
    GridX = 8, GridY = 8, Width = 8, Height = 6)]
[DashboardPanel("Latency Heatmap",
    "order_api_request_duration_seconds",
    Visualization = PanelVisualization.Heatmap,
    GridX = 16, GridY = 8, Width = 8, Height = 6)]
[AlertRule("OrderHighLatency",
    "order_api_request_duration_seconds",
    "histogram_quantile(0.95, rate(order_api_request_duration_seconds_bucket[5m])) > 0.5",
    Severity = AlertSeverity.Warning,
    For = "5m",
    NotifyChannels = ["#order-alerts", "oncall-orders"],
    Runbook = "https://docs.internal/runbooks/order-high-latency",
    Summary = "Order API p95 latency above 500ms",
    Description = "The 95th percentile latency for the Order API has exceeded " +
                  "500ms for 5 minutes. Check for database connection pool exhaustion " +
                  "or downstream payment service degradation.")]
[AlertRule("OrderHighErrorRate",
    "order_api_http_errors_total",
    "rate(order_api_http_errors_total[5m]) / rate(order_api_http_requests_total[5m]) > 0.05",
    Severity = AlertSeverity.Critical,
    For = "3m",
    NotifyChannels = ["#order-alerts", "oncall-orders", "engineering-leads"],
    Runbook = "https://docs.internal/runbooks/order-error-rate",
    Summary = "Order API error rate above 5%",
    Description = "More than 5% of Order API requests are failing. " +
                  "Immediate investigation required.")]
public sealed class OrderServiceObservability
{
    [HealthCheck("order-api-ready",
        Endpoint = "/health/ready",
        Timeout = "00:00:05",
        Retries = 3,
        Tags = ["order", "api"],
        Kind = HealthCheckKind.Readiness)]
    public void OrderApiReadiness() { }

    [HealthCheck("order-api-live",
        Endpoint = "/health/live",
        Timeout = "00:00:03",
        Retries = 1,
        Kind = HealthCheckKind.Liveness)]
    public void OrderApiLiveness() { }

    [HealthCheck("order-payment-check",
        Endpoint = "/health/payment",
        Timeout = "00:00:10",
        Retries = 2,
        Tags = ["order", "payment", "downstream"],
        Kind = HealthCheckKind.Readiness)]
    public void PaymentDownstreamHealth() { }

    [Metric("order_api_request_duration_seconds",
        Kind = MetricKind.Histogram,
        Unit = "seconds",
        Description = "HTTP request duration for Order API endpoints",
        Labels = ["endpoint", "method", "status_code"],
        HistogramBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0])]
    public void RequestLatency() { }

    [Metric("order_api_http_errors_total",
        Kind = MetricKind.Counter,
        Unit = "requests",
        Description = "Total HTTP error responses (4xx + 5xx)",
        Labels = ["endpoint", "status_code"])]
    public void ErrorCount() { }

    [Metric("order_api_http_requests_total",
        Kind = MetricKind.Counter,
        Unit = "requests",
        Description = "Total HTTP requests processed",
        Labels = ["endpoint", "method", "status_code"])]
    public void RequestCount() { }

    [Metric("order_queue_depth",
        Kind = MetricKind.Gauge,
        Unit = "messages",
        Description = "Number of orders waiting in the processing queue")]
    public void QueueDepth() { }

    [Metric("order_payment_success_ratio",
        Kind = MetricKind.Gauge,
        Unit = "ratio",
        Description = "Ratio of successful payment transactions (0.0 to 1.0)")]
    public void PaymentSuccessRate() { }
}

// <auto-generated by Ops.Observability.Generators />
namespace Ops.Observability.Generated;

public static class HealthCheckRegistration
{
    public static IHealthChecksBuilder AddGeneratedHealthChecks(
        this IHealthChecksBuilder builder)
    {
        builder.AddCheck("order-api-ready",
            new HttpHealthCheck(
                endpoint: "/health/ready",
                timeout: TimeSpan.FromSeconds(5),
                retries: 3),
            tags: ["order", "api"],
            failureStatus: HealthStatus.Unhealthy);

        builder.AddCheck("order-api-live",
            new HttpHealthCheck(
                endpoint: "/health/live",
                timeout: TimeSpan.FromSeconds(3),
                retries: 1),
            tags: [],
            failureStatus: HealthStatus.Unhealthy);

        builder.AddCheck("order-payment-check",
            new HttpHealthCheck(
                endpoint: "/health/payment",
                timeout: TimeSpan.FromSeconds(10),
                retries: 2),
            tags: ["order", "payment", "downstream"],
            failureStatus: HealthStatus.Degraded);

        return builder;
    }
}

// <auto-generated by Ops.Observability.Generators />
namespace Ops.Observability.Generated;

public static class MetricsRegistration
{
    private static readonly Meter OrderServiceMeter = new("OrderService", "2.4.0");

    public static readonly Histogram<double> RequestDuration =
        OrderServiceMeter.CreateHistogram<double>(
            "order_api_request_duration_seconds",
            unit: "seconds",
            description: "HTTP request duration for Order API endpoints");

    public static readonly Counter<long> HttpErrors =
        OrderServiceMeter.CreateCounter<long>(
            "order_api_http_errors_total",
            unit: "requests",
            description: "Total HTTP error responses (4xx + 5xx)");

    public static readonly Counter<long> HttpRequests =
        OrderServiceMeter.CreateCounter<long>(
            "order_api_http_requests_total",
            unit: "requests",
            description: "Total HTTP requests processed");

    public static readonly ObservableGauge<int> QueueDepth =
        OrderServiceMeter.CreateObservableGauge<int>(
            "order_queue_depth",
            observeValue: () => OrderQueueMetrics.CurrentDepth,
            unit: "messages",
            description: "Number of orders waiting in the processing queue");

    public static readonly ObservableGauge<double> PaymentSuccessRatio =
        OrderServiceMeter.CreateObservableGauge<double>(
            "order_payment_success_ratio",
            observeValue: () => PaymentMetrics.SuccessRatio,
            unit: "ratio",
            description: "Ratio of successful payment transactions (0.0 to 1.0)");

    public static IServiceCollection AddGeneratedMetrics(
        this IServiceCollection services)
    {
        services.AddOpenTelemetry()
            .WithMetrics(builder => builder
                .AddMeter("OrderService")
                .AddPrometheusExporter());
        return services;
    }
}

# <auto-generated by Ops.Observability.Generators />
# Alert rules for: OrderServiceObservability

groups:
  - name: order-service-alerts
    rules:
      - alert: OrderHighLatency
        expr: >-
          histogram_quantile(0.95,
            rate(order_api_request_duration_seconds_bucket[5m])
          ) > 0.5
        for: 5m
        labels:
          severity: warning
          service: order-service
          team: orders
        annotations:
          summary: "Order API p95 latency above 500ms"
          description: >-
            The 95th percentile latency for the Order API has exceeded
            500ms for 5 minutes. Check for database connection pool
            exhaustion or downstream payment service degradation.
          runbook_url: "https://docs.internal/runbooks/order-high-latency"
          notify_channels: "#order-alerts,oncall-orders"

      - alert: OrderHighErrorRate
        expr: >-
          rate(order_api_http_errors_total[5m])
          / rate(order_api_http_requests_total[5m])
          > 0.05
        for: 3m
        labels:
          severity: critical
          service: order-service
          team: orders
        annotations:
          summary: "Order API error rate above 5%"
          description: >-
            More than 5% of Order API requests are failing.
            Immediate investigation required.
          runbook_url: "https://docs.internal/runbooks/order-error-rate"
          notify_channels: "#order-alerts,oncall-orders,engineering-leads"

{
  "dashboard": {
    "title": "Order Service",
    "uid": "order-service-generated",
    "tags": ["orders", "payments", "generated"],
    "timezone": "utc",
    "refresh": "15s",
    "schemaVersion": 39,
    "panels": [
      {
        "id": 1,
        "title": "Request Latency (p95)",
        "type": "timeseries",
        "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
        "targets": [
          {
            "expr": "histogram_quantile(0.95, sum(rate(order_api_request_duration_seconds_bucket[5m])) by (le, endpoint, status_code))",
            "legendFormat": "{{endpoint}} ({{status_code}})"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "s",
            "thresholds": {
              "steps": [
                { "color": "green", "value": null },
                { "color": "yellow", "value": 0.25 },
                { "color": "red", "value": 0.5 }
              ]
            }
          }
        }
      },
      {
        "id": 2,
        "title": "Error Rate",
        "type": "timeseries",
        "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
        "targets": [
          {
            "expr": "rate(order_api_http_errors_total[5m])",
            "legendFormat": "{{endpoint}} ({{status_code}})"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "reqps"
          }
        }
      },
      {
        "id": 3,
        "title": "Active Orders in Queue",
        "type": "gauge",
        "gridPos": { "x": 0, "y": 8, "w": 8, "h": 6 },
        "targets": [
          {
            "expr": "order_queue_depth",
            "legendFormat": "queue depth"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "short",
            "thresholds": {
              "steps": [
                { "color": "green", "value": null },
                { "color": "yellow", "value": 100 },
                { "color": "red", "value": 500 }
              ]
            }
          }
        }
      },
      {
        "id": 4,
        "title": "Payment Success Rate",
        "type": "stat",
        "gridPos": { "x": 8, "y": 8, "w": 8, "h": 6 },
        "targets": [
          {
            "expr": "order_payment_success_ratio",
            "legendFormat": "success ratio"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "unit": "percentunit",
            "thresholds": {
              "steps": [
                { "color": "red", "value": null },
                { "color": "yellow", "value": 0.95 },
                { "color": "green", "value": 0.99 }
              ]
            }
          }
        }
      },
      {
        "id": 5,
        "title": "Latency Heatmap",
        "type": "heatmap",
        "gridPos": { "x": 16, "y": 8, "w": 8, "h": 6 },
        "targets": [
          {
            "expr": "sum(increase(order_api_request_duration_seconds_bucket[5m])) by (le)",
            "format": "heatmap",
            "legendFormat": "{{le}}"
          }
        ]
      }
    ]
  }
}

# <auto-generated by Ops.Observability.Generators />
# monitoring/servicemonitor.yaml -- Prometheus Operator scrape target
# Source: [Metric] attributes on OrderServiceObservability

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: order-service-metrics
  namespace: orders
  labels:
    app.kubernetes.io/name: order-service
    app.kubernetes.io/managed-by: ops.observability.generators
    # Prometheus Operator selector — matches the operator's serviceMonitorSelector
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: order-service
  namespaceSelector:
    matchNames:
      - orders
  endpoints:
    - port: metrics
      path: /metrics
      interval: 15s
      scrapeTimeout: 10s
      honorLabels: true
      # Relabeling to surface the [Metric] attribute names as Prometheus labels
      relabelings:
        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_version]
          targetLabel: service_version
        - sourceLabels: [__meta_kubernetes_namespace]
          targetLabel: namespace

# <auto-generated by Ops.Observability.Generators />
# monitoring/prometheusrule.yaml -- Prometheus Operator alert rules
# Source: [AlertRule] attributes on OrderServiceObservability
# Mirrors: prometheus-rules.yaml (same rules, CRD-wrapped for Operator clusters)

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: order-service-alerts
  namespace: orders
  labels:
    app.kubernetes.io/name: order-service
    app.kubernetes.io/managed-by: ops.observability.generators
    release: kube-prometheus-stack
    prometheus: kube-prometheus
spec:
  groups:
    - name: order-service-alerts
      rules:
        - alert: OrderHighLatency
          expr: >-
            histogram_quantile(0.95,
              rate(order_api_request_duration_seconds_bucket[5m])
            ) > 0.5
          for: 5m
          labels:
            severity: warning
            service: order-service
            team: orders
          annotations:
            summary: "Order API p95 latency above 500ms"
            description: >-
              The 95th percentile latency for the Order API has exceeded
              500ms for 5 minutes. Check for database connection pool
              exhaustion or downstream payment service degradation.
            runbook_url: "https://docs.internal/runbooks/order-high-latency"
            notify_channels: "#order-alerts,oncall-orders"

        - alert: OrderHighErrorRate
          expr: >-
            rate(order_api_http_errors_total[5m])
            / rate(order_api_http_requests_total[5m])
            > 0.05
          for: 3m
          labels:
            severity: critical
            service: order-service
            team: orders
          annotations:
            summary: "Order API error rate above 5%"
            description: >-
              More than 5% of Order API requests are failing.
              Immediate investigation required.
            runbook_url: "https://docs.internal/runbooks/order-error-rate"
            notify_channels: "#order-alerts,oncall-orders,engineering-leads"

# <auto-generated by Ops.Observability.Generators />

# Prometheus monitoring stack
resource "helm_release" "prometheus_rules" {
  name       = "order-service-rules"
  chart      = "prometheus-rules"
  namespace  = var.monitoring_namespace

  values = [file("${path.module}/prometheus-rules.yaml")]
}

# Grafana dashboard provisioning
resource "grafana_dashboard" "order_service" {
  folder      = grafana_folder.services.id
  config_json = file("${path.module}/grafana-dashboard.json")
}

# PagerDuty integration for critical alerts
resource "pagerduty_service" "order_service" {
  name                    = "Order Service"
  escalation_policy       = data.pagerduty_escalation_policy.engineering.id
  alert_creation          = "create_alerts_and_incidents"
  auto_resolve_timeout    = 14400
  acknowledgement_timeout = 600
}

[AlertRule("HighMemoryUsage",
    "order_api_memory_usage_bytes",     // <-- this metric is not declared anywhere
    "order_api_memory_usage_bytes > 1073741824",
    Severity = AlertSeverity.Warning)]
public sealed class OrderServiceObservability { }

// error OPS007: Alert 'HighMemoryUsage' references metric
//   'order_api_memory_usage_bytes' but no [Metric] with this name
//   exists in the compilation. Add [Metric("order_api_memory_usage_bytes", ...)]
//   or correct the metric name.

[AlertRule("DatabaseDown",
    "order_db_connection_pool_active",
    "order_db_connection_pool_active == 0",
    Severity = AlertSeverity.Critical,
    NotifyChannels = ["#order-alerts"])]
    // No Runbook specified
public sealed class OrderServiceObservability { }

// error OPS008: Alert 'DatabaseDown' has Severity = Critical but no
//   Runbook URL. Critical alerts must have a runbook so the on-call
//   engineer can act immediately. Set Runbook = "https://..."

[DashboardPanel("CPU Usage",
    "order_api_cpu_usage_percent",     // <-- not declared as a [Metric]
    Visualization = PanelVisualization.TimeSeries)]
public sealed class OrderServiceObservability { }

// warning OPS009: Dashboard panel 'CPU Usage' references metric
//   'order_api_cpu_usage_percent' which is not declared as a [Metric]
//   in this compilation. The panel will show 'No data' in Grafana.

// OrderServiceObservability exists but has no [HealthCheck] methods

// warning OPS010: No [HealthCheck] attributes found for service
//   'order-service'. Services without health checks cannot report
//   readiness to load balancers or orchestrators. Add at least one
//   [HealthCheck] with Kind = Readiness.

// Alert declares:
[AlertRule("OrderHighErrorRate", ..., Severity = AlertSeverity.Critical)]

// Incident DSL expects:
[IncidentEscalation("OrderHighErrorRate",
    Team = "orders",
    SlaMinutes = 15,
    AutoCreate = true)]

warning OPS030: Critical alert 'OrderHighErrorRate' has no matching
  [IncidentEscalation] in the Incident DSL. Critical alerts should
  have an escalation policy to ensure timely response.

[ServiceLevelIndicator("order-api-latency",
    Metric = "order_api_request_duration_seconds",
    Objective = "p95 < 500ms",
    Window = "30d")]

[SteadyStateProbe("order-error-rate",
    Metric = "order_api_http_errors_total",
    Assertion = "rate < 0.01",
    Description = "Error rate must stay below 1% during chaos experiment")]