Ops.DataGovernance -- Backup, Retention, GDPR, and Recovery📋

// =================================================================
// Ops.DataGovernance.Lib -- Data Governance DSL Attributes
// =================================================================

/// Classification level for data entities.
public enum ClassificationLevel
{
    Public,         // marketing pages, public API docs
    Internal,       // internal dashboards, non-sensitive configs
    Confidential,   // customer data, financial records
    Restricted      // PII, health records, payment card data
}

/// What happens when retention period expires.
public enum RetentionAction
{
    Archive,        // move to cold storage, keep accessible
    Delete,         // hard delete, irrecoverable
    Anonymize,      // replace PII with irreversible tokens
    Pseudonymize    // replace PII with reversible tokens (key kept separate)
}

/// GDPR lawful basis for processing personal data (Article 6).
public enum LawfulBasis
{
    Consent,            // data subject gave explicit consent
    Contract,           // processing necessary for contract performance
    LegalObligation,    // required by law (tax records, etc.)
    VitalInterest,      // protect someone's life
    PublicTask,          // public authority or public interest
    LegitimateInterest  // legitimate business interest (balancing test required)
}

/// GDPR deletion strategy when data subject exercises right to erasure.
public enum DeletionStrategy
{
    HardDelete,         // remove from all stores including backups
    SoftDelete,         // mark as deleted, purge after retention window
    Anonymize,          // replace identifying fields, keep statistical data
    CascadeDelete       // delete entity and all referencing entities
}

/// Declares backup policy for a database or data store.
[AttributeUsage(AttributeTargets.Class)]
public sealed class BackupPolicyAttribute : Attribute
{
    public string Database { get; }
    public string Frequency { get; }
    public string Retention { get; }
    public string StorageLocation { get; }
    public bool EncryptAtRest { get; }

    public BackupPolicyAttribute(
        string database,
        string frequency,
        string retention,
        string storageLocation,
        bool encryptAtRest = true)
    {
        Database = database;
        Frequency = frequency;
        Retention = retention;
        StorageLocation = storageLocation;
        EncryptAtRest = encryptAtRest;
    }

    /// <summary>Backup type: Full, Differential, or Incremental.</summary>
    public string BackupType { get; init; } = "Full";

    /// <summary>Enable point-in-time recovery (WAL archiving).</summary>
    public bool PointInTimeRecovery { get; init; } = false;

    /// <summary>Compression algorithm. Null = no compression.</summary>
    public string? Compression { get; init; } = "gzip";
}

/// Declares retention policy for a data entity.
[AttributeUsage(AttributeTargets.Class)]
public sealed class RetentionPolicyAttribute : Attribute
{
    public Type Entity { get; }
    public string MaxAge { get; }
    public RetentionAction Action { get; }

    public RetentionPolicyAttribute(
        Type entity, string maxAge, RetentionAction action)
    {
        Entity = entity;
        MaxAge = maxAge;
        Action = action;
    }

    /// <summary>
    /// Field that determines the entity's age (e.g., "ClosedAt", "CreatedAt").
    /// </summary>
    public string AgeField { get; init; } = "CreatedAt";

    /// <summary>Grace period after expiry before action is taken.</summary>
    public string GracePeriod { get; init; } = "0d";

    /// <summary>Run retention job in dry-run mode first.</summary>
    public bool DryRunFirst { get; init; } = true;
}

/// Maps an entity to GDPR processing record.
[AttributeUsage(AttributeTargets.Class)]
public sealed class GdprDataMapAttribute : Attribute
{
    public Type Entity { get; }
    public string[] PiiFields { get; }
    public LawfulBasis LawfulBasis { get; }
    public string RetentionPeriod { get; }
    public DeletionStrategy DeletionStrategy { get; }

    public GdprDataMapAttribute(
        Type entity,
        string[] piiFields,
        LawfulBasis lawfulBasis,
        string retentionPeriod,
        DeletionStrategy deletionStrategy)
    {
        Entity = entity;
        PiiFields = piiFields;
        LawfulBasis = lawfulBasis;
        RetentionPeriod = retentionPeriod;
        DeletionStrategy = deletionStrategy;
    }

    /// <summary>Purpose of processing (required by GDPR Article 30).</summary>
    public string Purpose { get; init; } = "";

    /// <summary>Data controller name.</summary>
    public string Controller { get; init; } = "";

    /// <summary>Third-party processors with access to this data.</summary>
    public string[]? Processors { get; init; }

    /// <summary>Whether cross-border transfer occurs.</summary>
    public bool CrossBorderTransfer { get; init; } = false;

    /// <summary>Transfer mechanism (SCCs, BCRs, Adequacy Decision).</summary>
    public string? TransferMechanism { get; init; }
}

/// Declares Recovery Point Objective and Recovery Time Objective.
[AttributeUsage(AttributeTargets.Class)]
public sealed class RecoveryObjectiveAttribute : Attribute
{
    /// <summary>Maximum acceptable data loss (e.g., "4h", "1h", "15m").</summary>
    public string RPO { get; }

    /// <summary>Maximum acceptable downtime (e.g., "1h", "30m", "5m").</summary>
    public string RTO { get; }

    public RecoveryObjectiveAttribute(string rpo, string rto)
    {
        RPO = rpo;
        RTO = rto;
    }

    /// <summary>
    /// Whether automated failover is required to meet RTO.
    /// </summary>
    public bool AutomatedFailover { get; init; } = false;

    /// <summary>Frequency of recovery drills (e.g., "quarterly").</summary>
    public string DrillFrequency { get; init; } = "quarterly";

    /// <summary>Notification channel for recovery events.</summary>
    public string NotifyChannel { get; init; } = "#ops-incidents";
}

/// Declarative test/dev data seeding.
[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
public sealed class SeedDataAttribute : Attribute
{
    public string Name { get; }
    public string[] Environments { get; }
    public int Order { get; }

    public SeedDataAttribute(
        string name, string[] environments, int order)
    {
        Name = name;
        Environments = environments;
        Order = order;
    }

    /// <summary>Seed data source: embedded JSON, factory method, or SQL.</summary>
    public string Source { get; init; } = "";

    /// <summary>Whether to skip if data already exists.</summary>
    public bool Idempotent { get; init; } = true;

    /// <summary>Number of entities to generate (for factory sources).</summary>
    public int Count { get; init; } = 1;
}

/// Classifies an entity's data sensitivity level.
[AttributeUsage(AttributeTargets.Class)]
public sealed class DataClassificationAttribute : Attribute
{
    public Type Entity { get; }
    public ClassificationLevel Level { get; }

    public DataClassificationAttribute(
        Type entity, ClassificationLevel level)
    {
        Entity = entity;
        Level = level;
    }

    /// <summary>Justification for the classification level.</summary>
    public string Justification { get; init; } = "";

    /// <summary>Review date for reclassification.</summary>
    public string ReviewDate { get; init; } = "";
}

[DeploymentApp("order-service")]

// Backup: every 4 hours, keep for 30 days, encrypted, with PITR
[BackupPolicy(
    database: "orders-db",
    frequency: "4h",
    retention: "30d",
    storageLocation: "s3://backups-eu-west-1/orders",
    encryptAtRest: true,
    PointInTimeRecovery = true,
    Compression = "zstd")]

// Recovery: 4-hour RPO (matches backup frequency), 30-minute RTO
[RecoveryObjective(
    rpo: "4h",
    rto: "30m",
    AutomatedFailover = true,
    DrillFrequency = "quarterly",
    NotifyChannel = "#orders-ops")]

// Retention: archive closed orders after 7 years (legal obligation)
[RetentionPolicy(
    typeof(Order), "7y", RetentionAction.Archive,
    AgeField = "ClosedAt",
    DryRunFirst = true)]

// Retention: anonymize inactive customers after 3 years
[RetentionPolicy(
    typeof(Customer), "3y", RetentionAction.Anonymize,
    AgeField = "LastActivityAt",
    GracePeriod = "30d")]

// GDPR: Customer entity contains PII
[GdprDataMap(
    typeof(Customer),
    piiFields: new[] { "Email", "FullName", "Phone", "Address" },
    LawfulBasis.Contract,
    retentionPeriod: "3y",
    DeletionStrategy.Anonymize,
    Purpose = "Order fulfillment and customer communication",
    Controller = "Acme Corp",
    Processors = new[] { "Stripe", "SendGrid" },
    CrossBorderTransfer = true,
    TransferMechanism = "SCCs")]

// GDPR: Order entity contains billing PII
[GdprDataMap(
    typeof(Order),
    piiFields: new[] { "BillingAddress", "BillingName" },
    LawfulBasis.Contract,
    retentionPeriod: "7y",
    DeletionStrategy.CascadeDelete,
    Purpose = "Financial record keeping")]

// Classification
[DataClassification(typeof(Customer), ClassificationLevel.Restricted,
    Justification = "Contains directly identifiable PII")]
[DataClassification(typeof(Order), ClassificationLevel.Confidential,
    Justification = "Contains billing information")]
[DataClassification(typeof(ProductCatalog), ClassificationLevel.Internal)]

// Seed data
[SeedData("test-customers", new[] { "Development", "Test" }, Order = 1,
    Source = "SeedFactories.CreateCustomers", Count = 50, Idempotent = true)]
[SeedData("test-orders", new[] { "Development", "Test" }, Order = 2,
    Source = "SeedFactories.CreateOrders", Count = 200, Idempotent = true)]
[SeedData("demo-data", new[] { "Staging" }, Order = 1,
    Source = "Seeds/demo-data.json", Idempotent = true)]
public partial class OrderServiceGovernance { }

// <auto-generated by Ops.DataGovernance.Generator />
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace OrderService.Generated;

public sealed class SeedDataRunner : IHostedService
{
    private readonly IServiceProvider _services;
    private readonly IHostEnvironment _env;
    private readonly ILogger<SeedDataRunner> _logger;

    public SeedDataRunner(
        IServiceProvider services,
        IHostEnvironment env,
        ILogger<SeedDataRunner> logger)
    {
        _services = services;
        _env = env;
        _logger = logger;
    }

    public async Task StartAsync(CancellationToken ct)
    {
        var seeds = new (string Name, string[] Envs, int Order, Func<IServiceProvider, CancellationToken, Task> Execute)[]
        {
            ("test-customers",
             new[] { "Development", "Test" }, 1,
             async (sp, c) =>
             {
                 var db = sp.GetRequiredService<OrderDbContext>();
                 if (await db.Customers.AnyAsync(c)) return; // idempotent
                 var entities = SeedFactories.CreateCustomers(50);
                 db.Customers.AddRange(entities);
                 await db.SaveChangesAsync(c);
             }),
            ("test-orders",
             new[] { "Development", "Test" }, 2,
             async (sp, c) =>
             {
                 var db = sp.GetRequiredService<OrderDbContext>();
                 if (await db.Orders.AnyAsync(c)) return;
                 var entities = SeedFactories.CreateOrders(200);
                 db.Orders.AddRange(entities);
                 await db.SaveChangesAsync(c);
             }),
            ("demo-data",
             new[] { "Staging" }, 1,
             async (sp, c) =>
             {
                 var db = sp.GetRequiredService<OrderDbContext>();
                 await SeedFromJson.ExecuteAsync(
                     db, "Seeds/demo-data.json", c);
             }),
        };

        var applicable = seeds
            .Where(s => s.Envs.Contains(_env.EnvironmentName))
            .OrderBy(s => s.Order);

        foreach (var seed in applicable)
        {
            _logger.LogInformation(
                "Executing seed '{Name}' (order {Order})",
                seed.Name, seed.Order);
            await seed.Execute(_services, ct);
        }
    }

    public Task StopAsync(CancellationToken ct) => Task.CompletedTask;
}

// <auto-generated by Ops.DataGovernance.Generator />
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace OrderService.Generated;

public sealed class RetentionJob : BackgroundService
{
    private readonly IServiceProvider _services;
    private readonly ILogger<RetentionJob> _logger;
    private readonly TimeSpan _interval = TimeSpan.FromHours(24);

    public RetentionJob(
        IServiceProvider services,
        ILogger<RetentionJob> logger)
    {
        _services = services;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken ct)
    {
        while (!ct.IsCancellationRequested)
        {
            await RunRetentionCycle(ct);
            await Task.Delay(_interval, ct);
        }
    }

    private async Task RunRetentionCycle(CancellationToken ct)
    {
        using var scope = _services.CreateScope();
        var db = scope.ServiceProvider
            .GetRequiredService<OrderDbContext>();
        var clock = scope.ServiceProvider
            .GetRequiredService<TimeProvider>();
        var now = clock.GetUtcNow();

        // Policy: Customer — 3y from LastActivityAt — Anonymize
        // Grace period: 30d
        var customerCutoff = now
            .AddYears(-3)
            .AddDays(-30);
        var customersToAnonymize = await db.Customers
            .Where(c => c.LastActivityAt < customerCutoff)
            .Where(c => !c.IsAnonymized)
            .ToListAsync(ct);

        _logger.LogInformation(
            "RetentionJob: {Count} Customer entities eligible " +
            "for anonymization", customersToAnonymize.Count);

        foreach (var customer in customersToAnonymize)
        {
            customer.Email = $"anon-{customer.Id}@redacted.local";
            customer.FullName = "[REDACTED]";
            customer.Phone = null;
            customer.Address = null;
            customer.IsAnonymized = true;
        }

        // Policy: Order — 7y from ClosedAt — Archive
        var orderCutoff = now.AddYears(-7);
        var ordersToArchive = await db.Orders
            .Where(o => o.ClosedAt != null && o.ClosedAt < orderCutoff)
            .Where(o => !o.IsArchived)
            .ToListAsync(ct);

        _logger.LogInformation(
            "RetentionJob: {Count} Order entities eligible " +
            "for archival", ordersToArchive.Count);

        foreach (var order in ordersToArchive)
        {
            await ArchiveToStorage(order, ct);
            order.IsArchived = true;
        }

        await db.SaveChangesAsync(ct);
    }

    private Task ArchiveToStorage(Order order, CancellationToken ct)
    {
        // Generated: serialize to archive store (S3, blob, etc.)
        return Task.CompletedTask;
    }
}

// <auto-generated by Ops.DataGovernance.Generator />
namespace OrderService.Generated;

/// <summary>
/// Registry of all PII-containing entities and their processing records.
/// Generated from [GdprDataMap] attributes.
/// </summary>
public static class GdprDataMap
{
    public static readonly IReadOnlyList<GdprProcessingRecord> Records =
        new GdprProcessingRecord[]
        {
            new(
                EntityType: typeof(Customer),
                PiiFields: new[] { "Email", "FullName", "Phone", "Address" },
                LawfulBasis: "Contract",
                RetentionPeriod: "3y",
                DeletionStrategy: "Anonymize",
                Purpose: "Order fulfillment and customer communication",
                Controller: "Acme Corp",
                Processors: new[] { "Stripe", "SendGrid" },
                CrossBorderTransfer: true,
                TransferMechanism: "SCCs"),
            new(
                EntityType: typeof(Order),
                PiiFields: new[] { "BillingAddress", "BillingName" },
                LawfulBasis: "Contract",
                RetentionPeriod: "7y",
                DeletionStrategy: "CascadeDelete",
                Purpose: "Financial record keeping",
                Controller: "",
                Processors: Array.Empty<string>(),
                CrossBorderTransfer: false,
                TransferMechanism: null),
        };

    /// <summary>
    /// Validates that declared PII fields exist on entity types.
    /// Called at startup.
    /// </summary>
    public static void Validate()
    {
        var errors = new List<string>();
        foreach (var record in Records)
        {
            foreach (var field in record.PiiFields)
            {
                var prop = record.EntityType.GetProperty(field);
                if (prop is null)
                {
                    errors.Add(
                        $"GdprDataMap: Entity '{record.EntityType.Name}' " +
                        $"declares PII field '{field}' but no such " +
                        $"property exists.");
                }
            }
        }

        if (errors.Count > 0)
        {
            throw new InvalidOperationException(
                "GDPR data map validation failed:\n" +
                string.Join("\n", errors));
        }
    }
}

public sealed record GdprProcessingRecord(
    Type EntityType,
    string[] PiiFields,
    string LawfulBasis,
    string RetentionPeriod,
    string DeletionStrategy,
    string Purpose,
    string Controller,
    string[] Processors,
    bool CrossBorderTransfer,
    string? TransferMechanism);

# auto-generated by Ops.DataGovernance.Generator
# Source: OrderServiceGovernance [BackupPolicy]
apiVersion: batch/v1
kind: CronJob
metadata:
  name: orders-db-backup
  namespace: order-service
  labels:
    app.kubernetes.io/component: backup
    ops.dsl/generator: data-governance
spec:
  schedule: "0 */4 * * *"          # every 4 hours
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backupLimit: 1
      template:
        spec:
          restartPolicy: OnFailure
          containers:
            - name: pg-backup
              image: postgres:16-alpine
              command:
                - /bin/sh
                - -c
                - |
                  set -euo pipefail
                  TIMESTAMP=$(date +%Y%m%d-%H%M%S)
                  FILENAME="orders-db-${TIMESTAMP}.sql.zst"
                  pg_dump "$DATABASE_URL" \
                    | zstd -19 \
                    | aws s3 cp - \
                        "s3://backups-eu-west-1/orders/${FILENAME}" \
                        --sse aws:kms
                  echo "Backup complete: ${FILENAME}"
              envFrom:
                - secretRef:
                    name: orders-db-credentials
              env:
                - name: AWS_DEFAULT_REGION
                  value: eu-west-1

# auto-generated by Ops.DataGovernance.Generator
# Source: OrderServiceGovernance [RecoveryObjective]
# Validates: RPO=4h, RTO=30m
apiVersion: batch/v1
kind: Job
metadata:
  name: orders-db-restore-test
  namespace: order-service-test
  labels:
    ops.dsl/generator: data-governance
    ops.dsl/test-type: recovery-drill
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      initContainers:
        - name: fetch-backup
          image: amazon/aws-cli:2
          command:
            - /bin/sh
            - -c
            - |
              LATEST=$(aws s3 ls s3://backups-eu-west-1/orders/ \
                | sort | tail -1 | awk '{print $4}')
              aws s3 cp \
                "s3://backups-eu-west-1/orders/${LATEST}" \
                /backup/latest.sql.zst
              # Verify backup age meets RPO (4h)
              BACKUP_TS=$(echo "$LATEST" \
                | grep -oP '\d{8}-\d{6}')
              # Fail if backup is older than RPO
          volumeMounts:
            - name: backup-vol
              mountPath: /backup
      containers:
        - name: restore-verify
          image: postgres:16-alpine
          command:
            - /bin/sh
            - -c
            - |
              set -euo pipefail
              START=$(date +%s)

              # Restore
              zstd -d /backup/latest.sql.zst -o /backup/latest.sql
              createdb -h localhost -U postgres restore_test
              psql -h localhost -U postgres -d restore_test \
                -f /backup/latest.sql

              END=$(date +%s)
              ELAPSED=$((END - START))
              echo "Restore completed in ${ELAPSED}s"

              # RTO check: must complete within 1800s (30m)
              if [ "$ELAPSED" -gt 1800 ]; then
                echo "FAIL: Restore took ${ELAPSED}s, RTO is 1800s"
                exit 1
              fi

              # Verify data integrity
              psql -h localhost -U postgres -d restore_test \
                -c "SELECT count(*) FROM customers;" \
                -c "SELECT count(*) FROM orders;"
          volumeMounts:
            - name: backup-vol
              mountPath: /backup
      volumes:
        - name: backup-vol
          emptyDir: {}

# auto-generated by Ops.DataGovernance.Generator
# Source: OrderServiceGovernance

# --- Backup Configuration ---

resource "aws_db_instance" "orders_db" {
  # ... existing resource, these are the governance-relevant settings:

  backup_retention_period = 30              # 30d retention
  backup_window           = "02:00-02:30"   # UTC
  copy_tags_to_snapshot   = true

  # Point-in-time recovery
  # WAL archiving enabled via backup_retention_period > 0

  # Encryption at rest
  storage_encrypted = true
  kms_key_id        = aws_kms_key.orders_backup.arn

  tags = {
    "ops.dsl/backup-frequency"  = "4h"
    "ops.dsl/retention"         = "30d"
    "ops.dsl/rpo"               = "4h"
    "ops.dsl/rto"               = "30m"
    "ops.dsl/data-classification" = "Restricted"
  }
}

# Cross-region replica for automated failover (RTO=30m)
resource "aws_db_instance" "orders_db_replica" {
  replicate_source_db    = aws_db_instance.orders_db.identifier
  instance_class         = aws_db_instance.orders_db.instance_class
  storage_encrypted      = true
  kms_key_id             = aws_kms_key.orders_backup_dr.arn
  availability_zone      = "eu-west-2a"   # different region

  tags = {
    "ops.dsl/purpose" = "disaster-recovery"
    "ops.dsl/rto"     = "30m"
  }
}

resource "aws_kms_key" "orders_backup" {
  description             = "Encryption key for orders-db backups"
  deletion_window_in_days = 30
  enable_key_rotation     = true
}

resource "aws_kms_key" "orders_backup_dr" {
  provider                = aws.eu-west-2
  description             = "Encryption key for orders-db DR replica"
  deletion_window_in_days = 30
  enable_key_rotation     = true
}

# --- Automated backup verification ---

resource "aws_cloudwatch_event_rule" "backup_drill" {
  name                = "orders-db-recovery-drill"
  description         = "Quarterly recovery drill (RecoveryObjective)"
  schedule_expression = "rate(90 days)"
}

resource "aws_cloudwatch_event_target" "backup_drill_target" {
  rule      = aws_cloudwatch_event_rule.backup_drill.name
  target_id = "orders-db-restore-test"
  arn       = aws_lambda_function.restore_drill.arn
}

# GDPR Record of Processing Activities
Generated: 2026-04-06T10:00:00Z
Source: OrderServiceGovernance

## Customer

| Field          | PII | Lawful Basis | Retention | Deletion Strategy |
|----------------|-----|-------------|-----------|-------------------|
| Email          | Yes | Contract    | 3 years   | Anonymize         |
| FullName       | Yes | Contract    | 3 years   | Anonymize         |
| Phone          | Yes | Contract    | 3 years   | Anonymize         |
| Address        | Yes | Contract    | 3 years   | Anonymize         |

- **Purpose:** Order fulfillment and customer communication
- **Controller:** Acme Corp
- **Processors:** Stripe, SendGrid
- **Cross-border Transfer:** Yes (Standard Contractual Clauses)
- **Classification:** Restricted

## Order

| Field          | PII | Lawful Basis | Retention | Deletion Strategy |
|----------------|-----|-------------|-----------|-------------------|
| BillingAddress | Yes | Contract    | 7 years   | Cascade Delete    |
| BillingName    | Yes | Contract    | 7 years   | Cascade Delete    |

- **Purpose:** Financial record keeping
- **Classification:** Confidential

warning DGV001: Entity 'PaymentMethod' has properties ['CardholderName',
'BillingAddress'] that appear to contain PII but no [GdprDataMap] is
declared. Add [GdprDataMap(typeof(PaymentMethod), ...)] to your governance
class.

error DGV002: Database 'analytics-db' is referenced by AnalyticsDbContext
but has no [BackupPolicy]. Add [BackupPolicy("analytics-db", ...)] to your
governance class.

error DGV003: Database 'orders-db' has [BackupPolicy] but no
[RecoveryObjective]. Declare RPO and RTO with
[RecoveryObjective(rpo: "...", rto: "...")].

warning DGV004: Entity 'NewsletterSubscription' uses LawfulBasis.Consent
with retention period '10y'. Consent-based processing typically requires
shorter retention. Consider reducing retention or changing lawful basis.

warning DGV-SEC001: Entity 'Customer' has [GdprDataMap] with PII fields
but no [AuditPolicy] from Ops.Security. GDPR Article 5(1)(f) requires
appropriate security, including audit trails for PII access.

warning DGV-DDD001: [GdprDataMap] references type 'OrderDto' which is not
an [AggregateRoot] or [Entity]. GDPR data maps should reference domain
entities, not DTOs. Did you mean typeof(Order)?