Spark runtime upgrade — negative control

{
  "coverage": {
    "definitions_parsed_without_error": null,
    "estate_enumerated": false,
    "files_scanned": 0,
    "jobs_inventoried": 0,
    "tasks_inventoried": 0,
    "workloads_dual_run": 1
  },
  "customer": "SparkPilot reference migration - NEGATIVE CONTROL, seeded difference injected (--seed-difference rounding). This run exists to prove the comparator can fail; it is not an assessment of the runtime upgrade",
  "data_provenance": "live_run_seeded_fault",
  "equivalence": [
    {
      "applied_tolerances": {},
      "assertions_performed": 10,
      "findings": [
        {
          "dataset": "q1_pricing_summary",
          "detail": "sum_charge: source type decimal(38,6) vs target type decimal(38,1)",
          "kind": "schema_type",
          "severity": "DIFFERENCE"
        },
        {
          "dataset": "q1_pricing_summary",
          "detail": "content hash differs: source 984eb4735fc1b990… vs target 5e7671c91ac6d5cc…",
          "kind": "row_hash",
          "severity": "DIFFERENCE"
        },
        {
          "dataset": "q1_pricing_summary",
          "detail": "sum_charge:max: source 1418207490.410181 vs target 1418207475.3 (tolerance rel=0.0, abs=0.0)",
          "kind": "aggregate_value",
          "severity": "DIFFERENCE"
        },
        {
          "dataset": "q1_pricing_summary",
          "detail": "sum_charge:min: source 1391007086.081212 vs target 1391007105.8 (tolerance rel=0.0, abs=0.0)",
          "kind": "aggregate_value",
          "severity": "DIFFERENCE"
        },
        {
          "dataset": "q1_pricing_summary",
          "detail": "sum_charge:sum: source 8427644657.016188 vs target 8427644818.5 (tolerance rel=0.0, abs=0.0)",
          "kind": "aggregate_value",
          "severity": "DIFFERENCE"
        }
      ],
      "source_platform": "emr_serverless:emr-6.15.0",
      "target_platform": "emr_serverless:emr-7.5.0",
      "verdict": "NOT_EQUIVALENT",
      "workload_id": "tpch-q1-reference-v1"
    }
  ],
  "evidence_links": {
    "EMR Serverless pricing": "https://aws.amazon.com/emr/pricing/",
    "EMR job run (emr615-baseline)": "application 00g7nppcfcrf8i09 / job run 00g7nppqou31tg0b in AWS account 400920695203, us-east-1",
    "EMR job run (emr750-seeded)": "application 00g7nppdqs2mr409 / job run 00g7nppt0p4fu80b in AWS account 400920695203, us-east-1",
    "billed utilisation + rates (emr615-baseline)": "0.208 vCPU-hr x $0.052624/vCPU-hr + 0.831 GB-hr x $0.0057785/GB-hr = $0.0157 (EMR Serverless us-east-1 x86_64; from aws emr-serverless get-job-run; recomputable in scripts/generate_reference_migration_report.py)",
    "billed utilisation + rates (emr750-seeded)": "0.227 vCPU-hr x $0.052624/vCPU-hr + 0.907 GB-hr x $0.0057785/GB-hr = $0.0172 (EMR Serverless us-east-1 x86_64; from aws emr-serverless get-job-run; recomputable in scripts/generate_reference_migration_report.py)",
    "comparator guard mutation run": "scripts/verify_equivalence_guards.py",
    "manifest probe": "src/sparkpilot/migration/workloads/probe.py",
    "raw run manifests": "docs/evidence/runs/",
    "reference workload": "src/sparkpilot/migration/workloads/reference_tpch_q1.py"
  },
  "generated_at": "2026-08-14T06:00:01+00:00",
  "git_commit": "b10aac5",
  "performance": [
    {
      "estimated_cost_usd": 0.015747725499999997,
      "platform": "emr_serverless:emr-6.15.0",
      "spark_version": "3.4.1-amzn-2",
      "wall_clock_seconds": 48.477,
      "workload_id": "tpch-q1-reference-v1"
    },
    {
      "estimated_cost_usd": 0.0171867475,
      "platform": "emr_serverless:emr-7.5.0",
      "spark_version": "3.5.2-amzn-1",
      "wall_clock_seconds": 126.015,
      "workload_id": "tpch-q1-reference-v1"
    }
  ],
  "performance_confounds": [
    "runs were not given stated, identical capacity; no run manifest records a worker count or size, so wall clock is not attributable to the platform",
    "EMR Serverless bills each worker for a one-minute minimum, so a run shorter than that is charged for capacity it did not consume and metered cost is not a clean proxy for resource-seconds"
  ],
  "rationale": [
    "1 dual run(s) produced different output on emr_serverless:emr-7.5.0 than on emr_serverless:emr-6.15.0",
    "no estate was inventoried, so nothing is known about workloads other than the one(s) run here",
    "no source was statically scanned for incompatible constructs"
  ],
  "recommendation": "DO_NOT_CUT_OVER",
  "scanner_findings_joined_to_jobs": false,
  "scanner_job_overlap": "unknown",
  "schema": "sparkpilot.portability-manifest/v1",
  "source_platform": "emr_serverless:emr-6.15.0",
  "substitutions": [],
  "target_platform": "emr_serverless:emr-7.5.0"
}