{ "date": "2026-11-07", "period": { "label": "March 2026", "start": "2026-03-01", "end": "2026-03-31" }, "models": [ "claude-sonnet-5-5", "claude-opus-5-5" ], "runs": 5, "questions": [ { "id": "mrr", "name": "MRR (Monthly Recurring Revenue)", "text": "What was MRR (Monthly Recurring Revenue) for March 2026?", "truth": 18400 }, { "id": "arr", "name": "ARR (Annual Recurring Revenue)", "text": "What was ARR (Annual Recurring Revenue) for March 2026?", "truth": 220800 }, { "id": "arpa", "name": "ARPA (Average Revenue Per Account)", "text": "What was ARPA (Average Revenue Per Account) for March 2026?", "truth": 1226.6666666666667 }, { "id": "mrr_growth_rate", "name": "MRR Growth Rate", "text": "What was MRR Growth Rate for March 2026?", "truth": 0.10843373493975904 }, { "id": "new_mrr", "name": "New MRR", "text": "What was New MRR for March 2026?", "truth": 2100 }, { "id": "expansion_mrr", "name": "Expansion MRR", "text": "What was Expansion MRR for March 2026?", "truth": 100 }, { "id": "churned_mrr", "name": "Churned MRR", "text": "What was Churned MRR for March 2026?", "truth": 0 }, { "id": "contraction_mrr", "name": "Contraction MRR", "text": "What was Contraction MRR for March 2026?", "truth": 400 }, { "id": "reactivation_mrr", "name": "Reactivation MRR", "text": "What was Reactivation MRR for March 2026?", "truth": 0 }, { "id": "carry_mrr", "name": "Carry MRR", "text": "What was Carry MRR for March 2026?", "truth": 16200 }, { "id": "net_new_mrr", "name": "Net New MRR", "text": "What was Net New MRR for March 2026?", "truth": 1800 }, { "id": "total_customers", "name": "Total Customers", "text": "What was Total Customers for March 2026?", "truth": 15 }, { "id": "new_customers", "name": "New Customers", "text": "What was New Customers for March 2026?", "truth": 2 }, { "id": "churned_customers", "name": "Churned Customers", "text": "What was Churned Customers for March 2026?", "truth": 0 }, { "id": "nrr", "name": "Net Revenue Retention (NRR)", "text": "What was Net Revenue Retention (NRR) for March 2026?", "truth": 0.9819277108433735 }, { "id": "grr", "name": "Gross Revenue Retention (GRR)", "text": "What was Gross Revenue Retention (GRR) for March 2026?", "truth": 0.9759036144578314 }, { "id": "logo_churn_rate", "name": "Logo Churn Rate (Customer Churn)", "text": "What was Logo Churn Rate (Customer Churn) for March 2026?", "truth": 0 }, { "id": "revenue_churn_rate", "name": "Revenue Churn Rate", "text": "What was Revenue Churn Rate for March 2026?", "truth": 0.024096385542168676 }, { "id": "nps", "name": "NPS (Net Promoter Score)", "text": "What was NPS (Net Promoter Score) for March 2026?", "truth": -25 }, { "id": "cac", "name": "CAC (Customer Acquisition Cost)", "text": "What was CAC (Customer Acquisition Cost) for March 2026?", "truth": 3000 }, { "id": "ltv", "name": "LTV (Customer Lifetime Value)", "text": "What was LTV (Customer Lifetime Value) for March 2026?", "truth": 47870.948012232424 }, { "id": "ltv_cac_ratio", "name": "LTV:CAC Ratio", "text": "What was LTV:CAC Ratio for March 2026?", "truth": 15.956982670744141 }, { "id": "cac_payback", "name": "CAC Payback Period", "text": "What was CAC Payback Period for March 2026?", "truth": 2.600742311770943 }, { "id": "gross_margin", "name": "Gross Margin", "text": "What was Gross Margin for March 2026?", "truth": 0.9403669724770642 } ], "results": [ { "model": "gasp-standard-mcp", "condition": "layer", "metric": "mrr", "run": 1, "sql": "SELECT (SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active') AS mrr", "truth": 18400, "ms": 1, "value": 18400, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "arr", "run": 1, "sql": "SELECT (((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) * 12) AS arr", "truth": 220800, "ms": 0, "value": 220800, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "arpa", "run": 1, "sql": "SELECT (CAST(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active')), 0)) AS arpa", "truth": 1226.6666666666667, "ms": 1, "value": 1226.6666666666667, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "mrr_growth_rate", "run": 1, "sql": "SELECT (CAST((((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) - ((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = (DATE '2026-03-31' - INTERVAL '1 month') AND status = 'active'))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = (DATE '2026-03-31' - INTERVAL '1 month') AND status = 'active')), 0)) AS mrr_growth_rate", "truth": 0.10843373493975904, "ms": 1, "value": 0.10843373493975904, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "new_mrr", "run": 1, "sql": "SELECT (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new') AS new_mrr", "truth": 2100, "ms": 0, "value": 2100, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "expansion_mrr", "run": 1, "sql": "SELECT (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'expansion') AS expansion_mrr", "truth": 100, "ms": 0, "value": 100, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "churned_mrr", "run": 1, "sql": "SELECT (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')) AS churned_mrr", "truth": 0, "ms": 0, "value": 0, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "contraction_mrr", "run": 1, "sql": "SELECT (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')) AS contraction_mrr", "truth": 400, "ms": 0, "value": 400, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "reactivation_mrr", "run": 1, "sql": "SELECT (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'reactivation') AS reactivation_mrr", "truth": 0, "ms": 0, "value": 0, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "carry_mrr", "run": 1, "sql": "SELECT ((((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active')) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')))) AS carry_mrr", "truth": 16200, "ms": 0, "value": 16200, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "net_new_mrr", "run": 1, "sql": "SELECT ((((((SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')) + ((SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'expansion'))) + ((SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'reactivation'))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')))) AS net_new_mrr", "truth": 1800, "ms": 0, "value": 1800, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "total_customers", "run": 1, "sql": "SELECT (SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active') AS total_customers", "truth": 15, "ms": 0, "value": 15, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "new_customers", "run": 1, "sql": "SELECT (SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new') AS new_customers", "truth": 2, "ms": 0, "value": 2, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "churned_customers", "run": 1, "sql": "SELECT (SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS churned_customers", "truth": 0, "ms": 0, "value": 0, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "nrr", "run": 1, "sql": "SELECT (CAST((((((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active' AND customer_id IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))) + ((SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'expansion' AND account_ref IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active')))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction' AND account_ref IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn' AND account_ref IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active' AND customer_id IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))), 0)) AS nrr", "truth": 0.9819277108433735, "ms": 5, "value": 0.9819277108433735, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "grr", "run": 1, "sql": "SELECT (CAST(((((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active' AND customer_id IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction' AND account_ref IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))))) - ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn' AND account_ref IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active' AND customer_id IN (SELECT customer_id FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active'))), 0)) AS grr", "truth": 0.9759036144578314, "ms": 5, "value": 0.9759036144578314, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "logo_churn_rate", "run": 1, "sql": "SELECT (CAST(((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')) AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-01' AND lifecycle_status = 'active')), 0)) AS logo_churn_rate", "truth": 0, "ms": 1, "value": 0, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "revenue_churn_rate", "run": 1, "sql": "SELECT (CAST((((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))) + ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active')), 0)) AS revenue_churn_rate", "truth": 0.024096385542168676, "ms": 0, "value": 0.024096385542168676, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "nps", "run": 1, "sql": "SELECT ((CAST(((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND (survey_type = 'customer' AND nps_score >= 9)) - (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND (survey_type = 'customer' AND nps_score <= 6))) AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer'), 0)) * 100) AS nps", "truth": -25, "ms": 0, "value": -25, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "cac", "run": 1, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')), 0)) AS cac", "truth": 3000, "ms": 0, "value": 3000, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "ltv", "run": 1, "sql": "SELECT (CAST((((CAST(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active')), 0))) * ((CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)))) AS DOUBLE PRECISION) / NULLIF(((CAST((((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))) + ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active')), 0))), 0)) AS ltv", "truth": 47870.948012232424, "ms": 1, "value": 47870.948012232424, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "ltv_cac_ratio", "run": 1, "sql": "SELECT (CAST(((CAST((((CAST(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active')), 0))) * ((CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)))) AS DOUBLE PRECISION) / NULLIF(((CAST((((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))) + ((-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')))) AS DOUBLE PRECISION) / NULLIF(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-01' AND status = 'active')), 0))), 0))) AS DOUBLE PRECISION) / NULLIF(((CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')), 0))), 0)) AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 1, "value": 15.956982670744141, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "cac_payback", "run": 1, "sql": "SELECT (CAST(((CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')), 0))) AS DOUBLE PRECISION) / NULLIF((((CAST(((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')) AS DOUBLE PRECISION) / NULLIF(((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active')), 0))) * ((CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)))), 0)) AS cac_payback", "truth": 2.600742311770943, "ms": 1, "value": 2.600742311770943, "category": "correct" }, { "model": "gasp-standard-mcp", "condition": "layer", "metric": "gross_margin", "run": 1, "sql": "SELECT (CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 0, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 2441, "input_tokens": 31, "output_tokens": 205, "cache_read": 0, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 2451, "input_tokens": 31, "output_tokens": 217, "cache_read": 0, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM latest\nWHERE status = 'active'", "truth": 18400, "ms": 2479, "input_tokens": 31, "output_tokens": 229, "cache_read": 0, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0)\nFROM latest\nWHERE status = 'active'", "truth": 18400, "ms": 2503, "input_tokens": 31, "output_tokens": 243, "cache_read": 0, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE LOWER(status) = 'active'", "truth": 18400, "ms": 2577, "input_tokens": 31, "output_tokens": 277, "cache_read": 0, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arr", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 220800, "ms": 2707, "input_tokens": 30, "output_tokens": 279, "cache_read": 0, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 220800, "ms": 2123, "input_tokens": 30, "output_tokens": 239, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE status = 'active'", "truth": 220800, "ms": 2174, "input_tokens": 30, "output_tokens": 218, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arpa", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE status = 'active'", "truth": 1226.6666666666667, "ms": 2106, "input_tokens": 30, "output_tokens": 276, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arr", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE status = 'active'", "truth": 220800, "ms": 2298, "input_tokens": 30, "output_tokens": 240, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arpa", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / COUNT(DISTINCT customer_id) AS arpa\nFROM latest\nWHERE status = 'active'", "truth": 1226.6666666666667, "ms": 2271, "input_tokens": 30, "output_tokens": 273, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 220800, "ms": 2847, "input_tokens": 30, "output_tokens": 324, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arpa", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE status = 'active'", "truth": 1226.6666666666667, "ms": 2188, "input_tokens": 30, "output_tokens": 299, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arpa", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE status = 'active'", "truth": 1226.6666666666667, "ms": 2300, "input_tokens": 30, "output_tokens": 278, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "arpa", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE status = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-03-31')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-31')", "truth": 1226.6666666666667, "ms": 2398, "input_tokens": 30, "output_tokens": 343, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 1, "sql": "WITH feb AS (\n SELECT SUM(mrr) AS m FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s WHERE status = 'active'\n),\nmar AS (\n SELECT SUM(mrr) AS m FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n ) s WHERE status = 'active'\n)\nSELECT (mar.m - feb.m) / NULLIF(feb.m, 0) AS mrr_growth_rate\nFROM feb, mar", "truth": 0.10843373493975904, "ms": 3456, "input_tokens": 23, "output_tokens": 512, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 2, "sql": "WITH mar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nfeb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT\n (SELECT COALESCE(SUM(mrr),0) FROM mar WHERE status = 'active')::numeric\n / NULLIF((SELECT SUM(mrr) FROM feb WHERE status = 'active'),0)\n - 1 AS mrr_growth_rate", "truth": 0.10843373493975904, "ms": 3286, "input_tokens": 23, "output_tokens": 473, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 3, "sql": "WITH mar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nfeb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nm AS (SELECT COALESCE(SUM(mrr),0) AS v FROM mar WHERE status = 'active'),\nf AS (SELECT COALESCE(SUM(mrr),0) AS v FROM feb WHERE status = 'active')\nSELECT (m.v - f.v) / NULLIF(f.v, 0) AS mrr_growth_rate\nFROM m, f", "truth": 0.10843373493975904, "ms": 3029, "input_tokens": 23, "output_tokens": 457, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr, mrr_change)), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(event_type) IN ('new', 'new_business', 'new business')", "truth": 2100, "ms": 2249, "input_tokens": 18, "output_tokens": 240, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr, mrr_change)), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND lower(event_type) = 'new'", "truth": 2100, "ms": 1825, "input_tokens": 18, "output_tokens": 179, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 5, "sql": "WITH feb AS (\n SELECT SUM(mrr) AS m FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE LOWER(status) = 'active'\n),\nmar AS (\n SELECT SUM(mrr) AS m FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE LOWER(status) = 'active'\n)\nSELECT (mar.m - feb.m) / NULLIF(feb.m, 0) AS mrr_growth_rate\nFROM feb, mar", "truth": 0.10843373493975904, "ms": 3209, "input_tokens": 23, "output_tokens": 498, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr, mrr_change)), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(event_type) IN ('new', 'new_business', 'new business')", "truth": 2100, "ms": 2106, "input_tokens": 18, "output_tokens": 243, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr, mrr_change)), 0)\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(event_type) = 'new'", "truth": 2100, "ms": 3152, "input_tokens": 18, "output_tokens": 345, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 4, "sql": "WITH feb AS (\n SELECT SUM(mrr) AS mrr FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) a\n),\nmar AS (\n SELECT SUM(mrr) AS mrr FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n ) b\n)\nSELECT (mar.mrr - feb.mrr) / NULLIF(feb.mrr, 0) AS mrr_growth_rate\nFROM feb, mar", "truth": 0.10843373493975904, "ms": 4951, "input_tokens": 23, "output_tokens": 681, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr, mrr_change)), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(event_type) IN ('new', 'new_business', 'new business', 'new_mrr')", "truth": 2100, "ms": 2460, "input_tokens": 18, "output_tokens": 299, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0)\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND expansion_classification = 'expansion'\n AND mrr_change > 0", "truth": 100, "ms": 2806, "input_tokens": 21, "output_tokens": 283, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0)\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change > 0\n AND LOWER(COALESCE(change_nature, event_type)) LIKE '%expansion%'\n AND (expansion_classification IS NULL OR LOWER(expansion_classification) NOT LIKE '%non%')", "truth": 100, "ms": 3538, "input_tokens": 21, "output_tokens": 411, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(expansion_classification) = 'expansion'\n AND mrr_change > 0", "truth": 100, "ms": 3649, "input_tokens": 21, "output_tokens": 362, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(expansion_classification) = 'expansion'\n AND mrr_change > 0", "truth": 100, "ms": 3390, "input_tokens": 21, "output_tokens": 356, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_mrr", "run": 1, "sql": "SELECT COALESCE(ABS(SUM(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE event_type = 'churn'\n AND booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 0, "ms": 3077, "input_tokens": 20, "output_tokens": 355, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0)\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(expansion_classification) = 'expansion'\n AND mrr_change > 0", "truth": 100, "ms": 3195, "input_tokens": 21, "output_tokens": 305, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0)\nFROM gasp_revenue_events\nWHERE event_type = 'churn'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 2587, "input_tokens": 20, "output_tokens": 235, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE event_type ILIKE 'churn%'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 3585, "input_tokens": 20, "output_tokens": 403, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE event_type = 'churn'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 3298, "input_tokens": 20, "output_tokens": 362, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_mrr", "run": 5, "sql": "SELECT COALESCE(-SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE event_type = 'churn'\n AND booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 0, "ms": 3092, "input_tokens": 20, "output_tokens": 363, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 1, "sql": "SELECT ABS(COALESCE(SUM(mrr_change), 0)) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND change_nature = 'contraction'\n AND mrr_change < 0", "truth": 400, "ms": 4274, "input_tokens": 21, "output_tokens": 550, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 4212, "input_tokens": 21, "output_tokens": 554, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0)\nFROM gasp_revenue_events\nWHERE lifecycle_attribution ILIKE 'reactivation'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 2656, "input_tokens": 22, "output_tokens": 254, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 3794, "input_tokens": 21, "output_tokens": 489, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS contraction_mrr\nFROM gasp_amendment_events\nWHERE change_nature = 'contraction'\n AND event_date >= DATE '2026-03-01'\n AND event_date <= DATE '2026-03-31'", "truth": 400, "ms": 4503, "input_tokens": 21, "output_tokens": 519, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE lifecycle_attribution ILIKE 'reactivation'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 2242, "input_tokens": 22, "output_tokens": 205, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 4507, "input_tokens": 21, "output_tokens": 554, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND (LOWER(lifecycle_attribution) = 'reactivation'\n OR LOWER(event_type) = 'reactivation')", "truth": 0, "ms": 2643, "input_tokens": 22, "output_tokens": 291, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0)\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(lifecycle_attribution) = 'reactivation'", "truth": 0, "ms": 2687, "input_tokens": 22, "output_tokens": 297, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(COALESCE(new_mrr,0) - COALESCE(prior_mrr,0)),0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(lifecycle_attribution) = 'reactivation'", "truth": 0, "ms": 2557, "input_tokens": 22, "output_tokens": 258, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "carry_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 16200, "ms": 3650, "input_tokens": 20, "output_tokens": 480, "cache_read": 6427, "value": 18400, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "carry_mrr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM latest\nWHERE LOWER(status) = 'active'\n AND contract_start < DATE '2026-03-01'", "truth": 16200, "ms": 4398, "input_tokens": 20, "output_tokens": 549, "cache_read": 6427, "value": 16300, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "carry_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE LOWER(status) = 'active'", "truth": 16200, "ms": 3040, "input_tokens": 20, "output_tokens": 439, "cache_read": 6427, "value": 18400, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "carry_mrr", "run": 3, "sql": "SELECT SUM(mrr) AS carry_mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 16200, "ms": 4967, "input_tokens": 20, "output_tokens": 646, "cache_read": 6427, "value": 18400, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(COALESCE(mrr_change, COALESCE(new_mrr,0) - COALESCE(prior_mrr,0))), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3801, "input_tokens": 20, "output_tokens": 463, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "carry_mrr", "run": 4, "sql": "WITH feb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nmar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mar.mrr), 0) AS carry_mrr\nFROM mar\nJOIN feb ON feb.subscription_id = mar.subscription_id\nWHERE feb.mrr > 0 AND mar.mrr > 0", "truth": 16200, "ms": 5768, "input_tokens": 20, "output_tokens": 791, "cache_read": 6427, "value": 16300, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3365, "input_tokens": 20, "output_tokens": 337, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(COALESCE(mrr_change, COALESCE(new_mrr,0) - COALESCE(prior_mrr,0))), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3471, "input_tokens": 20, "output_tokens": 423, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 4, "sql": "SELECT SUM(mrr_change) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3364, "input_tokens": 20, "output_tokens": 384, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3700, "input_tokens": 20, "output_tokens": 482, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "total_customers", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) *\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(*)::numeric AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND (lifecycle_status IS NULL OR lower(lifecycle_status) NOT IN ('churned','cancelled','canceled'))", "truth": 15, "ms": 2887, "input_tokens": 19, "output_tokens": 372, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "total_customers", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(*)::numeric\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND (lifecycle_status IS NULL OR lower(lifecycle_status) NOT IN ('churned','cancelled','canceled','inactive'))", "truth": 15, "ms": 2721, "input_tokens": 19, "output_tokens": 356, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "total_customers", "run": 2, "sql": "SELECT COUNT(*) AS total_customers\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n) c", "truth": 15, "ms": 3135, "input_tokens": 19, "output_tokens": 326, "cache_read": 6427, "value": 17, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM gasp_customers\nWHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 2, "ms": 2082, "input_tokens": 18, "output_tokens": 169, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "total_customers", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(*)::numeric\nFROM latest\nWHERE lower(lifecycle_status) = 'active'\n AND (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')", "truth": 15, "ms": 2882, "input_tokens": 19, "output_tokens": 352, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM gasp_customers\nWHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 2, "ms": 2105, "input_tokens": 18, "output_tokens": 178, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id)::numeric AS new_customers\nFROM gasp_customers\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'", "truth": 2, "ms": 2285, "input_tokens": 18, "output_tokens": 167, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "total_customers", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(*)::numeric AS total_customers\nFROM latest\nWHERE COALESCE(LOWER(lifecycle_status), '') NOT IN ('churned', 'cancelled', 'canceled')", "truth": 15, "ms": 3476, "input_tokens": 19, "output_tokens": 409, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_customers", "run": 4, "sql": "SELECT COUNT(*)::numeric\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC\n) c\nWHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 2, "ms": 2489, "input_tokens": 18, "output_tokens": 213, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT customer_id)::numeric AS churned_customers\nFROM gasp_lifecycle_events\nWHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(new_state) = 'churned'", "truth": 0, "ms": 2233, "input_tokens": 20, "output_tokens": 197, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "new_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM gasp_customers\nWHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": 2, "ms": 2244, "input_tokens": 18, "output_tokens": 185, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT customer_id)::numeric AS churned_customers\nFROM gasp_lifecycle_events\nWHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND (LOWER(new_state) = 'churned' OR LOWER(state_category) = 'churned')", "truth": 0, "ms": 2476, "input_tokens": 20, "output_tokens": 220, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE event_date BETWEEN '2026-03-01' AND '2026-03-31'\n AND (LOWER(new_state) = 'churned' OR LOWER(state_category) = 'churned')", "truth": 0, "ms": 2321, "input_tokens": 20, "output_tokens": 179, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(new_state) = 'churned'", "truth": 0, "ms": 2211, "input_tokens": 20, "output_tokens": 185, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "churned_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT customer_id)::numeric\nFROM gasp_lifecycle_events\nWHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND LOWER(new_state) = 'churned'", "truth": 0, "ms": 2229, "input_tokens": 20, "output_tokens": 191, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nrr", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr_start, mrr_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(COALESCE(mrr_end, 0)) / NULLIF(SUM(mrr_start), 0) AS nrr\nFROM latest\nWHERE mrr_start > 0", "truth": 0.9819277108433735, "ms": 4195, "input_tokens": 28, "output_tokens": 614, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(COALESCE(mrr_end, 0)) / NULLIF(SUM(mrr_start), 0) AS nrr\nFROM latest\nWHERE mrr_start > 0\n AND snapshot_day >= DATE '2026-03-01'", "truth": 0.9819277108433735, "ms": 3837, "input_tokens": 28, "output_tokens": 577, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nrr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(COALESCE(mrr_end, 0)) / NULLIF(SUM(mrr_start), 0) AS nrr\nFROM latest\nWHERE mrr_start > 0", "truth": 0.9819277108433735, "ms": 4382, "input_tokens": 28, "output_tokens": 630, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nrr", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(COALESCE(mrr_end, 0)) / NULLIF(SUM(mrr_start), 0) AS nrr\nFROM latest\nWHERE mrr_start > 0", "truth": 0.9819277108433735, "ms": 3902, "input_tokens": 28, "output_tokens": 557, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nrr", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncohort AS (\n SELECT s.subscription_id, s.mrr AS start_mrr,\n CASE WHEN e.status = 'active' THEN COALESCE(e.mrr, 0) ELSE 0 END AS end_mrr\n FROM start_snap s\n LEFT JOIN end_snap e ON e.subscription_id = s.subscription_id\n WHERE s.status = 'active'\n)\nSELECT SUM(end_mrr) / NULLIF(SUM(start_mrr), 0) AS nrr\nFROM cohort", "truth": 0.9819277108433735, "ms": 6239, "input_tokens": 28, "output_tokens": 928, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "grr", "run": 1, "sql": "WITH start_mrr AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncohort AS (\n SELECT subscription_id, mrr FROM start_mrr WHERE mrr > 0\n),\nlosses AS (\n SELECT COALESCE(SUM(-e.mrr_change), 0) AS lost\n FROM gasp_revenue_events e\n JOIN cohort c ON c.subscription_id = e.subscription_id\n WHERE e.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND e.mrr_change < 0\n)\nSELECT (SUM(c.mrr) - (SELECT lost FROM losses)) / NULLIF(SUM(c.mrr), 0) AS grr\nFROM cohort c", "truth": 0.9759036144578314, "ms": 6459, "input_tokens": 29, "output_tokens": 927, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "grr", "run": 5, "sql": "WITH s AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\ne AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(LEAST(COALESCE(e.mrr, 0), s.mrr)) / NULLIF(SUM(s.mrr), 0) AS grr\nFROM s\nLEFT JOIN e ON e.subscription_id = s.subscription_id\nWHERE s.mrr > 0", "truth": 0.9759036144578314, "ms": 5632, "input_tokens": 29, "output_tokens": 775, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "grr", "run": 4, "sql": "WITH s AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\ne AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(LEAST(COALESCE(e.mrr, 0), s.mrr)) / NULLIF(SUM(s.mrr), 0) AS grr\nFROM s\nLEFT JOIN e ON e.subscription_id = s.subscription_id\nWHERE s.mrr > 0", "truth": 0.9759036144578314, "ms": 6099, "input_tokens": 29, "output_tokens": 840, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "grr", "run": 3, "sql": "WITH s AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n AND lower(status) = 'active'\n ORDER BY subscription_id, snapshot_day DESC\n),\ne AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(LEAST(s.mrr, CASE WHEN lower(e.status) = 'active' THEN COALESCE(e.mrr, 0) ELSE 0 END)) / NULLIF(SUM(s.mrr), 0) AS grr\nFROM s\nLEFT JOIN e ON e.subscription_id = s.subscription_id", "truth": 0.9759036144578314, "ms": 6503, "input_tokens": 29, "output_tokens": 888, "cache_read": 6427, "value": 0.9, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 2, "sql": "WITH prior AS (\n SELECT DISTINCT ON (customer_id) customer_id, new_state\n FROM gasp_lifecycle_events\n WHERE event_date < DATE '2026-03-01'\n ORDER BY customer_id, event_date DESC, event_id DESC\n),\nstart_active AS (\n SELECT customer_id FROM prior\n WHERE lower(new_state) NOT LIKE '%churn%'\n),\nchurned AS (\n SELECT DISTINCT customer_id\n FROM gasp_lifecycle_events\n WHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(new_state) LIKE '%churn%'\n)\nSELECT (SELECT COUNT(*) FROM churned c JOIN start_active s USING (customer_id))::numeric\n / NULLIF((SELECT COUNT(*) FROM start_active), 0) AS logo_churn_rate", "truth": 0, "ms": 3984, "input_tokens": 31, "output_tokens": 508, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "grr", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncohort AS (\n SELECT subscription_id, mrr\n FROM start_snap\n WHERE lower(status) = 'active'\n),\nlosses AS (\n SELECT COALESCE(SUM(-e.mrr_change), 0) AS lost\n FROM gasp_revenue_events e\n JOIN cohort c ON c.subscription_id = e.subscription_id\n WHERE e.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND e.mrr_change < 0\n)\nSELECT (SELECT SUM(mrr) FROM cohort) AS start_mrr_placeholder_removed,\n 1\nFROM (SELECT 1) x\nWHERE false\nUNION ALL\nSELECT ((SELECT SUM(mrr) FROM cohort) - (SELECT lost FROM losses)) / (SELECT SUM(mrr) FROM cohort), 1", "truth": 0.9759036144578314, "ms": 7317, "input_tokens": 29, "output_tokens": 1230, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 1, "sql": "WITH base AS (\n SELECT DISTINCT customer_id\n FROM gasp_customers\n WHERE acquisition_date < DATE '2026-03-01'\n AND customer_id NOT IN (\n SELECT customer_id FROM gasp_lifecycle_events\n WHERE lower(new_state) = 'churned' AND event_date < DATE '2026-03-01'\n )\n),\nchurned AS (\n SELECT DISTINCT customer_id\n FROM gasp_lifecycle_events\n WHERE lower(new_state) = 'churned'\n AND event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND customer_id IN (SELECT customer_id FROM base)\n)\nSELECT (SELECT COUNT(*) FROM churned)::numeric / NULLIF((SELECT COUNT(*) FROM base), 0) AS logo_churn_rate", "truth": 0, "ms": 5272, "input_tokens": 31, "output_tokens": 720, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 3, "sql": "WITH last_state AS (\n SELECT DISTINCT ON (customer_id) customer_id, new_state\n FROM gasp_lifecycle_events\n WHERE event_date < DATE '2026-03-01'\n ORDER BY customer_id, event_date DESC, event_id DESC\n),\nbase AS (\n SELECT customer_id FROM last_state\n WHERE lower(new_state) NOT IN ('churned','churn','cancelled','canceled')\n),\nchurned AS (\n SELECT DISTINCT e.customer_id\n FROM gasp_lifecycle_events e\n JOIN base b ON b.customer_id = e.customer_id\n WHERE e.event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(e.new_state) IN ('churned','churn','cancelled','canceled')\n)\nSELECT (SELECT COUNT(*) FROM churned)::numeric / NULLIF((SELECT COUNT(*) FROM base), 0) AS logo_churn_rate", "truth": 0, "ms": 4004, "input_tokens": 31, "output_tokens": 571, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 4, "sql": "WITH last_before AS (\n SELECT DISTINCT ON (customer_id) customer_id, new_state\n FROM gasp_lifecycle_events\n WHERE event_date < DATE '2026-03-01'\n ORDER BY customer_id, event_date DESC, event_id DESC\n),\nbase AS (\n SELECT customer_id FROM last_before\n WHERE lower(new_state) NOT LIKE 'churn%'\n),\nchurned AS (\n SELECT DISTINCT e.customer_id\n FROM gasp_lifecycle_events e\n JOIN base b ON b.customer_id = e.customer_id\n WHERE e.event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(e.new_state) LIKE 'churn%'\n)\nSELECT (SELECT COUNT(*) FROM churned)::numeric / NULLIF((SELECT COUNT(*) FROM base), 0) AS logo_churn_rate", "truth": 0, "ms": 4778, "input_tokens": 31, "output_tokens": 636, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 5, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-02-28'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nbase AS (\n SELECT customer_id FROM start_snap WHERE lower(lifecycle_status) = 'active'\n)\nSELECT COUNT(*) FILTER (WHERE lower(e.lifecycle_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM base b\nJOIN end_snap e USING (customer_id)", "truth": 0, "ms": 5579, "input_tokens": 31, "output_tokens": 798, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 3, "sql": "WITH begin_mrr AS (\n SELECT SUM(mrr) AS mrr\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n),\nlost AS (\n SELECT SUM(-mrr_change) AS amt\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.amt / NULLIF(begin_mrr.mrr, 0)\nFROM lost, begin_mrr", "truth": 0.024096385542168676, "ms": 5422, "input_tokens": 24, "output_tokens": 726, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 1, "sql": "WITH start_mrr AS (\n SELECT SUM(mrr) AS m\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n),\nlost AS (\n SELECT -SUM(mrr_change) AS l\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n AND event_type IN ('churn','contraction','downgrade')\n)\nSELECT lost.l / start_mrr.m\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 5617, "input_tokens": 24, "output_tokens": 836, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 2, "sql": "WITH churn AS (\n SELECT COALESCE(-SUM(mrr_change),0) AS churned\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n AND event_type IN ('churn','contraction','downgrade')\n),\nstart_mrr AS (\n SELECT SUM(mrr_change) AS mrr\n FROM gasp_revenue_events\n WHERE booked_on < DATE '2026-03-01'\n)\nSELECT churn.churned / NULLIF(start_mrr.mrr,0) AS revenue_churn_rate\nFROM churn, start_mrr", "truth": 0.024096385542168676, "ms": 5572, "input_tokens": 24, "output_tokens": 796, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nps", "run": 1, "sql": "SELECT (COUNT(*) FILTER (WHERE nps_score >= 9) - COUNT(*) FILTER (WHERE nps_score <= 6))::numeric / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'", "truth": -25, "ms": 2191, "input_tokens": 27, "output_tokens": 271, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nps", "run": 3, "sql": "SELECT (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric) / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE survey_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND nps_score IS NOT NULL\n AND (survey_completed IS NULL OR survey_completed = TRUE)", "truth": -25, "ms": 2616, "input_tokens": 27, "output_tokens": 314, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nps", "run": 2, "sql": "SELECT (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric) / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE survey_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND nps_score IS NOT NULL\n AND COALESCE(survey_completed, TRUE) = TRUE", "truth": -25, "ms": 2758, "input_tokens": 27, "output_tokens": 331, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nps", "run": 4, "sql": "SELECT (COUNT(*) FILTER (WHERE nps_score >= 9) - COUNT(*) FILTER (WHERE nps_score <= 6))::numeric / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'\n AND nps_score IS NOT NULL\n AND COALESCE(survey_completed, TRUE) = TRUE", "truth": -25, "ms": 2739, "input_tokens": 27, "output_tokens": 361, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 4, "sql": "WITH start_mrr AS (\n SELECT SUM(mrr) AS m\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n),\nlost AS (\n SELECT SUM(-mrr_change) AS l\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.l / start_mrr.m\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 5311, "input_tokens": 24, "output_tokens": 737, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 5, "sql": "WITH start_mrr AS (\n SELECT SUM(mrr) AS m\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n),\nlost AS (\n SELECT -SUM(mrr_change) AS l\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type IN ('churn', 'contraction')\n AND mrr_change < 0\n)\nSELECT lost.l / NULLIF(start_mrr.m, 0) AS revenue_churn_rate\nFROM start_mrr, lost", "truth": 0.024096385542168676, "ms": 5286, "input_tokens": 24, "output_tokens": 781, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "nps", "run": 5, "sql": "SELECT (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric) / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND COALESCE(survey_completed, TRUE) = TRUE", "truth": -25, "ms": 2302, "input_tokens": 27, "output_tokens": 266, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac", "run": 1, "sql": "SELECT\n (SELECT SUM(cost)\n FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS cac", "truth": 3000, "ms": 3306, "input_tokens": 30, "output_tokens": 448, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac", "run": 2, "sql": "SELECT\n (SELECT SUM(cost)\n FROM gasp_cost_events\n WHERE lower(lifecycle_attribution) = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS cac", "truth": 3000, "ms": 3332, "input_tokens": 30, "output_tokens": 462, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac", "run": 4, "sql": "SELECT\n (SELECT SUM(cost)\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS cac", "truth": 3000, "ms": 3334, "input_tokens": 30, "output_tokens": 477, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac", "run": 3, "sql": "SELECT\n (SELECT SUM(cost)\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(lifecycle_attribution) = 'acquisition')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS cac", "truth": 3000, "ms": 3804, "input_tokens": 30, "output_tokens": 535, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac", "run": 5, "sql": "SELECT\n (SELECT SUM(cost)\n FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS cac", "truth": 3000, "ms": 3288, "input_tokens": 30, "output_tokens": 487, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv", "run": 1, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE s.status = 'active') AS start_cnt,\n COUNT(*) FILTER (WHERE s.status = 'active' AND COALESCE(e.status, 'churned') <> 'active') AS churned_cnt\n FROM start_snap s\n LEFT JOIN end_snap e ON e.subscription_id = s.subscription_id\n),\narpa AS (\n SELECT AVG(mrr) AS arpa\n FROM end_snap\n WHERE status = 'active'\n)\nSELECT a.arpa / NULLIF(c.churned_cnt::numeric / NULLIF(c.start_cnt, 0), 0) AS ltv\nFROM arpa a, churn c", "truth": 47870.948012232424, "ms": 6058, "input_tokens": 29, "output_tokens": 918, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv", "run": 2, "sql": "WITH s_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ns_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\narpa AS (\n SELECT AVG(mrr) AS arpa FROM s_end WHERE status = 'active'\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE e.subscription_id IS NULL OR e.status <> 'active')::numeric\n / NULLIF(COUNT(*), 0) AS churn_rate\n FROM s_start s\n LEFT JOIN s_end e ON e.subscription_id = s.subscription_id\n WHERE s.status = 'active'\n)\nSELECT arpa.arpa / NULLIF(churn.churn_rate, 0) AS ltv\nFROM arpa, churn", "truth": 47870.948012232424, "ms": 5751, "input_tokens": 29, "output_tokens": 846, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv", "run": 4, "sql": "WITH s_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\ns_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE LOWER(a.status) = 'active') AS start_active,\n COUNT(*) FILTER (WHERE LOWER(a.status) = 'active'\n AND LOWER(COALESCE(b.status, 'churned')) IN ('churned','cancelled','canceled')) AS churned\n FROM s_start a\n LEFT JOIN s_end b ON b.subscription_id = a.subscription_id\n),\narpa AS (\n SELECT AVG(mrr) AS arpa\n FROM s_end\n WHERE LOWER(status) = 'active'\n)\nSELECT arpa.arpa / NULLIF(churn.churned::numeric / NULLIF(churn.start_active, 0), 0) AS ltv\nFROM arpa, churn", "truth": 47870.948012232424, "ms": 6340, "input_tokens": 29, "output_tokens": 982, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv", "run": 3, "sql": "WITH feb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nmar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nchurn AS (\n SELECT\n COUNT(*) AS begin_count,\n COUNT(*) FILTER (WHERE m.subscription_id IS NULL OR LOWER(m.status) <> 'active') AS churned_count\n FROM feb f\n LEFT JOIN mar m ON m.subscription_id = f.subscription_id\n WHERE LOWER(f.status) = 'active'\n),\narpa AS (\n SELECT AVG(mrr) AS arpa\n FROM mar\n WHERE LOWER(status) = 'active'\n)\nSELECT arpa.arpa / NULLIF(churn.churned_count::numeric / NULLIF(churn.begin_count, 0), 0) AS ltv\nFROM arpa, churn", "truth": 47870.948012232424, "ms": 6516, "input_tokens": 29, "output_tokens": 940, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv", "run": 5, "sql": "WITH cur AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nprev AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE p.status = 'active') AS start_active,\n COUNT(*) FILTER (WHERE p.status = 'active' AND COALESCE(c.status, 'churned') <> 'active') AS churned\n FROM prev p\n LEFT JOIN cur c ON c.subscription_id = p.subscription_id\n),\narpa AS (\n SELECT AVG(mrr) AS arpa FROM cur WHERE status = 'active'\n)\nSELECT arpa.arpa / NULLIF(churn.churned::numeric / NULLIF(churn.start_active, 0), 0) AS ltv\nFROM arpa, churn", "truth": 47870.948012232424, "ms": 7686, "input_tokens": 29, "output_tokens": 1069, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 1, "sql": "WITH cac AS (\n SELECT (SELECT SUM(cost) FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN '2026-03-01' AND '2026-03-31')\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date BETWEEN '2026-03-01' AND '2026-03-31'), 0) AS cac\n),\nlatest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive AS (\n SELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa,\n COUNT(DISTINCT customer_id) AS n_active\n FROM latest\n WHERE status = 'active'\n),\nchurn AS (\n SELECT COUNT(DISTINCT customer_id) AS churned\n FROM gasp_lifecycle_events\n WHERE new_state = 'churned'\n AND event_date BETWEEN '2026-03-01' AND '2026-03-31'\n)\nSELECT (active.arpa / NULLIF(churn.churned::numeric / NULLIF(active.n_active + churn.churned, 0), 0))\n / NULLIF(cac.cac, 0) AS ltv_cac_ratio\nFROM cac, active, churn", "truth": 15.956982670744141, "ms": 8510, "input_tokens": 25, "output_tokens": 1284, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 2, "sql": "WITH cac AS (\n SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date BETWEEN '2026-03-01' AND '2026-03-31'), 0) AS cac\n),\nlatest_sub AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\narpa AS (\n SELECT AVG(mrr) AS arpa FROM latest_sub WHERE status = 'active'\n),\nchurn AS (\n SELECT\n (SELECT COUNT(DISTINCT customer_id) FROM gasp_lifecycle_events\n WHERE event_date BETWEEN '2026-03-01' AND '2026-03-31'\n AND new_state ILIKE 'churn%')::numeric\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date < '2026-03-01'), 0) AS churn_rate\n)\nSELECT (arpa.arpa / NULLIF(churn.churn_rate, 0)) / NULLIF(cac.cac, 0) AS ltv_cac_ratio\nFROM cac, arpa, churn", "truth": 15.956982670744141, "ms": 7063, "input_tokens": 25, "output_tokens": 1113, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac_payback", "run": 1, "sql": "SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT SUM(COALESCE(new_mrr, mrr_change)) FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type ILIKE 'new%'), 0) AS cac_payback_months", "truth": 2.600742311770943, "ms": 5216, "input_tokens": 22, "output_tokens": 663, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 3, "sql": "WITH sub_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsub_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\narpa AS (\n SELECT AVG(mrr) AS v FROM sub_end WHERE status = 'active'\n),\nstart_cust AS (\n SELECT COUNT(DISTINCT customer_id) AS n FROM sub_start WHERE status = 'active'\n),\nchurned AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_lifecycle_events\n WHERE event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND new_state = 'churned'\n),\nnew_cust AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nacq_cost AS (\n SELECT SUM(cost) AS v\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lifecycle_attribution = 'acquisition'\n)\nSELECT (arpa.v / (churned.n::numeric / NULLIF(start_cust.n, 0)))\n / (acq_cost.v / NULLIF(new_cust.n, 0)) AS ltv_cac_ratio\nFROM arpa, start_cust, churned, new_cust, acq_cost", "truth": 15.956982670744141, "ms": 7995, "input_tokens": 25, "output_tokens": 1283, "cache_read": 6427, "value": null, "category": "sql error", "error": "division by zero" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 5, "sql": "WITH cac AS (\n SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN '2026-03-01' AND '2026-03-31')\n /\n NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date BETWEEN '2026-03-01' AND '2026-03-31'), 0) AS cac\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\n FROM subs\n WHERE status = 'active'\n),\nstart_active AS (\n SELECT COUNT(*) AS n FROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= '2026-02-28'\n ORDER BY customer_id, as_of_date DESC\n ) t\n WHERE lifecycle_status = 'active'\n),\nchurned AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_lifecycle_events\n WHERE new_state = 'churned'\n AND event_date BETWEEN '2026-03-01' AND '2026-03-31'\n)\nSELECT (arpa.arpa / NULLIF(churned.n::numeric / NULLIF(start_active.n, 0), 0)) / cac.cac AS ltv_cac_ratio\nFROM cac, arpa, start_active, churned", "truth": 15.956982670744141, "ms": 8010, "input_tokens": 25, "output_tokens": 1208, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac_payback", "run": 2, "sql": "SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT SUM(COALESCE(new_mrr, mrr_change)) FROM gasp_revenue_events\n WHERE booked_on BETWEEN '2026-03-01' AND '2026-03-31'\n AND event_type = 'new'\n AND lifecycle_attribution = 'acquisition'), 0) AS cac_payback_months", "truth": 2.600742311770943, "ms": 4907, "input_tokens": 22, "output_tokens": 598, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\narpa AS (\n SELECT AVG(mrr) AS v FROM latest WHERE status = 'active'\n),\nchurn AS (\n SELECT\n (SELECT COUNT(DISTINCT customer_id) FROM gasp_lifecycle_events\n WHERE new_state = 'churned' AND event_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')::numeric\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date < DATE '2026-03-01'), 0) AS r\n),\ncac AS (\n SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'), 0) AS v\n)\nSELECT (arpa.v / NULLIF(churn.r, 0)) / NULLIF(cac.v, 0) AS ltv_cac_ratio\nFROM arpa, churn, cac", "truth": 15.956982670744141, "ms": 8745, "input_tokens": 25, "output_tokens": 1274, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac_payback", "run": 3, "sql": "SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(cost_function) IN ('sales','marketing','sales_marketing','sales and marketing'))\n /\n NULLIF((SELECT SUM(COALESCE(new_mrr, mrr_change)) FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(event_type) IN ('new','new_business','new business')), 0) AS cac_payback_months", "truth": 2.600742311770943, "ms": 5208, "input_tokens": 22, "output_tokens": 637, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac_payback", "run": 4, "sql": "SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT SUM(new_mrr) FROM gasp_revenue_events\n WHERE booked_on BETWEEN '2026-03-01' AND '2026-03-31'\n AND event_type = 'new'), 0) AS cac_payback_months", "truth": 2.600742311770943, "ms": 3877, "input_tokens": 22, "output_tokens": 443, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "gross_margin", "run": 2, "sql": "SELECT (r.rev - c.cogs) / NULLIF(r.rev, 0) AS gross_margin\nFROM (\n SELECT COALESCE(SUM(revenue), 0) AS rev\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n) r\nCROSS JOIN (\n SELECT COALESCE(SUM(cost), 0) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND (LOWER(cost_type) IN ('cogs', 'cost_of_revenue', 'cost of revenue')\n OR LOWER(cost_function) IN ('cogs', 'cost_of_revenue', 'cost of revenue'))\n) c", "truth": 0.9403669724770642, "ms": 3466, "input_tokens": 20, "output_tokens": 492, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "cac_payback", "run": 5, "sql": "SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lifecycle_attribution = 'acquisition')\n /\n NULLIF((SELECT SUM(COALESCE(new_mrr, mrr_change)) FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac_payback_months", "truth": 2.600742311770943, "ms": 4768, "input_tokens": 22, "output_tokens": 614, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "gross_margin", "run": 1, "sql": "SELECT (r.rev - c.cogs) / NULLIF(r.rev, 0) AS gross_margin\nFROM (\n SELECT SUM(revenue) AS rev\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n) r,\n(\n SELECT COALESCE(SUM(cost), 0) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND (cost_type ILIKE '%cogs%' OR cost_function ILIKE '%cogs%'\n OR cost_type ILIKE '%cost of%' OR cost_function ILIKE '%cost of%')\n) c", "truth": 0.9403669724770642, "ms": 4770, "input_tokens": 20, "output_tokens": 618, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "gross_margin", "run": 3, "sql": "SELECT (r.rev - c.cogs) / NULLIF(r.rev, 0) AS gross_margin\nFROM (\n SELECT COALESCE(SUM(revenue), 0) AS rev\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n) r,\n(\n SELECT COALESCE(SUM(cost), 0) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND (LOWER(cost_type) = 'cogs' OR LOWER(cost_function) = 'cogs')\n) c", "truth": 0.9403669724770642, "ms": 4471, "input_tokens": 20, "output_tokens": 610, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 1852, "input_tokens": 1011, "output_tokens": 247, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 2657, "input_tokens": 1011, "output_tokens": 324, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "gross_margin", "run": 4, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND (cost_type ILIKE '%cogs%' OR cost_function ILIKE '%cogs%'\n OR cost_type ILIKE '%cost of revenue%' OR cost_function ILIKE '%cost of revenue%')\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 4188, "input_tokens": 20, "output_tokens": 612, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-sonnet-5-5", "condition": "raw", "metric": "gross_margin", "run": 5, "sql": "SELECT (r.rev - c.cogs) / NULLIF(r.rev, 0) AS gross_margin\nFROM (\n SELECT SUM(revenue) AS rev\n FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n) r,\n(\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND (LOWER(cost_type) = 'cogs' OR LOWER(cost_function) = 'cogs')\n) c", "truth": 0.9403669724770642, "ms": 3914, "input_tokens": 20, "output_tokens": 518, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 1595, "input_tokens": 1011, "output_tokens": 214, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 2176, "input_tokens": 1011, "output_tokens": 284, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 18400, "ms": 1542, "input_tokens": 1011, "output_tokens": 209, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 220800, "ms": 2339, "input_tokens": 1417, "output_tokens": 220, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 220800, "ms": 2298, "input_tokens": 1417, "output_tokens": 232, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 220800, "ms": 2278, "input_tokens": 1417, "output_tokens": 249, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM gasp_subscriptions\nWHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')", "truth": 220800, "ms": 2617, "input_tokens": 1417, "output_tokens": 253, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n) s\nWHERE status = 'active'", "truth": 220800, "ms": 2548, "input_tokens": 1417, "output_tokens": 263, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arpa", "run": 1, "sql": "WITH sub AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT (SELECT COALESCE(SUM(mrr), 0) FROM sub WHERE status = 'active')::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM cust WHERE lifecycle_status = 'active'), 0) AS arpa", "truth": 1226.6666666666667, "ms": 3683, "input_tokens": 1268, "output_tokens": 513, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arpa", "run": 3, "sql": "WITH sub_snap AS (\n SELECT subscription_id, mrr, status\n FROM (\n SELECT s.*, ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY snapshot_day DESC) rn\n FROM gasp_subscriptions s\n WHERE snapshot_day <= DATE '2026-03-31'\n ) t\n WHERE rn = 1\n),\ncust_snap AS (\n SELECT customer_id, lifecycle_status\n FROM (\n SELECT c.*, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY as_of_date DESC) rn\n FROM gasp_customers c\n WHERE as_of_date <= DATE '2026-03-31'\n ) t\n WHERE rn = 1\n)\nSELECT CAST((SELECT COALESCE(SUM(mrr), 0) FROM sub_snap WHERE status = 'active') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM cust_snap WHERE lifecycle_status = 'active'), 0) AS arpa", "truth": 1226.6666666666667, "ms": 3544, "input_tokens": 1268, "output_tokens": 519, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arpa", "run": 2, "sql": "WITH sub AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT (SELECT COALESCE(SUM(mrr),0) FROM sub WHERE status = 'active')::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM cust WHERE lifecycle_status = 'active'), 0) AS arpa", "truth": 1226.6666666666667, "ms": 3785, "input_tokens": 1268, "output_tokens": 522, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arpa", "run": 5, "sql": "WITH sub AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT (SELECT COALESCE(SUM(mrr), 0) FROM sub WHERE status = 'active')::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM cust WHERE lifecycle_status = 'active'), 0) AS arpa", "truth": 1226.6666666666667, "ms": 3167, "input_tokens": 1268, "output_tokens": 416, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "arpa", "run": 4, "sql": "WITH s AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nc AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT CAST((SELECT COALESCE(SUM(mrr), 0) FROM s WHERE status = 'active') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM c WHERE lifecycle_status = 'active'), 0) AS arpa", "truth": 1226.6666666666667, "ms": 3539, "input_tokens": 1268, "output_tokens": 539, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 1, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n),\nprev AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n)\nSELECT (cur.m - prev.m) / NULLIF(prev.m, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 3183, "input_tokens": 1139, "output_tokens": 465, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1526, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1481, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 2, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active'\n),\nprev AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28')\n AND status = 'active'\n)\nSELECT (cur.v - prev.v) / NULLIF(prev.v, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 3584, "input_tokens": 1139, "output_tokens": 479, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 3, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr),0) AS m\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nprev AS (\n SELECT COALESCE(SUM(mrr),0) AS m\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28')\n)\nSELECT (cur.m - prev.m) / NULLIF(prev.m, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 3357, "input_tokens": 1139, "output_tokens": 411, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1431, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 4, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active'\n),\nprev AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28')\n AND status = 'active'\n)\nSELECT (cur.m - prev.m) / NULLIF(prev.m, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 3380, "input_tokens": 1139, "output_tokens": 465, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1497, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1527, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 5, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active'\n),\nprev AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28')\n AND status = 'active'\n)\nSELECT (cur.m - prev.m) / NULLIF(prev.m, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 3611, "input_tokens": 1139, "output_tokens": 464, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1530, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1486, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1420, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1563, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 1, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1195, "input_tokens": 903, "output_tokens": 109, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1608, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 2, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1329, "input_tokens": 903, "output_tokens": 109, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 3, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1282, "input_tokens": 903, "output_tokens": 109, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 4, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1224, "input_tokens": 903, "output_tokens": 109, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 5, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1335, "input_tokens": 903, "output_tokens": 109, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 1, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1507, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 2, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1635, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1154, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 5, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1538, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 4, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1565, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 3, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1656, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1110, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1133, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1240, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1161, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 1, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0)) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2202, "input_tokens": 1507, "output_tokens": 289, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 1, "sql": "SELECT\n COALESCE((SELECT SUM(mrr) FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')), 0)\n - (-COALESCE((SELECT SUM(mrr_change) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'), 0))\n - (-COALESCE((SELECT SUM(mrr_change) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'), 0))\n AS carry_mrr", "truth": 16200, "ms": 3647, "input_tokens": 1343, "output_tokens": 454, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 2, "sql": "SELECT\n (SELECT COALESCE(SUM(s.mrr), 0)\n FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (\n SELECT MAX(s2.snapshot_day)\n FROM gasp_subscriptions s2\n WHERE s2.subscription_id = s.subscription_id\n AND s2.snapshot_day <= DATE '2026-03-01'\n ))\n - (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n - (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n AS carry_mrr", "truth": 16200, "ms": 3867, "input_tokens": 1343, "output_tokens": 527, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM latest WHERE status = 'active')\n + (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')\n + (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')\n AS carry_mrr", "truth": 16200, "ms": 4954, "input_tokens": 1343, "output_tokens": 544, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM latest WHERE status = 'active')\n + (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')\n + (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')\n AS carry_mrr", "truth": 16200, "ms": 4392, "input_tokens": 1343, "output_tokens": 482, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 5, "sql": "SELECT\n (SELECT COALESCE(SUM(mrr),0) FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n ) s WHERE status = 'active')\n - (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n - (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n AS carry_mrr", "truth": 16200, "ms": 4189, "input_tokens": 1343, "output_tokens": 591, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 2, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0)) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2294, "input_tokens": 1507, "output_tokens": 289, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 3, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0)) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2227, "input_tokens": 1507, "output_tokens": 289, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "total_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n) latest\nWHERE lifecycle_status = 'active'", "truth": 15, "ms": 2150, "input_tokens": 851, "output_tokens": 214, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "total_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n) s\nWHERE lifecycle_status = 'active'", "truth": 15, "ms": 2486, "input_tokens": 851, "output_tokens": 234, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0)) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2900, "input_tokens": 1507, "output_tokens": 395, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "total_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n) s\nWHERE lifecycle_status = 'active'", "truth": 15, "ms": 2410, "input_tokens": 851, "output_tokens": 254, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 5, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0)) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2985, "input_tokens": 1507, "output_tokens": 411, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "total_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n) latest\nWHERE lifecycle_status = 'active'", "truth": 15, "ms": 2633, "input_tokens": 851, "output_tokens": 261, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1622, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "total_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM gasp_customers\nWHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n AND lifecycle_status = 'active'", "truth": 15, "ms": 1969, "input_tokens": 851, "output_tokens": 168, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1556, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1524, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1543, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "new_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1566, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1408, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1604, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1485, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1535, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "churned_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1560, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "grr", "run": 1, "sql": "WITH cohort AS (\n SELECT customer_id\n FROM (\n SELECT customer_id, lifecycle_status,\n ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY as_of_date DESC) AS rn\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ) c\n WHERE rn = 1 AND lifecycle_status = 'active'\n),\nsub AS (\n SELECT subscription_id, customer_id, mrr, status\n FROM (\n SELECT subscription_id, customer_id, mrr, status,\n ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY snapshot_day DESC) AS rn\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ) s\n WHERE rn = 1\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM sub\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.m + ev.contraction + ev.churn)::double precision / NULLIF(start_mrr.m, 0) AS grr\nFROM start_mrr, ev", "truth": 0.9759036144578314, "ms": 3647, "input_tokens": 2324, "output_tokens": 661, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nrr", "run": 1, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS con,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS chn\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.v + ev.exp - ev.con - ev.chn) / NULLIF(start_mrr.v, 0) AS nrr\nFROM start_mrr, ev", "truth": 0.9819277108433735, "ms": 6014, "input_tokens": 3200, "output_tokens": 920, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nrr", "run": 2, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbase AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr0\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp_mrr,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS con_chg,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS chu_chg\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (base.mrr0 + ev.exp_mrr + ev.con_chg + ev.chu_chg)::double precision / NULLIF(base.mrr0, 0) AS nrr\nFROM base, ev", "truth": 0.9819277108433735, "ms": 6134, "input_tokens": 3200, "output_tokens": 1023, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nrr", "run": 3, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contr,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.v + ev.exp - ev.contr - ev.churn)::double precision / NULLIF(start_mrr.v, 0) AS nrr\nFROM start_mrr, ev", "truth": 0.9819277108433735, "ms": 5752, "input_tokens": 3200, "output_tokens": 969, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nrr", "run": 4, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS expansion,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.mrr + ev.expansion + ev.contraction + ev.churn) / NULLIF(start_mrr.mrr, 0) AS nrr\nFROM start_mrr, ev", "truth": 0.9819277108433735, "ms": 5806, "input_tokens": 3200, "output_tokens": 985, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nrr", "run": 5, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS con,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS chu\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.m + ev.exp + ev.con + ev.chu) / NULLIF(start_mrr.m, 0) AS nrr\nFROM start_mrr, ev", "truth": 0.9819277108433735, "ms": 5715, "input_tokens": 3200, "output_tokens": 944, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "grr", "run": 2, "sql": "WITH cohort AS (\n SELECT customer_id\n FROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n ) c\n WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT s.customer_id, s.mrr\n FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE s.status = 'active'\n AND s.customer_id IN (SELECT customer_id FROM cohort)\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM subs\n),\ncontr AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT ((start_mrr.v + contr.v + churn.v) / NULLIF(start_mrr.v, 0))::double precision AS grr\nFROM start_mrr, contr, churn", "truth": 0.9759036144578314, "ms": 3880, "input_tokens": 2324, "output_tokens": 696, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 2, "sql": "WITH snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n),\nden AS (\n SELECT COUNT(DISTINCT c.customer_id) AS n\n FROM gasp_customers c\n JOIN snap s ON c.as_of_date = s.d\n WHERE c.lifecycle_status = 'active'\n),\nnum AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n)\nSELECT CAST(num.n AS DOUBLE PRECISION) / NULLIF(den.n, 0) AS logo_churn_rate\nFROM num, den", "truth": 0, "ms": 2403, "input_tokens": 1307, "output_tokens": 391, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 1, "sql": "SELECT CAST((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01') AND lifecycle_status = 'active'), 0) AS logo_churn_rate", "truth": 0, "ms": 2674, "input_tokens": 1307, "output_tokens": 418, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "grr", "run": 3, "sql": "WITH cohort AS (\n SELECT customer_id\n FROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n ) c\n WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT customer_id, subscription_id, mrr, status\n FROM (\n SELECT DISTINCT ON (subscription_id) customer_id, subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontr AS (\n SELECT COALESCE(-SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT COALESCE(-SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (start_mrr.v - contr.v - churn.v)::double precision / NULLIF(start_mrr.v, 0)\nFROM start_mrr, contr, churn", "truth": 0.9759036144578314, "ms": 3777, "input_tokens": 2324, "output_tokens": 693, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "grr", "run": 4, "sql": "WITH cohort AS (\n SELECT customer_id\n FROM gasp_customers c\n WHERE lifecycle_status = 'active'\n AND as_of_date = (\n SELECT MAX(as_of_date) FROM gasp_customers c2\n WHERE c2.customer_id = c.customer_id AND c2.as_of_date <= DATE '2026-03-01'\n )\n),\nsubs AS (\n SELECT s.customer_id, s.mrr\n FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (\n SELECT MAX(s2.snapshot_day) FROM gasp_subscriptions s2\n WHERE s2.subscription_id = s.subscription_id AND s2.snapshot_day <= DATE '2026-03-01'\n )\n AND s.customer_id IN (SELECT customer_id FROM cohort)\n),\nbase AS (\n SELECT COALESCE(SUM(mrr), 0) AS start_mrr FROM subs\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (base.start_mrr + ev.contraction + ev.churn)::double precision / NULLIF(base.start_mrr, 0) AS grr\nFROM base, ev", "truth": 0.9759036144578314, "ms": 3481, "input_tokens": 2324, "output_tokens": 614, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "grr", "run": 5, "sql": "WITH cohort AS (\n SELECT customer_id\n FROM (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n ) c\n WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT customer_id, subscription_id, mrr, status\n FROM (\n SELECT DISTINCT ON (subscription_id) customer_id, subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active' AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontr AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT ((start_mrr.v - contr.v - churn.v) / NULLIF(start_mrr.v, 0))::float8 AS grr\nFROM start_mrr, contr, churn", "truth": 0.9759036144578314, "ms": 5744, "input_tokens": 2324, "output_tokens": 698, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 5, "sql": "SELECT CAST((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01') AND lifecycle_status = 'active'), 0) AS logo_churn_rate", "truth": 0, "ms": 3004, "input_tokens": 1307, "output_tokens": 421, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT CAST((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM latest WHERE lifecycle_status = 'active'), 0) AS logo_churn_rate", "truth": 0, "ms": 3817, "input_tokens": 1307, "output_tokens": 518, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 4, "sql": "SELECT CAST((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS DOUBLE PRECISION)\n/ NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01') AND lifecycle_status = 'active'), 0) AS logo_churn_rate", "truth": 0, "ms": 3264, "input_tokens": 1307, "output_tokens": 468, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nden AS (\n SELECT COALESCE(SUM(mrr), 0) AS m FROM latest WHERE status = 'active'\n),\nnum AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT num.n::double precision / NULLIF(den.m, 0) AS revenue_churn_rate\nFROM num, den", "truth": 0.024096385542168676, "ms": 3302, "input_tokens": 1501, "output_tokens": 508, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nps", "run": 1, "sql": "SELECT\n (CAST(\n (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score >= 9))\n - (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score <= 6))\n AS DOUBLE PRECISION)\n / NULLIF(COUNT(nps_score) FILTER (WHERE survey_type = 'customer'), 0)) * 100 AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 2084, "input_tokens": 1086, "output_tokens": 229, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nps", "run": 2, "sql": "SELECT (\n (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score >= 9)\n - COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score <= 6))::double precision\n / NULLIF(COUNT(nps_score) FILTER (WHERE survey_type = 'customer'), 0)\n) * 100 AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 1928, "input_tokens": 1086, "output_tokens": 211, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nden AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM latest\n WHERE status = 'active'\n),\nnum AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n - COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS lost\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT num.lost::double precision / NULLIF(den.mrr, 0) AS revenue_churn_rate\nFROM num, den", "truth": 0.024096385542168676, "ms": 3157, "input_tokens": 1501, "output_tokens": 463, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 4, "sql": "SELECT\n (\n -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn')\n -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction')\n )::double precision\n / NULLIF((\n SELECT COALESCE(SUM(mrr), 0) FROM (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n ) s\n WHERE status = 'active'\n ), 0) AS revenue_churn_rate", "truth": 0.024096385542168676, "ms": 3544, "input_tokens": 1501, "output_tokens": 503, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nden AS (\n SELECT COALESCE(SUM(mrr), 0) AS d FROM latest WHERE status = 'active'\n),\nnum AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type IN ('churn', 'contraction')\n)\nSELECT CAST(num.n AS DOUBLE PRECISION) / NULLIF(den.d, 0) AS revenue_churn_rate\nFROM num, den", "truth": 0.024096385542168676, "ms": 3663, "input_tokens": 1501, "output_tokens": 526, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 2, "sql": "WITH latest AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n),\nden AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT d FROM latest)\n AND status = 'active'\n),\nnum AS (\n SELECT -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n - COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS lost\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT CAST(num.lost AS DOUBLE PRECISION) / NULLIF(den.mrr, 0) AS revenue_churn_rate\nFROM num, den", "truth": 0.024096385542168676, "ms": 3856, "input_tokens": 1501, "output_tokens": 547, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nps", "run": 3, "sql": "SELECT (\n (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score >= 9)\n - COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score <= 6))::double precision\n / NULLIF(COUNT(nps_score) FILTER (WHERE survey_type = 'customer'), 0)\n) * 100 AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 1939, "input_tokens": 1086, "output_tokens": 211, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nps", "run": 4, "sql": "SELECT (\n (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score >= 9)\n - COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score <= 6))::double precision\n / NULLIF(COUNT(nps_score) FILTER (WHERE survey_type = 'customer'), 0)\n) * 100 AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 1834, "input_tokens": 1086, "output_tokens": 211, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "nps", "run": 5, "sql": "SELECT (\n (COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score >= 9)\n - COUNT(nps_score) FILTER (WHERE survey_type = 'customer' AND nps_score <= 6))::double precision\n / NULLIF(COUNT(nps_score) FILTER (WHERE survey_type = 'customer'), 0)\n) * 100 AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 2120, "input_tokens": 1086, "output_tokens": 211, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac", "run": 1, "sql": "SELECT\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac", "truth": 3000, "ms": 1741, "input_tokens": 1618, "output_tokens": 221, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac", "run": 2, "sql": "SELECT\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac", "truth": 3000, "ms": 1693, "input_tokens": 1618, "output_tokens": 220, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac", "run": 3, "sql": "SELECT\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac", "truth": 3000, "ms": 1575, "input_tokens": 1618, "output_tokens": 220, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac", "run": 4, "sql": "SELECT\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac", "truth": 3000, "ms": 1567, "input_tokens": 1618, "output_tokens": 221, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac", "run": 5, "sql": "SELECT\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0) AS cac", "truth": 3000, "ms": 1650, "input_tokens": 1618, "output_tokens": 220, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv", "run": 2, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n )::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers c\n WHERE lifecycle_status = 'active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n ), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ), 0)\n , 0)\n AS ltv", "truth": 47870.948012232424, "ms": 4645, "input_tokens": 2945, "output_tokens": 859, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv", "run": 1, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status='active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31'))::double precision\n / NULLIF((SELECT COUNT(DISTINCT c.customer_id) FROM gasp_customers c\n WHERE c.lifecycle_status='active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')),0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function='infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'),0)\n )\n /\n NULLIF(\n (\n (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type='churn'))\n + (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type='contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status='active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')),0)\n ,0) AS ltv", "truth": 47870.948012232424, "ms": 4667, "input_tokens": 2945, "output_tokens": 841, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv", "run": 3, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n )::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers c\n WHERE c.lifecycle_status = 'active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n ), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ), 0)\n , 0) AS ltv", "truth": 47870.948012232424, "ms": 4679, "input_tokens": 2945, "output_tokens": 863, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv", "run": 4, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status='active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n )::double precision\n / NULLIF((SELECT COUNT(DISTINCT c.customer_id) FROM gasp_customers c\n WHERE c.lifecycle_status='active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n ),0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function='infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'),0)\n )\n /\n NULLIF(\n (\n (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type='churn'))\n + (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type='contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE s.status='active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ),0)\n ,0) AS ltv", "truth": 47870.948012232424, "ms": 4631, "input_tokens": 2945, "output_tokens": 849, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv", "run": 5, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n )::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers c\n WHERE lifecycle_status = 'active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n ), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change),0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr),0) FROM gasp_subscriptions s\n WHERE status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ), 0)\n , 0) AS ltv", "truth": 47870.948012232424, "ms": 4648, "input_tokens": 2945, "output_tokens": 857, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 1, "sql": "SELECT\n (\n (\n (SELECT COALESCE(SUM(mrr), 0)::double precision\n FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')\n / NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active'), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (\n (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-01' AND status = 'active'), 0)\n ), 0)\n )\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(cost), 0)::double precision FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0)\n ), 0) AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 5209, "input_tokens": 3209, "output_tokens": 981, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac_payback", "run": 1, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)\n )\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active')::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n AND lifecycle_status = 'active'), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n ),\n 0) AS cac_payback", "truth": 2.600742311770943, "ms": 4027, "input_tokens": 2676, "output_tokens": 734, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 3, "sql": "SELECT\n (\n (\n (SELECT COALESCE(SUM(mrr), 0)::double precision\n FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')\n / NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active'), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (\n -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn')\n - (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction')\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-01' AND status = 'active'), 0)\n ), 0)\n )\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(cost), 0)::double precision FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0)\n ), 0)\n AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 5048, "input_tokens": 3209, "output_tokens": 978, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 5, "sql": "SELECT\n (\n (\n (SELECT COALESCE(SUM(mrr), 0)::double precision\n FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')\n / NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active'), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (\n (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-01' AND status = 'active'), 0)\n ), 0)\n )\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(cost), 0)::double precision FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0)\n ), 0) AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 5174, "input_tokens": 3209, "output_tokens": 981, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 4, "sql": "SELECT\n (\n (\n (SELECT COALESCE(SUM(mrr), 0)::double precision\n FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-31' AND status = 'active')\n / NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers\n WHERE as_of_date = DATE '2026-03-31' AND lifecycle_status = 'active'), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n /\n NULLIF(\n (\n (\n (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr), 0) FROM gasp_subscriptions\n WHERE snapshot_day = DATE '2026-03-01' AND status = 'active'), 0)\n ), 0)\n )\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(cost), 0)::double precision FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'gtm')\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'), 0)\n ), 0) AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 5243, "input_tokens": 3209, "output_tokens": 982, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 2, "sql": "SELECT\n (\n (\n (SELECT COALESCE(SUM(mrr), 0)::double precision\n FROM gasp_subscriptions s\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n )\n / NULLIF((SELECT COUNT(DISTINCT customer_id)\n FROM gasp_customers c\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n ), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n )\n /\n NULLIF(\n (\n (\n (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n + (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n )::double precision\n / NULLIF((SELECT COALESCE(SUM(mrr), 0)\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ), 0)\n ), 0)\n /\n NULLIF(\n (\n (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)\n ), 0)\n AS ltv_cac_ratio", "truth": 15.956982670744141, "ms": 5667, "input_tokens": 3209, "output_tokens": 1092, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "gross_margin", "run": 1, "sql": "SELECT (CAST((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure') AS DOUBLE PRECISION)\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 1847, "input_tokens": 1526, "output_tokens": 280, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac_payback", "run": 2, "sql": "WITH cac AS (\n SELECT (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31' AND cost_function='gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on BETWEEN '2026-03-01' AND '2026-03-31' AND event_type='new'),0) AS v\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\narpa AS (\n SELECT (SELECT COALESCE(SUM(mrr),0) FROM subs WHERE status='active')::double precision\n / NULLIF((SELECT COUNT(DISTINCT customer_id) FROM cust WHERE lifecycle_status='active'),0) AS v\n),\ngm AS (\n SELECT ((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials\n WHERE period_start BETWEEN '2026-03-01' AND '2026-03-31')\n - (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31' AND cost_function='infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue),0) FROM gasp_financials\n WHERE period_start BETWEEN '2026-03-01' AND '2026-03-31'),0) AS v\n)\nSELECT cac.v / NULLIF(arpa.v * gm.v, 0) AS cac_payback\nFROM cac, arpa, gm", "truth": 2.600742311770943, "ms": 5955, "input_tokens": 2676, "output_tokens": 998, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "gross_margin", "run": 2, "sql": "SELECT (CAST((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure') AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 1788, "input_tokens": 1526, "output_tokens": 276, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "gross_margin", "run": 3, "sql": "SELECT (CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 1737, "input_tokens": 1526, "output_tokens": 278, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "gross_margin", "run": 4, "sql": "SELECT (CAST(((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure')) AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 1906, "input_tokens": 1526, "output_tokens": 278, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac_payback", "run": 5, "sql": "SELECT\n (\n (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)\n )\n /\n NULLIF(\n (\n (\n (SELECT COALESCE(SUM(s.mrr), 0) FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31'))::double precision\n / NULLIF((SELECT COUNT(DISTINCT c.customer_id) FROM gasp_customers c\n WHERE c.lifecycle_status = 'active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')), 0)\n )\n *\n (\n ((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')\n - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'))::double precision\n / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)\n )\n ), 0\n ) AS cac_payback", "truth": 2.600742311770943, "ms": 4023, "input_tokens": 2676, "output_tokens": 758, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "gross_margin", "run": 5, "sql": "SELECT (CAST((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31') - (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure') AS DOUBLE PRECISION) / NULLIF((SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'), 0)) AS gross_margin", "truth": 0.9403669724770642, "ms": 1870, "input_tokens": 1526, "output_tokens": 276, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac_payback", "run": 3, "sql": "WITH cac AS (\n SELECT\n (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31' AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on BETWEEN '2026-03-01' AND '2026-03-31' AND event_type = 'new'), 0) AS v\n),\nmrr AS (\n SELECT COALESCE(SUM(mrr),0) AS v FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= '2026-03-31')\n AND status = 'active'\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id) AS v FROM gasp_customers\n WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= '2026-03-31')\n AND lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(revenue),0) AS v FROM gasp_financials\n WHERE period_start BETWEEN '2026-03-01' AND '2026-03-31'\n),\ninfra AS (\n SELECT COALESCE(SUM(cost),0) AS v FROM gasp_cost_events\n WHERE incurred_on BETWEEN '2026-03-01' AND '2026-03-31' AND cost_function = 'infrastructure'\n)\nSELECT (SELECT v FROM cac)\n / NULLIF(\n ((SELECT v FROM mrr)::double precision / NULLIF((SELECT v FROM cust),0))\n * (((SELECT v FROM rev) - (SELECT v FROM infra))::double precision / NULLIF((SELECT v FROM rev),0)),\n 0) AS cac_payback", "truth": 2.600742311770943, "ms": 5617, "input_tokens": 2676, "output_tokens": 887, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-sonnet-5-5", "condition": "gasp", "metric": "cac_payback", "run": 4, "sql": "WITH cac AS (\n SELECT (SELECT COALESCE(SUM(cost),0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0) AS v\n),\nmrr AS (\n SELECT COALESCE(SUM(mrr),0)::double precision AS v FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id)::double precision AS v FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nrev AS (\n SELECT COALESCE(SUM(revenue),0)::double precision AS v FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ninfra AS (\n SELECT COALESCE(SUM(cost),0)::double precision AS v FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'infrastructure'\n)\nSELECT (SELECT v FROM cac)\n / NULLIF(((SELECT v FROM mrr) / NULLIF((SELECT v FROM cust),0))\n * (((SELECT v FROM rev) - (SELECT v FROM infra)) / NULLIF((SELECT v FROM rev),0)), 0) AS cac_payback", "truth": 2.600742311770943, "ms": 6135, "input_tokens": 2676, "output_tokens": 1003, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr_march_2026\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 18400, "ms": 3340, "input_tokens": 31, "output_tokens": 337, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM latest\nWHERE lower(status) = 'active'", "truth": 18400, "ms": 3127, "input_tokens": 31, "output_tokens": 351, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status, contract_start, contract_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM latest\nWHERE LOWER(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-03-31')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-31')", "truth": 18400, "ms": 3496, "input_tokens": 31, "output_tokens": 403, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr_march_2026\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 18400, "ms": 3369, "input_tokens": 31, "output_tokens": 368, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr_march_2026\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 18400, "ms": 4370, "input_tokens": 31, "output_tokens": 455, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arr", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status, contract_start, contract_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-03-31')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-31')", "truth": 220800, "ms": 4078, "input_tokens": 30, "output_tokens": 380, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 220800, "ms": 3852, "input_tokens": 30, "output_tokens": 354, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status, contract_start, contract_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-03-31')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-31')", "truth": 220800, "ms": 4290, "input_tokens": 30, "output_tokens": 384, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 220800, "ms": 3917, "input_tokens": 30, "output_tokens": 338, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arpa", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr)::numeric / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE LOWER(status) = 'active'\n AND mrr > 0", "truth": 1226.6666666666667, "ms": 3712, "input_tokens": 30, "output_tokens": 387, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arr", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status, contract_start, contract_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-03-31')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-31')", "truth": 220800, "ms": 4010, "input_tokens": 30, "output_tokens": 392, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arpa", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive AS (\n SELECT customer_id, mrr\n FROM latest\n WHERE LOWER(status) = 'active'\n AND mrr > 0\n)\nSELECT SUM(mrr)::numeric / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM active", "truth": 1226.6666666666667, "ms": 4498, "input_tokens": 30, "output_tokens": 514, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arpa", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM latest\nWHERE LOWER(status) = 'active'\n AND mrr > 0", "truth": 1226.6666666666667, "ms": 3484, "input_tokens": 30, "output_tokens": 370, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arpa", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive AS (\n SELECT customer_id, mrr\n FROM latest\n WHERE LOWER(status) = 'active'\n)\nSELECT SUM(mrr)::numeric / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM active", "truth": 1226.6666666666667, "ms": 3626, "input_tokens": 30, "output_tokens": 407, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "arpa", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive AS (\n SELECT customer_id, mrr\n FROM latest\n WHERE LOWER(status) = 'active'\n AND COALESCE(mrr, 0) > 0\n)\nSELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\nFROM active", "truth": 1226.6666666666667, "ms": 3447, "input_tokens": 30, "output_tokens": 406, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ntotals AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM start_snap WHERE lower(status) = 'active') AS mrr_start,\n (SELECT COALESCE(SUM(mrr), 0) FROM end_snap WHERE lower(status) = 'active') AS mrr_end\n)\nSELECT (mrr_end - mrr_start) / NULLIF(mrr_start, 0) AS mrr_growth_rate\nFROM totals", "truth": 0.10843373493975904, "ms": 5501, "input_tokens": 23, "output_tokens": 665, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 1, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ntotals AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM start_snap WHERE lower(status) = 'active') AS start_mrr,\n (SELECT COALESCE(SUM(mrr), 0) FROM end_snap WHERE lower(status) = 'active') AS end_mrr\n)\nSELECT (end_mrr - start_mrr) / NULLIF(start_mrr, 0) AS mrr_growth_rate\nFROM totals", "truth": 0.10843373493975904, "ms": 6403, "input_tokens": 23, "output_tokens": 755, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 3, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ntotals AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM start_snap WHERE LOWER(status) = 'active') AS start_mrr,\n (SELECT COALESCE(SUM(mrr), 0) FROM end_snap WHERE LOWER(status) = 'active') AS end_mrr\n)\nSELECT (end_mrr - start_mrr) / NULLIF(start_mrr, 0) AS mrr_growth_rate\nFROM totals", "truth": 0.10843373493975904, "ms": 5532, "input_tokens": 23, "output_tokens": 675, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 4, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ntotals AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM start_snap WHERE status = 'active') AS start_mrr,\n (SELECT COALESCE(SUM(mrr), 0) FROM end_snap WHERE status = 'active') AS end_mrr\n)\nSELECT (end_mrr - start_mrr) / NULLIF(start_mrr, 0) AS mrr_growth_rate\nFROM totals", "truth": 0.10843373493975904, "ms": 5542, "input_tokens": 23, "output_tokens": 666, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND LOWER(event_type) IN ('new', 'new_business', 'new business', 'new_mrr', 'new mrr', 'new_subscription')", "truth": 2100, "ms": 5202, "input_tokens": 18, "output_tokens": 503, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE lower(event_type) IN ('new', 'new_business', 'new business')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'", "truth": 2100, "ms": 3674, "input_tokens": 18, "output_tokens": 356, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "mrr_growth_rate", "run": 5, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ns AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr FROM start_snap WHERE lower(status) = 'active'\n),\ne AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr FROM end_snap WHERE lower(status) = 'active'\n)\nSELECT (e.mrr - s.mrr) / NULLIF(s.mrr, 0) AS mrr_growth_rate\nFROM s, e", "truth": 0.10843373493975904, "ms": 6527, "input_tokens": 23, "output_tokens": 753, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE lower(event_type) IN ('new', 'new_business', 'new_mrr', 'new business')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 2100, "ms": 3712, "input_tokens": 18, "output_tokens": 390, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE LOWER(event_type) = 'new'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'", "truth": 2100, "ms": 3106, "input_tokens": 18, "output_tokens": 323, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE lower(event_type) IN ('new', 'new_business', 'new business', 'new_logo')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'", "truth": 2100, "ms": 3671, "input_tokens": 18, "output_tokens": 383, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE lower(event_type) = 'expansion'\n AND mrr_change > 0\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'", "truth": 100, "ms": 5022, "input_tokens": 21, "output_tokens": 491, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE LOWER(event_type) IN ('expansion', 'upsell', 'upgrade', 'cross_sell', 'cross-sell')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 100, "ms": 5202, "input_tokens": 21, "output_tokens": 543, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND LOWER(event_type) IN ('expansion', 'upgrade', 'upsell', 'cross_sell', 'cross-sell', 'add_on', 'seat_expansion')\n AND mrr_change > 0", "truth": 100, "ms": 5015, "input_tokens": 21, "output_tokens": 526, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND mrr_change > 0\n AND (\n LOWER(event_type) IN ('expansion', 'upgrade', 'upsell', 'cross_sell', 'cross-sell')\n OR LOWER(expansion_classification) = 'expansion'\n )", "truth": 100, "ms": 6664, "input_tokens": 21, "output_tokens": 719, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND (LOWER(change_nature) = 'churn' OR LOWER(event_type) = 'churn')\n AND mrr_change < 0", "truth": 0, "ms": 4518, "input_tokens": 20, "output_tokens": 464, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "expansion_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(new_mrr - prior_mrr), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND prior_mrr > 0\n AND new_mrr > prior_mrr", "truth": 100, "ms": 5672, "input_tokens": 21, "output_tokens": 597, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE event_type ILIKE '%churn%'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 3820, "input_tokens": 20, "output_tokens": 405, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE LOWER(event_type) IN ('churn', 'churned', 'cancellation', 'cancelled')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 3779, "input_tokens": 20, "output_tokens": 415, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_mrr", "run": 3, "sql": "WITH churn_events AS (\n SELECT DISTINCT ON (event_id)\n event_id,\n mrr_change,\n prior_mrr,\n new_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND lower(trim(event_type)) LIKE 'churn%'\n ORDER BY event_id\n)\nSELECT COALESCE(\n SUM(ABS(COALESCE(mrr_change, new_mrr - prior_mrr, -prior_mrr))),\n 0\n ) AS churned_mrr\nFROM churn_events", "truth": 0, "ms": 4924, "input_tokens": 20, "output_tokens": 528, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(ABS(mrr_change)), 0) AS churned_mrr\nFROM gasp_revenue_events\nWHERE LOWER(event_type) = 'churn'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'", "truth": 0, "ms": 4038, "input_tokens": 20, "output_tokens": 389, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM (\n SELECT DISTINCT ON (event_id) event_id, mrr_change, prior_mrr, new_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n) e\nWHERE mrr_change < 0\n AND COALESCE(prior_mrr, 0) > 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 4865, "input_tokens": 21, "output_tokens": 523, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 2, "sql": "WITH ev AS (\n SELECT DISTINCT ON (event_id)\n event_id, prior_mrr, new_mrr, mrr_change\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n ORDER BY event_id\n)\nSELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM ev\nWHERE mrr_change < 0\n AND COALESCE(prior_mrr, 0) > 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 5829, "input_tokens": 21, "output_tokens": 591, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND mrr_change < 0\n AND COALESCE(prior_mrr, 0) > 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 6379, "input_tokens": 21, "output_tokens": 441, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM (\n SELECT DISTINCT ON (event_id) event_id, mrr_change, prior_mrr, new_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n ORDER BY event_id\n) e\nWHERE mrr_change < 0\n AND COALESCE(prior_mrr, 0) > 0\n AND COALESCE(new_mrr, 0) > 0", "truth": 400, "ms": 4924, "input_tokens": 21, "output_tokens": 514, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "contraction_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(-mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND mrr_change < 0\n AND COALESCE(new_mrr, prior_mrr + mrr_change) > 0", "truth": 400, "ms": 5014, "input_tokens": 21, "output_tokens": 477, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND (LOWER(event_type) LIKE '%reactivat%'\n OR LOWER(lifecycle_attribution) LIKE '%reactivat%')", "truth": 0, "ms": 4491, "input_tokens": 22, "output_tokens": 468, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE lower(event_type) IN ('reactivation', 'reactivated', 'reactivate')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 4836, "input_tokens": 22, "output_tokens": 485, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE LOWER(event_type) IN ('reactivation', 'reactivated', 'reactivate')\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 0, "ms": 3925, "input_tokens": 22, "output_tokens": 373, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND (\n LOWER(event_type) LIKE 'reactivat%'\n OR LOWER(lifecycle_attribution) LIKE 'reactivat%'\n )", "truth": 0, "ms": 4325, "input_tokens": 22, "output_tokens": 443, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "reactivation_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND (\n LOWER(event_type) LIKE '%reactivat%'\n OR LOWER(lifecycle_attribution) LIKE '%reactivat%'\n )", "truth": 0, "ms": 4195, "input_tokens": 22, "output_tokens": 430, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "carry_mrr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status, contract_start, contract_end\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM latest\nWHERE lower(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-02-28')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-01')", "truth": 16200, "ms": 5914, "input_tokens": 20, "output_tokens": 631, "cache_read": 6427, "value": 16600, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "carry_mrr", "run": 1, "sql": "WITH snap AS (\n SELECT s.*,\n ROW_NUMBER() OVER (PARTITION BY s.subscription_id ORDER BY s.snapshot_day DESC) AS rn\n FROM gasp_subscriptions s\n WHERE s.snapshot_day <= DATE '2026-02-28'\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM snap\nWHERE rn = 1\n AND LOWER(status) = 'active'\n AND (contract_start IS NULL OR contract_start <= DATE '2026-02-28')\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-01')", "truth": 16200, "ms": 6352, "input_tokens": 20, "output_tokens": 641, "cache_read": 6427, "value": 16600, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "carry_mrr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM latest\nWHERE status = 'active'", "truth": 16200, "ms": 5563, "input_tokens": 20, "output_tokens": 539, "cache_read": 6427, "value": 16600, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 4092, "input_tokens": 20, "output_tokens": 317, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "carry_mrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM latest\nWHERE LOWER(status) = 'active'", "truth": 16200, "ms": 6516, "input_tokens": 20, "output_tokens": 648, "cache_read": 6427, "value": 16600, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "carry_mrr", "run": 5, "sql": "WITH latest AS (\n SELECT s.*,\n ROW_NUMBER() OVER (PARTITION BY s.subscription_id ORDER BY s.snapshot_day DESC) AS rn\n FROM gasp_subscriptions s\n WHERE s.snapshot_day <= DATE '2026-02-28'\n)\nSELECT COALESCE(SUM(mrr), 0) AS carry_mrr\nFROM latest\nWHERE rn = 1\n AND LOWER(status) = 'active'\n AND contract_start <= DATE '2026-02-28'\n AND (contract_end IS NULL OR contract_end >= DATE '2026-03-01')", "truth": 16200, "ms": 6391, "input_tokens": 20, "output_tokens": 670, "cache_read": 6427, "value": 16600, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3518, "input_tokens": 20, "output_tokens": 325, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3601, "input_tokens": 20, "output_tokens": 307, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM (\n SELECT DISTINCT ON (event_id) event_id, mrr_change\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n ORDER BY event_id\n) e", "truth": 1800, "ms": 4600, "input_tokens": 20, "output_tokens": 403, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "total_customers", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, acquisition_date, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id)::numeric AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND COALESCE(LOWER(lifecycle_status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive', 'prospect', 'trial')", "truth": 15, "ms": 5420, "input_tokens": 19, "output_tokens": 567, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "net_new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 6368, "input_tokens": 20, "output_tokens": 303, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "total_customers", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, lifecycle_status, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id)::numeric AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND COALESCE(LOWER(lifecycle_status), '') NOT LIKE '%churn%'", "truth": 15, "ms": 5985, "input_tokens": 19, "output_tokens": 627, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "total_customers", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, lifecycle_status, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id)::numeric AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND COALESCE(lower(lifecycle_status), '') NOT LIKE '%churn%'\n AND COALESCE(lower(lifecycle_status), '') NOT IN ('cancelled', 'canceled', 'inactive', 'prospect', 'trial', 'lead', 'terminated')", "truth": 15, "ms": 6284, "input_tokens": 19, "output_tokens": 680, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "total_customers", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, acquisition_date, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(*)::numeric AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND COALESCE(lifecycle_status, '') NOT ILIKE '%churn%'", "truth": 15, "ms": 5326, "input_tokens": 19, "output_tokens": 572, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "total_customers", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, lifecycle_status, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM latest\nWHERE (acquisition_date IS NULL OR acquisition_date <= DATE '2026-03-31')\n AND COALESCE(LOWER(lifecycle_status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive', 'prospect', 'lead', 'trial')", "truth": 15, "ms": 5982, "input_tokens": 19, "output_tokens": 611, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_customers", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM latest\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date < DATE '2026-04-01'", "truth": 2, "ms": 3471, "input_tokens": 18, "output_tokens": 374, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM gasp_customers\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'", "truth": 2, "ms": 2260, "input_tokens": 18, "output_tokens": 209, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_customers", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM latest\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'", "truth": 2, "ms": 4005, "input_tokens": 18, "output_tokens": 449, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE LOWER(new_state) = 'churned'\n AND event_date >= DATE '2026-03-01'\n AND event_date <= DATE '2026-03-31'", "truth": 0, "ms": 2399, "input_tokens": 20, "output_tokens": 236, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_customers", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM latest\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'", "truth": 2, "ms": 3705, "input_tokens": 18, "output_tokens": 387, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "new_customers", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC NULLS LAST\n)\nSELECT COUNT(DISTINCT customer_id) AS new_customers\nFROM latest\nWHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'", "truth": 2, "ms": 4052, "input_tokens": 18, "output_tokens": 435, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE LOWER(new_state) = 'churned'\n AND (prior_state IS NULL OR LOWER(prior_state) <> 'churned')\n AND event_date >= DATE '2026-03-01'\n AND event_date <= DATE '2026-03-31'", "truth": 0, "ms": 3029, "input_tokens": 20, "output_tokens": 298, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id)::numeric AS churned_customers\nFROM gasp_lifecycle_events\nWHERE LOWER(new_state) = 'churned'\n AND event_date >= DATE '2026-03-01'\n AND event_date <= DATE '2026-03-31'", "truth": 0, "ms": 3255, "input_tokens": 20, "output_tokens": 311, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE event_date >= DATE '2026-03-01'\n AND event_date < DATE '2026-04-01'\n AND LOWER(new_state) LIKE 'churn%'\n AND (prior_state IS NULL OR LOWER(prior_state) NOT LIKE 'churn%')", "truth": 0, "ms": 3574, "input_tokens": 20, "output_tokens": 367, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "churned_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT customer_id) AS churned_customers\nFROM gasp_lifecycle_events\nWHERE LOWER(new_state) LIKE 'churn%'\n AND (prior_state IS NULL OR LOWER(prior_state) NOT LIKE 'churn%')\n AND event_date >= DATE '2026-03-01'\n AND event_date <= DATE '2026-03-31'", "truth": 0, "ms": 3373, "input_tokens": 20, "output_tokens": 335, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nrr", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS start_mrr\n FROM start_snap\n WHERE mrr > 0\n AND COALESCE(LOWER(status), '') NOT LIKE '%churn%'\n AND COALESCE(LOWER(status), '') NOT LIKE '%cancel%'\n GROUP BY customer_id\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS end_mrr\n FROM end_snap\n WHERE mrr > 0\n AND COALESCE(LOWER(status), '') NOT LIKE '%churn%'\n AND COALESCE(LOWER(status), '') NOT LIKE '%cancel%'\n GROUP BY customer_id\n)\nSELECT SUM(COALESCE(e.end_mrr, 0)) / NULLIF(SUM(s.start_mrr), 0) AS nrr\nFROM start_cust s\nLEFT JOIN end_cust e ON e.customer_id = s.customer_id", "truth": 0.9819277108433735, "ms": 10604, "input_tokens": 28, "output_tokens": 1169, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nrr", "run": 1, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS start_mrr\n FROM start_snap\n WHERE COALESCE(lower(status), '') NOT IN ('churned','cancelled','canceled','expired','inactive','terminated')\n GROUP BY customer_id\n HAVING SUM(mrr) > 0\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS end_mrr\n FROM end_snap\n WHERE COALESCE(lower(status), '') NOT IN ('churned','cancelled','canceled','expired','inactive','terminated')\n GROUP BY customer_id\n)\nSELECT SUM(COALESCE(e.end_mrr, 0)) / NULLIF(SUM(s.start_mrr), 0) AS nrr\nFROM start_cust s\nLEFT JOIN end_cust e ON e.customer_id = s.customer_id", "truth": 0.9819277108433735, "ms": 11485, "input_tokens": 28, "output_tokens": 1215, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nrr", "run": 4, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, COALESCE(mrr, 0) AS mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, COALESCE(mrr, 0) AS mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncohort AS (\n SELECT customer_id, SUM(mrr) AS start_mrr\n FROM start_snap\n GROUP BY customer_id\n HAVING SUM(mrr) > 0\n),\ncohort_end AS (\n SELECT e.customer_id, SUM(e.mrr) AS end_mrr\n FROM end_snap e\n JOIN cohort c ON c.customer_id = e.customer_id\n GROUP BY e.customer_id\n)\nSELECT ROUND(\n COALESCE((SELECT SUM(end_mrr) FROM cohort_end), 0)\n / NULLIF((SELECT SUM(start_mrr) FROM cohort), 0),\n 4) AS nrr", "truth": 0.9819277108433735, "ms": 11010, "input_tokens": 28, "output_tokens": 1193, "cache_read": 6427, "value": 0.9819, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nrr", "run": 3, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS start_mrr\n FROM start_snap\n WHERE COALESCE(LOWER(status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive')\n GROUP BY customer_id\n HAVING SUM(mrr) > 0\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS end_mrr\n FROM end_snap\n WHERE COALESCE(LOWER(status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive')\n GROUP BY customer_id\n)\nSELECT SUM(COALESCE(e.end_mrr, 0)) / NULLIF(SUM(s.start_mrr), 0) AS nrr\nFROM start_cust s\nLEFT JOIN end_cust e ON e.customer_id = s.customer_id", "truth": 0.9819277108433735, "ms": 11107, "input_tokens": 28, "output_tokens": 1198, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "grr", "run": 1, "sql": "WITH snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n),\nbase AS (\n SELECT s.subscription_id, SUM(s.mrr) AS mrr\n FROM gasp_subscriptions s\n JOIN snap ON s.snapshot_day = snap.d\n WHERE s.mrr > 0\n GROUP BY s.subscription_id\n),\nlosses AS (\n SELECT r.subscription_id, SUM(-r.mrr_change) AS lost\n FROM gasp_revenue_events r\n JOIN base b ON b.subscription_id = r.subscription_id\n WHERE r.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND r.mrr_change < 0\n GROUP BY r.subscription_id\n)\nSELECT (SUM(b.mrr) - SUM(LEAST(COALESCE(l.lost, 0), b.mrr))) / NULLIF(SUM(b.mrr), 0) AS grr\nFROM base b\nLEFT JOIN losses l ON l.subscription_id = b.subscription_id", "truth": 0.9759036144578314, "ms": 10087, "input_tokens": 29, "output_tokens": 960, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nrr", "run": 5, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS start_mrr\n FROM start_snap\n WHERE mrr > 0\n AND COALESCE(LOWER(status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive', 'expired')\n GROUP BY customer_id\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS end_mrr\n FROM end_snap\n WHERE mrr > 0\n AND COALESCE(LOWER(status), '') NOT IN ('churned', 'cancelled', 'canceled', 'inactive', 'expired')\n GROUP BY customer_id\n)\nSELECT SUM(COALESCE(e.end_mrr, 0)) / NULLIF(SUM(s.start_mrr), 0) AS nrr\nFROM start_cust s\nLEFT JOIN end_cust e ON e.customer_id = s.customer_id", "truth": 0.9819277108433735, "ms": 12579, "input_tokens": 28, "output_tokens": 1394, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "grr", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstarting AS (\n SELECT subscription_id, mrr\n FROM start_snap\n WHERE mrr > 0\n),\nlosses AS (\n SELECT e.subscription_id,\n SUM(-e.mrr_change) AS lost\n FROM gasp_revenue_events e\n JOIN starting s ON s.subscription_id = e.subscription_id\n WHERE e.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND e.mrr_change < 0\n GROUP BY e.subscription_id\n)\nSELECT (SUM(s.mrr) - SUM(LEAST(COALESCE(l.lost, 0), s.mrr))) / NULLIF(SUM(s.mrr), 0) AS grr\nFROM starting s\nLEFT JOIN losses l ON l.subscription_id = s.subscription_id", "truth": 0.9759036144578314, "ms": 8595, "input_tokens": 29, "output_tokens": 911, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "grr", "run": 3, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day < DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstarting AS (\n SELECT subscription_id, mrr\n FROM start_snap\n WHERE mrr > 0 AND COALESCE(LOWER(status), 'active') NOT IN ('churned', 'cancelled', 'canceled', 'inactive')\n),\nlosses AS (\n SELECT COALESCE(SUM(-r.mrr_change), 0) AS lost\n FROM gasp_revenue_events r\n JOIN starting s ON s.subscription_id = r.subscription_id\n WHERE r.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND r.mrr_change < 0\n)\nSELECT ((SELECT SUM(mrr) FROM starting) - (SELECT lost FROM losses))\n / NULLIF((SELECT SUM(mrr) FROM starting), 0) AS grr", "truth": 0.9759036144578314, "ms": 9793, "input_tokens": 29, "output_tokens": 996, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "grr", "run": 4, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncohort AS (\n SELECT s.subscription_id,\n s.mrr AS start_mrr,\n LEAST(s.mrr, GREATEST(COALESCE(e.mrr, 0), 0)) AS retained_mrr\n FROM start_snap s\n LEFT JOIN end_snap e ON e.subscription_id = s.subscription_id\n WHERE s.mrr > 0\n)\nSELECT SUM(retained_mrr) / NULLIF(SUM(start_mrr), 0) AS grr\nFROM cohort", "truth": 0.9759036144578314, "ms": 9302, "input_tokens": 29, "output_tokens": 995, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "grr", "run": 5, "sql": "WITH snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n),\nstart_subs AS (\n SELECT s.subscription_id, s.mrr\n FROM gasp_subscriptions s\n JOIN snap ON s.snapshot_day = snap.d\n WHERE s.mrr > 0\n),\nstart_mrr AS (\n SELECT SUM(mrr) AS v FROM start_subs\n),\nlosses AS (\n SELECT COALESCE(SUM(-e.mrr_change), 0) AS v\n FROM gasp_revenue_events e\n JOIN start_subs ss ON ss.subscription_id = e.subscription_id\n WHERE e.booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND e.mrr_change < 0\n)\nSELECT (start_mrr.v - losses.v) / NULLIF(start_mrr.v, 0) AS grr\nFROM start_mrr, losses", "truth": 0.9759036144578314, "ms": 9220, "input_tokens": 29, "output_tokens": 950, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 1, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM start_snap WHERE lower(lifecycle_status) = 'active'\n)\nSELECT\n COUNT(*) FILTER (WHERE lower(e.lifecycle_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM cohort c\nLEFT JOIN end_snap e ON e.customer_id = c.customer_id", "truth": 0, "ms": 9092, "input_tokens": 31, "output_tokens": 1005, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nbase AS (\n SELECT customer_id FROM start_snap\n WHERE lower(lifecycle_status) = 'active'\n)\nSELECT\n COUNT(*) FILTER (WHERE lower(e.lifecycle_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM base b\nLEFT JOIN end_snap e ON e.customer_id = b.customer_id", "truth": 0, "ms": 9222, "input_tokens": 31, "output_tokens": 969, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 3, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-02-28'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nbase AS (\n SELECT customer_id FROM start_snap WHERE lower(lifecycle_status) = 'active'\n)\nSELECT\n COUNT(*) FILTER (WHERE lower(e.lifecycle_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM base b\nLEFT JOIN end_snap e ON e.customer_id = b.customer_id", "truth": 0, "ms": 6906, "input_tokens": 31, "output_tokens": 762, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 4, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nbase AS (\n SELECT customer_id FROM start_snap WHERE lower(lifecycle_status) = 'active'\n)\nSELECT\n COUNT(*) FILTER (WHERE lower(e.lifecycle_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM base b\nLEFT JOIN end_snap e ON e.customer_id = b.customer_id", "truth": 0, "ms": 7167, "input_tokens": 31, "output_tokens": 735, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 1, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT SUM(mrr) AS mrr\n FROM start_snap\n WHERE lower(status) = 'active'\n),\nlost AS (\n SELECT SUM(ABS(mrr_change)) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(event_type) IN ('churn', 'contraction')\n)\nSELECT COALESCE(lost.lost_mrr, 0) / NULLIF(start_mrr.mrr, 0) AS revenue_churn_rate\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 8341, "input_tokens": 24, "output_tokens": 821, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 2, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT SUM(mrr) AS mrr\n FROM start_snap\n WHERE LOWER(status) = 'active'\n),\nlost AS (\n SELECT SUM(-mrr_change) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.lost_mrr / NULLIF(start_mrr.mrr, 0) AS revenue_churn_rate\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 9220, "input_tokens": 24, "output_tokens": 909, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "logo_churn_rate", "run": 5, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-02-28'\n ORDER BY customer_id, as_of_date DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nbase AS (\n SELECT s.customer_id, e.lifecycle_status AS end_status\n FROM start_snap s\n LEFT JOIN end_snap e ON e.customer_id = s.customer_id\n WHERE LOWER(s.lifecycle_status) = 'active'\n)\nSELECT\n COUNT(*) FILTER (WHERE LOWER(end_status) = 'churned')::numeric\n / NULLIF(COUNT(*), 0) AS logo_churn_rate\nFROM base", "truth": 0, "ms": 9412, "input_tokens": 31, "output_tokens": 932, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 3, "sql": "WITH snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nopening AS (\n SELECT SUM(mrr) AS opening_mrr\n FROM snap\n WHERE lower(status) = 'active' AND mrr > 0\n),\nlost AS (\n SELECT SUM(-mrr_change) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.lost_mrr / NULLIF(opening.opening_mrr, 0) AS revenue_churn_rate\nFROM lost, opening", "truth": 0.024096385542168676, "ms": 9260, "input_tokens": 24, "output_tokens": 950, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nps", "run": 1, "sql": "SELECT\n (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric\n - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric)\n / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date >= DATE '2026-03-01'\n AND survey_date < DATE '2026-04-01'", "truth": -25, "ms": 3372, "input_tokens": 27, "output_tokens": 341, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nps", "run": 2, "sql": "SELECT\n (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric\n - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric)\n / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'\n AND nps_score IS NOT NULL", "truth": -25, "ms": 3171, "input_tokens": 27, "output_tokens": 313, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nps", "run": 3, "sql": "SELECT\n (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric\n - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric)\n / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 3306, "input_tokens": 27, "output_tokens": 322, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nps", "run": 4, "sql": "SELECT\n (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric\n - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric)\n / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date >= DATE '2026-03-01'\n AND survey_date <= DATE '2026-03-31'", "truth": -25, "ms": 3276, "input_tokens": 27, "output_tokens": 312, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT SUM(mrr) AS mrr\n FROM latest\n WHERE lower(status) = 'active'\n),\nlost AS (\n SELECT COALESCE(SUM(-mrr_change), 0) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.lost_mrr / NULLIF(start_mrr.mrr, 0) AS revenue_churn_rate\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 9635, "input_tokens": 24, "output_tokens": 991, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "nps", "run": 5, "sql": "SELECT\n (COUNT(*) FILTER (WHERE nps_score >= 9)::numeric\n - COUNT(*) FILTER (WHERE nps_score <= 6)::numeric)\n / NULLIF(COUNT(*), 0) AS nps\nFROM gasp_survey_responses\nWHERE nps_score IS NOT NULL\n AND survey_date >= DATE '2026-03-01'\n AND survey_date < DATE '2026-04-01'", "truth": -25, "ms": 2944, "input_tokens": 27, "output_tokens": 314, "cache_read": 6427, "value": -0.16666666666666666, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "revenue_churn_rate", "run": 5, "sql": "WITH opening AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT SUM(mrr) AS mrr\n FROM opening\n WHERE status = 'active'\n),\nlost AS (\n SELECT SUM(-mrr_change) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND mrr_change < 0\n)\nSELECT lost.lost_mrr / NULLIF(start_mrr.mrr, 0) AS revenue_churn_rate\nFROM lost, start_mrr", "truth": 0.024096385542168676, "ms": 8061, "input_tokens": 24, "output_tokens": 846, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac", "run": 1, "sql": "WITH acq_costs AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND lower(lifecycle_attribution) = 'acquisition'\n),\nnew_customers AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'\n)\nSELECT acq_costs.total_cost / NULLIF(new_customers.n, 0) AS cac\nFROM acq_costs, new_customers", "truth": 3000, "ms": 5485, "input_tokens": 30, "output_tokens": 599, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac", "run": 4, "sql": "WITH acq_cost AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE LOWER(lifecycle_attribution) = 'acquisition'\n AND incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n),\nnew_customers AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date < DATE '2026-04-01'\n)\nSELECT acq_cost.total_cost / NULLIF(new_customers.n, 0) AS cac\nFROM acq_cost, new_customers", "truth": 3000, "ms": 5523, "input_tokens": 30, "output_tokens": 608, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac", "run": 2, "sql": "WITH sm_cost AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND (LOWER(cost_function) LIKE '%sales%' OR LOWER(cost_function) LIKE '%marketing%')\n AND (lifecycle_attribution IS NULL\n OR LOWER(lifecycle_attribution) LIKE '%acqui%'\n OR LOWER(lifecycle_attribution) LIKE '%new%')\n),\nnew_customers AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date < DATE '2026-04-01'\n)\nSELECT sm_cost.total_cost / NULLIF(new_customers.n, 0) AS cac\nFROM sm_cost, new_customers", "truth": 3000, "ms": 8620, "input_tokens": 30, "output_tokens": 985, "cache_read": 6427, "value": 0, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac", "run": 3, "sql": "WITH spend AS (\n SELECT SUM(cost) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND LOWER(cost_function) IN ('sales', 'marketing')\n AND LOWER(lifecycle_attribution) LIKE 'acqui%'\n),\nnew_customers AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date <= DATE '2026-03-31'\n)\nSELECT s.total_cost / NULLIF(c.n, 0) AS cac\nFROM spend s\nCROSS JOIN new_customers c", "truth": 3000, "ms": 8439, "input_tokens": 30, "output_tokens": 893, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac", "run": 5, "sql": "WITH acq_costs AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND lower(lifecycle_attribution) LIKE 'acqui%'\n),\nlatest_customers AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n ORDER BY customer_id, as_of_date DESC\n),\nnew_customers AS (\n SELECT COUNT(*) AS n\n FROM latest_customers\n WHERE acquisition_date >= DATE '2026-03-01'\n AND acquisition_date < DATE '2026-04-01'\n)\nSELECT acq_costs.total_cost / NULLIF(new_customers.n, 0) AS cac\nFROM acq_costs, new_customers", "truth": 3000, "ms": 8213, "input_tokens": 30, "output_tokens": 936, "cache_read": 6427, "value": 3350, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv", "run": 1, "sql": "WITH s_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\ns_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM s_start\n WHERE status = 'active' AND mrr > 0\n GROUP BY customer_id\n),\nend_cust AS (\n SELECT DISTINCT customer_id\n FROM s_end\n WHERE status = 'active' AND mrr > 0\n),\nstats AS (\n SELECT\n COUNT(*)::numeric AS start_customers,\n SUM(sc.mrr) AS start_mrr,\n COUNT(*) FILTER (WHERE ec.customer_id IS NULL)::numeric AS churned_customers\n FROM start_cust sc\n LEFT JOIN end_cust ec ON ec.customer_id = sc.customer_id\n)\nSELECT (start_mrr / NULLIF(start_customers, 0))\n / NULLIF(churned_customers / NULLIF(start_customers, 0), 0) AS ltv\nFROM stats", "truth": 47870.948012232424, "ms": 8616, "input_tokens": 29, "output_tokens": 1047, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv", "run": 2, "sql": "WITH start_subs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_subs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT DISTINCT customer_id FROM start_subs WHERE lower(status) = 'active'\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM end_subs WHERE lower(status) = 'active'\n GROUP BY customer_id\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric\n / NULLIF(COUNT(*), 0) AS churn_rate\n FROM start_cust s\n LEFT JOIN end_cust e ON e.customer_id = s.customer_id\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(*), 0) AS arpa FROM end_cust\n)\nSELECT arpa.arpa / NULLIF(churn.churn_rate, 0) AS ltv\nFROM arpa, churn", "truth": 47870.948012232424, "ms": 10051, "input_tokens": 29, "output_tokens": 1052, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv", "run": 3, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM start_snap\n WHERE LOWER(status) = 'active'\n GROUP BY customer_id\n),\nend_cust AS (\n SELECT DISTINCT customer_id\n FROM end_snap\n WHERE LOWER(status) = 'active'\n),\nmetrics AS (\n SELECT\n COUNT(*) AS start_customers,\n SUM(s.mrr) AS start_mrr,\n COUNT(*) FILTER (WHERE e.customer_id IS NULL) AS churned_customers\n FROM start_cust s\n LEFT JOIN end_cust e ON e.customer_id = s.customer_id\n)\nSELECT\n (start_mrr / NULLIF(start_customers, 0))\n / NULLIF(churned_customers::numeric / NULLIF(start_customers, 0), 0) AS ltv\nFROM metrics", "truth": 47870.948012232424, "ms": 9720, "input_tokens": 29, "output_tokens": 1131, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv", "run": 4, "sql": "WITH start_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nend_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_customers AS (\n SELECT DISTINCT customer_id\n FROM start_snap\n WHERE status = 'active' AND mrr > 0\n),\nend_customers AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM end_snap\n WHERE status = 'active' AND mrr > 0\n GROUP BY customer_id\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric\n / NULLIF(COUNT(*), 0) AS churn_rate\n FROM start_customers s\n LEFT JOIN end_customers e ON e.customer_id = s.customer_id\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(*), 0) AS arpa\n FROM end_customers\n)\nSELECT arpa.arpa / NULLIF(churn.churn_rate, 0) AS ltv\nFROM arpa\nCROSS JOIN churn", "truth": 47870.948012232424, "ms": 9172, "input_tokens": 29, "output_tokens": 1028, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv", "run": 5, "sql": "WITH s_start AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\ns_end AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT DISTINCT customer_id\n FROM s_start\n WHERE lower(status) = 'active' AND COALESCE(mrr, 0) > 0\n),\nend_cust AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM s_end\n WHERE lower(status) = 'active' AND COALESCE(mrr, 0) > 0\n GROUP BY customer_id\n),\nchurn AS (\n SELECT\n COUNT(*)::numeric AS start_n,\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric AS churned_n\n FROM start_cust s\n LEFT JOIN end_cust e ON e.customer_id = s.customer_id\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(*), 0)::numeric AS arpa\n FROM end_cust\n)\nSELECT a.arpa / NULLIF(c.churned_n / NULLIF(c.start_n, 0), 0) AS ltv\nFROM arpa a\nCROSS JOIN churn c", "truth": 47870.948012232424, "ms": 9318, "input_tokens": 29, "output_tokens": 1118, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 2, "sql": "WITH subs_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsubs_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_cust AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM subs_start\n WHERE lower(status) = 'active'\n GROUP BY customer_id\n),\nend_cust AS (\n SELECT DISTINCT customer_id\n FROM subs_end\n WHERE lower(status) = 'active'\n),\nchurn AS (\n SELECT\n COUNT(*)::numeric AS start_count,\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric AS churned,\n SUM(s.mrr) AS start_mrr\n FROM start_cust s\n LEFT JOIN end_cust e ON e.customer_id = s.customer_id\n),\ncac AS (\n SELECT\n (SELECT SUM(cost) FROM gasp_cost_events\n WHERE lower(lifecycle_attribution) = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31') AS acq_cost,\n (SELECT COUNT(DISTINCT customer_id) FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31')::numeric AS new_customers\n)\nSELECT\n ((c.start_mrr / NULLIF(c.start_count, 0)) / NULLIF(c.churned / NULLIF(c.start_count, 0), 0))\n / NULLIF(a.acq_cost / NULLIF(a.new_customers, 0), 0) AS ltv_cac_ratio\nFROM churn c CROSS JOIN cac a", "truth": 15.956982670744141, "ms": 13507, "input_tokens": 25, "output_tokens": 1556, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 1, "sql": "WITH subs_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsubs_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive_start AS (\n SELECT DISTINCT customer_id FROM subs_start WHERE lower(status) = 'active'\n),\nactive_end AS (\n SELECT DISTINCT customer_id FROM subs_end WHERE lower(status) = 'active'\n),\nchurn AS (\n SELECT\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric\n / NULLIF(COUNT(*), 0) AS churn_rate\n FROM active_start s\n LEFT JOIN active_end e ON e.customer_id = s.customer_id\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(DISTINCT customer_id), 0) AS arpa\n FROM subs_end\n WHERE lower(status) = 'active'\n),\nacq_cost AS (\n SELECT SUM(cost) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(lifecycle_attribution) LIKE 'acqui%'\n),\nnew_customers AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM gasp_customers\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT\n (a.arpa / NULLIF(c.churn_rate, 0))\n / NULLIF(ac.total_cost / NULLIF(nc.n, 0), 0) AS ltv_cac_ratio\nFROM arpa a, churn c, acq_cost ac, new_customers nc", "truth": 15.956982670744141, "ms": 15556, "input_tokens": 25, "output_tokens": 1795, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac_payback", "run": 1, "sql": "WITH acq_cost AS (\n SELECT SUM(cost) AS total_cost\n FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n),\nnew_mrr AS (\n SELECT SUM(mrr_change) AS total_new_mrr\n FROM gasp_revenue_events\n WHERE lifecycle_attribution = 'acquisition'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT acq_cost.total_cost / NULLIF(new_mrr.total_new_mrr, 0) AS cac_payback_months\nFROM acq_cost, new_mrr", "truth": 2.600742311770943, "ms": 8479, "input_tokens": 22, "output_tokens": 873, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 5, "sql": "WITH start_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28'\n),\nend_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31'\n),\nstart_active AS (\n SELECT DISTINCT s.customer_id\n FROM gasp_subscriptions s JOIN start_snap ss ON s.snapshot_day = ss.d\n WHERE s.status = 'active'\n),\nend_active AS (\n SELECT s.customer_id, SUM(s.mrr) AS mrr\n FROM gasp_subscriptions s JOIN end_snap es ON s.snapshot_day = es.d\n WHERE s.status = 'active'\n GROUP BY s.customer_id\n),\nchurn AS (\n SELECT COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric\n / NULLIF(COUNT(*), 0) AS churn_rate\n FROM start_active sa LEFT JOIN end_active e ON e.customer_id = sa.customer_id\n),\narpa AS (\n SELECT SUM(mrr) / NULLIF(COUNT(*), 0) AS arpa FROM end_active\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nnew_cust AS (\n SELECT COUNT(*) AS n FROM cust\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nacq_cost AS (\n SELECT SUM(cost) AS c FROM gasp_cost_events\n WHERE lifecycle_attribution = 'acquisition'\n AND incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT (a.arpa / NULLIF(ch.churn_rate, 0))\n / NULLIF(ac.c / NULLIF(nc.n, 0), 0) AS ltv_cac_ratio\nFROM arpa a, churn ch, new_cust nc, acq_cost ac", "truth": 15.956982670744141, "ms": 11292, "input_tokens": 25, "output_tokens": 1368, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 4, "sql": "WITH sub_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsub_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, status, mrr\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_customers AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM sub_start\n WHERE status = 'active'\n GROUP BY customer_id\n),\nend_customers AS (\n SELECT DISTINCT customer_id\n FROM sub_end\n WHERE status = 'active'\n),\nchurn AS (\n SELECT\n COUNT(*)::numeric AS start_cnt,\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric AS churned_cnt,\n SUM(s.mrr) AS start_mrr\n FROM start_customers s\n LEFT JOIN end_customers e ON e.customer_id = s.customer_id\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) customer_id, acquisition_date\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nnew_customers AS (\n SELECT COUNT(*)::numeric AS n\n FROM cust\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nsm_cost AS (\n SELECT SUM(cost) AS total\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND (cost_function ILIKE '%sales%' OR cost_function ILIKE '%marketing%')\n)\nSELECT\n ((c.start_mrr / NULLIF(c.start_cnt, 0)) / NULLIF(c.churned_cnt / NULLIF(c.start_cnt, 0), 0))\n / NULLIF(sm.total / NULLIF(nc.n, 0), 0) AS ltv_cac_ratio\nFROM churn c\nCROSS JOIN new_customers nc\nCROSS JOIN sm_cost sm", "truth": 15.956982670744141, "ms": 12333, "input_tokens": 25, "output_tokens": 1500, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "ltv_cac_ratio", "run": 3, "sql": "WITH sub_end AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsub_start AS (\n SELECT DISTINCT ON (subscription_id) *\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nactive_end AS (\n SELECT customer_id, SUM(mrr) AS mrr\n FROM sub_end\n WHERE lower(status) = 'active'\n GROUP BY customer_id\n HAVING SUM(mrr) > 0\n),\nactive_start AS (\n SELECT DISTINCT customer_id\n FROM sub_start\n WHERE lower(status) = 'active' AND mrr > 0\n),\nchurn AS (\n SELECT COUNT(*)::numeric AS start_cnt,\n COUNT(*) FILTER (WHERE e.customer_id IS NULL)::numeric AS churned_cnt\n FROM active_start s\n LEFT JOIN active_end e ON e.customer_id = s.customer_id\n),\narpa AS (\n SELECT AVG(mrr) AS arpa FROM active_end\n),\ncust AS (\n SELECT DISTINCT ON (customer_id) *\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nnew_cust AS (\n SELECT COUNT(*)::numeric AS n\n FROM cust\n WHERE acquisition_date BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nsm_cost AS (\n SELECT SUM(cost) AS c\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND lower(cost_function) IN ('sales', 'marketing')\n)\nSELECT (a.arpa / NULLIF(ch.churned_cnt / NULLIF(ch.start_cnt, 0), 0))\n / NULLIF(s.c / NULLIF(n.n, 0), 0) AS ltv_cac_ratio\nFROM arpa a, churn ch, new_cust n, sm_cost s", "truth": 15.956982670744141, "ms": 14860, "input_tokens": 25, "output_tokens": 1778, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac_payback", "run": 2, "sql": "WITH acq_cost AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND LOWER(lifecycle_attribution) = 'acquisition'\n),\nnew_mrr AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS total_new_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND LOWER(change_nature) = 'new'\n)\nSELECT acq_cost.total_cost / NULLIF(new_mrr.total_new_mrr, 0) AS cac_payback_months\nFROM acq_cost, new_mrr", "truth": 2.600742311770943, "ms": 7157, "input_tokens": 22, "output_tokens": 849, "cache_read": 6427, "value": null, "category": "null" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac_payback", "run": 3, "sql": "WITH acq_cost AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE lower(lifecycle_attribution) = 'acquisition'\n AND incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n),\nnew_mrr AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS total_new_mrr\n FROM gasp_revenue_events\n WHERE lower(lifecycle_attribution) = 'acquisition'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n)\nSELECT acq_cost.total_cost / NULLIF(new_mrr.total_new_mrr, 0) AS cac_payback_months\nFROM acq_cost, new_mrr", "truth": 2.600742311770943, "ms": 7356, "input_tokens": 22, "output_tokens": 855, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "gross_margin", "run": 1, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start < DATE '2026-04-01'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND (\n lower(cost_type) IN ('cogs', 'cost_of_goods_sold', 'cost of goods sold', 'cost_of_revenue', 'cost of revenue', 'direct')\n OR lower(cost_type) LIKE '%cogs%'\n )\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 5410, "input_tokens": 20, "output_tokens": 594, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac_payback", "run": 4, "sql": "WITH acq_cost AS (\n SELECT COALESCE(SUM(cost), 0) AS total_cost\n FROM gasp_cost_events\n WHERE LOWER(lifecycle_attribution) = 'acquisition'\n AND incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n),\nnew_mrr AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS total_new_mrr\n FROM gasp_revenue_events\n WHERE LOWER(lifecycle_attribution) = 'acquisition'\n AND booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT acq_cost.total_cost / NULLIF(new_mrr.total_new_mrr, 0) AS cac_payback_months\nFROM acq_cost, new_mrr", "truth": 2.600742311770943, "ms": 8099, "input_tokens": 22, "output_tokens": 903, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "gross_margin", "run": 2, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start < DATE '2026-04-01'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND (\n cost_type ILIKE '%cogs%'\n OR cost_type ILIKE '%cost of goods%'\n OR cost_type ILIKE '%cost_of_goods%'\n OR cost_type ILIKE '%cost of revenue%'\n OR cost_type ILIKE '%cost_of_revenue%'\n )\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev\nCROSS JOIN cogs", "truth": 0.9403669724770642, "ms": 5579, "input_tokens": 20, "output_tokens": 641, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "cac_payback", "run": 5, "sql": "WITH cac_spend AS (\n SELECT COALESCE(SUM(cost), 0) AS spend\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND lower(lifecycle_attribution) LIKE 'acqui%'\n),\nnew_mrr AS (\n SELECT COALESCE(SUM(mrr_change), 0) AS mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on < DATE '2026-04-01'\n AND lower(event_type) LIKE 'new%'\n)\nSELECT c.spend / NULLIF(n.mrr, 0) AS cac_payback_months\nFROM cac_spend c\nCROSS JOIN new_mrr n", "truth": 2.600742311770943, "ms": 7464, "input_tokens": 22, "output_tokens": 836, "cache_read": 6427, "value": 3.1904761904761907, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "gross_margin", "run": 3, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start <= DATE '2026-03-31'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND (\n cost_type ILIKE '%cogs%'\n OR cost_type ILIKE '%cost of%'\n OR cost_type ILIKE '%cost_of%'\n )\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 5539, "input_tokens": 20, "output_tokens": 640, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "gross_margin", "run": 4, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start < DATE '2026-04-01'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on < DATE '2026-04-01'\n AND (lower(cost_type) LIKE '%cogs%'\n OR lower(cost_type) LIKE '%cost of%'\n OR lower(cost_type) LIKE '%cost_of%')\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev CROSS JOIN cogs", "truth": 0.9403669724770642, "ms": 5320, "input_tokens": 20, "output_tokens": 604, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr", "run": 1, "sql": "SELECT COALESCE(SUM(s.mrr), 0) AS mrr\nFROM gasp_subscriptions s\nWHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(s.status)) = 'active'", "truth": 18400, "ms": 3878, "input_tokens": 1011, "output_tokens": 393, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr", "run": 2, "sql": "SELECT COALESCE(SUM(s.mrr), 0) AS mrr\nFROM gasp_subscriptions s\nWHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(s.status)) = 'active'", "truth": 18400, "ms": 3775, "input_tokens": 1011, "output_tokens": 376, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr", "run": 4, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) AS mrr\nFROM latest\nWHERE LOWER(TRIM(status)) = 'active'", "truth": 18400, "ms": 2870, "input_tokens": 1011, "output_tokens": 311, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "raw", "metric": "gross_margin", "run": 5, "sql": "WITH rev AS (\n SELECT SUM(revenue) AS revenue\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start < DATE '2026-04-01'\n),\ncogs AS (\n SELECT SUM(cost) AS cogs\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on < DATE '2026-04-01'\n AND lower(cost_type) IN ('cogs', 'cost_of_goods_sold', 'cost of goods sold', 'cost_of_revenue', 'cost of revenue')\n)\nSELECT (rev.revenue - COALESCE(cogs.cogs, 0)) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev CROSS JOIN cogs", "truth": 0.9403669724770642, "ms": 5450, "input_tokens": 20, "output_tokens": 638, "cache_read": 6427, "value": 1, "category": "wrong value" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr", "run": 3, "sql": "SELECT COALESCE(SUM(s.mrr), 0) AS mrr\nFROM gasp_subscriptions s\nWHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(s.status)) = 'active'", "truth": 18400, "ms": 3798, "input_tokens": 1011, "output_tokens": 349, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr", "run": 5, "sql": "SELECT COALESCE(SUM(s.mrr), 0) AS mrr\nFROM gasp_subscriptions s\nWHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n )\n AND LOWER(TRIM(s.status)) = 'active'", "truth": 18400, "ms": 4078, "input_tokens": 1011, "output_tokens": 392, "cache_read": 6427, "value": 18400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arr", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr) FILTER (WHERE status = 'active'), 0) * 12 AS arr\nFROM latest", "truth": 220800, "ms": 2956, "input_tokens": 1417, "output_tokens": 310, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arr", "run": 2, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr) FILTER (WHERE status = 'active'), 0) * 12 AS arr\nFROM latest", "truth": 220800, "ms": 4405, "input_tokens": 1417, "output_tokens": 434, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arr", "run": 4, "sql": "SELECT COALESCE(SUM(s.mrr), 0) * 12 AS arr\nFROM gasp_subscriptions s\nWHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n )\n AND LOWER(TRIM(s.status)) = 'active'", "truth": 220800, "ms": 3882, "input_tokens": 1417, "output_tokens": 382, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arr", "run": 3, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(TRIM(status)) = 'active'", "truth": 220800, "ms": 5070, "input_tokens": 1417, "output_tokens": 483, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arr", "run": 1, "sql": "WITH latest AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n)\nSELECT COALESCE(SUM(mrr), 0) * 12 AS arr\nFROM latest\nWHERE LOWER(TRIM(status)) = 'active'", "truth": 220800, "ms": 5512, "input_tokens": 1417, "output_tokens": 518, "cache_read": 6427, "value": 220800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arpa", "run": 2, "sql": "SELECT\n CAST((\n SELECT COALESCE(SUM(s.mrr), 0)\n FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (\n SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31'\n )\n ) AS NUMERIC)\n / NULLIF((\n SELECT COUNT(DISTINCT c.customer_id)\n FROM gasp_customers c\n WHERE c.lifecycle_status = 'active'\n AND c.as_of_date = (\n SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31'\n )\n ), 0) AS arpa", "truth": 1226.6666666666667, "ms": 4393, "input_tokens": 1268, "output_tokens": 457, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arpa", "run": 1, "sql": "WITH sub_snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n),\ncust_snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n JOIN sub_snap ss ON s.snapshot_day = ss.d\n WHERE LOWER(TRIM(s.status)) = 'active'\n),\ncust AS (\n SELECT COUNT(DISTINCT c.customer_id) AS v\n FROM gasp_customers c\n JOIN cust_snap cs ON c.as_of_date = cs.d\n WHERE LOWER(TRIM(c.lifecycle_status)) = 'active'\n)\nSELECT mrr.v::numeric / NULLIF(cust.v, 0) AS arpa\nFROM mrr, cust", "truth": 1226.6666666666667, "ms": 5334, "input_tokens": 1268, "output_tokens": 580, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arpa", "run": 4, "sql": "SELECT\n CAST((\n SELECT COALESCE(SUM(s.mrr), 0)\n FROM gasp_subscriptions s\n WHERE s.snapshot_day = (\n SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n )\n AND LOWER(TRIM(s.status)) = 'active'\n ) AS DOUBLE PRECISION)\n / NULLIF((\n SELECT COUNT(DISTINCT c.customer_id)\n FROM gasp_customers c\n WHERE c.as_of_date = (\n SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n )\n AND LOWER(TRIM(c.lifecycle_status)) = 'active'\n ), 0) AS arpa", "truth": 1226.6666666666667, "ms": 4861, "input_tokens": 1268, "output_tokens": 510, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arpa", "run": 5, "sql": "WITH sub_snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n),\ncust_snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n JOIN sub_snap ss ON s.snapshot_day = ss.d\n WHERE s.status = 'active'\n),\ncust AS (\n SELECT COUNT(DISTINCT c.customer_id) AS v\n FROM gasp_customers c\n JOIN cust_snap cs ON c.as_of_date = cs.d\n WHERE c.lifecycle_status = 'active'\n)\nSELECT CAST(mrr.v AS NUMERIC) / NULLIF(cust.v, 0) AS arpa\nFROM mrr, cust", "truth": 1226.6666666666667, "ms": 4348, "input_tokens": 1268, "output_tokens": 523, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 1, "sql": "WITH mar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nfeb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nm AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM mar WHERE lower(trim(status)) = 'active') AS cur,\n (SELECT COALESCE(SUM(mrr), 0) FROM feb WHERE lower(trim(status)) = 'active') AS prev\n)\nSELECT (cur - prev)::numeric / NULLIF(prev, 0) AS mrr_growth_rate\nFROM m", "truth": 0.10843373493975904, "ms": 4611, "input_tokens": 1139, "output_tokens": 543, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "arpa", "run": 3, "sql": "WITH sub_snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n),\ncust_snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n JOIN sub_snap ss ON s.snapshot_day = ss.d\n WHERE s.status = 'active'\n),\ncust AS (\n SELECT COUNT(DISTINCT c.customer_id) AS v\n FROM gasp_customers c\n JOIN cust_snap cs ON c.as_of_date = cs.d\n WHERE c.lifecycle_status = 'active'\n)\nSELECT CAST(mrr.v AS NUMERIC) / NULLIF(cust.v, 0) AS arpa\nFROM mrr, cust", "truth": 1226.6666666666667, "ms": 5752, "input_tokens": 1268, "output_tokens": 660, "cache_read": 6427, "value": 1226.6666666666667, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1445, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1533, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1538, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1538, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 2, "sql": "WITH cur AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nprev AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nm AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM cur WHERE LOWER(TRIM(status)) = 'active') AS mrr_cur,\n (SELECT COALESCE(SUM(mrr), 0) FROM prev WHERE LOWER(TRIM(status)) = 'active') AS mrr_prev\n)\nSELECT (mrr_cur - mrr_prev)::numeric / NULLIF(mrr_prev, 0) AS mrr_growth_rate\nFROM m", "truth": 0.10843373493975904, "ms": 5521, "input_tokens": 1139, "output_tokens": 626, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 5, "sql": "WITH cur AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nprev AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nm AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM cur WHERE status = 'active') AS cur_mrr,\n (SELECT COALESCE(SUM(mrr), 0) FROM prev WHERE status = 'active') AS prev_mrr\n)\nSELECT (cur_mrr - prev_mrr)::numeric / NULLIF(prev_mrr, 0) AS mrr_growth_rate\nFROM m", "truth": 0.10843373493975904, "ms": 4875, "input_tokens": 1139, "output_tokens": 578, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1423, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 4, "sql": "WITH mar AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nfeb AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-02-28'\n ORDER BY subscription_id, snapshot_day DESC\n),\nm AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM mar WHERE lower(trim(status)) = 'active') AS cur,\n (SELECT COALESCE(SUM(mrr), 0) FROM feb WHERE lower(trim(status)) = 'active') AS prev\n)\nSELECT (cur - prev) / NULLIF(prev, 0) AS mrr_growth_rate\nFROM m", "truth": 0.10843373493975904, "ms": 5119, "input_tokens": 1139, "output_tokens": 605, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2100, "ms": 1467, "input_tokens": 854, "output_tokens": 106, "cache_read": 6427, "value": 2100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1303, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "mrr_growth_rate", "run": 3, "sql": "WITH cur AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nprev AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-02-28')\n)\nSELECT (cur.mrr - prev.mrr)::numeric / NULLIF(prev.mrr, 0) AS mrr_growth_rate\nFROM cur, prev", "truth": 0.10843373493975904, "ms": 6969, "input_tokens": 1139, "output_tokens": 794, "cache_read": 6427, "value": 0.10843373493975904, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1408, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1354, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "expansion_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS expansion_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'expansion'", "truth": 100, "ms": 1380, "input_tokens": 897, "output_tokens": 110, "cache_read": 6427, "value": 100, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 1, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS churned_mrr", "truth": 0, "ms": 1397, "input_tokens": 903, "output_tokens": 108, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 2, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS churned_mrr", "truth": 0, "ms": 1623, "input_tokens": 903, "output_tokens": 108, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 3, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn') AS churned_mrr", "truth": 0, "ms": 1486, "input_tokens": 903, "output_tokens": 116, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 4, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS churned_mrr", "truth": 0, "ms": 1411, "input_tokens": 903, "output_tokens": 108, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 1, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction') AS contraction_mrr", "truth": 400, "ms": 1375, "input_tokens": 918, "output_tokens": 110, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_mrr", "run": 5, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn') AS churned_mrr", "truth": 0, "ms": 1467, "input_tokens": 903, "output_tokens": 108, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 2, "sql": "SELECT -(COALESCE(SUM(mrr_change), 0)) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1487, "input_tokens": 918, "output_tokens": 113, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 3, "sql": "SELECT -COALESCE(SUM(mrr_change), 0) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1460, "input_tokens": 918, "output_tokens": 111, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 4, "sql": "SELECT -(COALESCE(SUM(mrr_change), 0)) AS contraction_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'", "truth": 400, "ms": 1398, "input_tokens": 918, "output_tokens": 113, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "contraction_mrr", "run": 5, "sql": "SELECT -(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction') AS contraction_mrr", "truth": 400, "ms": 1346, "input_tokens": 918, "output_tokens": 110, "cache_read": 6427, "value": 400, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 2, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1786, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 1, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1918, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 3, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1891, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 5, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1800, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "reactivation_mrr", "run": 4, "sql": "SELECT COALESCE(SUM(mrr_change), 0) AS reactivation_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'reactivation'", "truth": 0, "ms": 1846, "input_tokens": 925, "output_tokens": 112, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 2, "sql": "WITH snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbegin_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM snap WHERE status = 'active'\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn_change,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction_change\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT b.v - (-e.churn_change) - (-e.contraction_change) AS carry_mrr\nFROM begin_mrr b CROSS JOIN ev e", "truth": 16200, "ms": 4369, "input_tokens": 1343, "output_tokens": 520, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 1, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0))\n AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 3935, "input_tokens": 1507, "output_tokens": 408, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 1, "sql": "WITH latest_sub AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbeginning AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM latest_sub\n WHERE status = 'active'\n),\nchurned AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS amt\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS amt\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n)\nSELECT b.mrr - c.amt - ct.amt AS carry_mrr\nFROM beginning b, churned c, contraction ct", "truth": 16200, "ms": 5358, "input_tokens": 1343, "output_tokens": 621, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 4, "sql": "WITH sub_at_start AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbeginning AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM sub_at_start\n WHERE status = 'active'\n),\nchurned AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n)\nSELECT (SELECT v FROM beginning) - (SELECT v FROM churned) - (SELECT v FROM contraction) AS carry_mrr", "truth": 16200, "ms": 5602, "input_tokens": 1343, "output_tokens": 672, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 3, "sql": "WITH snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbegin_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM snap WHERE status = 'active'\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n)\nSELECT (SELECT v FROM begin_mrr) - (SELECT v FROM churn) - (SELECT v FROM contraction) AS carry_mrr", "truth": 16200, "ms": 5730, "input_tokens": 1343, "output_tokens": 682, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "carry_mrr", "run": 5, "sql": "WITH sub_at_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nbeginning AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM sub_at_start\n WHERE status = 'active'\n),\nchurned AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n)\nSELECT (SELECT v FROM beginning) - (SELECT v FROM churned) - (SELECT v FROM contraction) AS carry_mrr", "truth": 16200, "ms": 6550, "input_tokens": 1343, "output_tokens": 702, "cache_read": 6427, "value": 16200, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 4, "sql": "SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'new'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0)\n + COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'reactivation'), 0)\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0))\n - (-COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0))\n AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 2540, "input_tokens": 1507, "output_tokens": 290, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 3, "sql": "SELECT\n (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')\n+ (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'expansion')\n+ (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'reactivation')\n- (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n- (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n AS net_new_mrr", "truth": 1800, "ms": 3620, "input_tokens": 1507, "output_tokens": 479, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 2, "sql": "SELECT\n COALESCE(SUM(CASE WHEN lower(trim(event_type)) = 'new' THEN mrr_change END), 0)\n + COALESCE(SUM(CASE WHEN lower(trim(event_type)) = 'expansion' THEN mrr_change END), 0)\n + COALESCE(SUM(CASE WHEN lower(trim(event_type)) = 'reactivation' THEN mrr_change END), 0)\n - (-COALESCE(SUM(CASE WHEN lower(trim(event_type)) = 'churn' THEN mrr_change END), 0))\n - (-COALESCE(SUM(CASE WHEN lower(trim(event_type)) = 'contraction' THEN mrr_change END), 0))\n AS net_new_mrr\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'", "truth": 1800, "ms": 4292, "input_tokens": 1507, "output_tokens": 491, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "net_new_mrr", "run": 5, "sql": "SELECT\n (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new')\n+ (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'expansion')\n+ (SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'reactivation')\n- (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'churn'))\n- (-(SELECT COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'contraction'))\n AS net_new_mrr", "truth": 1800, "ms": 3648, "input_tokens": 1507, "output_tokens": 479, "cache_read": 6427, "value": 1800, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "total_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM gasp_customers\nWHERE as_of_date = (\n SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(lifecycle_status)) = 'active'", "truth": 15, "ms": 3688, "input_tokens": 851, "output_tokens": 367, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "total_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM gasp_customers\nWHERE as_of_date = (\n SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(lifecycle_status)) = 'active'", "truth": 15, "ms": 2282, "input_tokens": 851, "output_tokens": 211, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1457, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1461, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1439, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "total_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT c.customer_id) AS total_customers\nFROM gasp_customers c\nWHERE c.as_of_date = (\n SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(c.lifecycle_status)) = 'active'", "truth": 15, "ms": 4673, "input_tokens": 851, "output_tokens": 436, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "total_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT c.customer_id) AS total_customers\nFROM gasp_customers c\nWHERE c.as_of_date = (\n SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n)\nAND LOWER(TRIM(c.lifecycle_status)) = 'active'", "truth": 15, "ms": 3814, "input_tokens": 851, "output_tokens": 390, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1558, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "new_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT account_ref) AS new_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'new'", "truth": 2, "ms": 1495, "input_tokens": 905, "output_tokens": 102, "cache_read": 6427, "value": 2, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_customers", "run": 1, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1493, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "total_customers", "run": 5, "sql": "WITH latest AS (\n SELECT DISTINCT ON (customer_id)\n customer_id,\n lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n AND customer_id IS NOT NULL\n ORDER BY customer_id, as_of_date DESC\n)\nSELECT COUNT(DISTINCT customer_id) AS total_customers\nFROM latest\nWHERE LOWER(TRIM(lifecycle_status)) = 'active'", "truth": 15, "ms": 3860, "input_tokens": 851, "output_tokens": 390, "cache_read": 6427, "value": 15, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_customers", "run": 2, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1395, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_customers", "run": 3, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1441, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_customers", "run": 5, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1365, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "churned_customers", "run": 4, "sql": "SELECT COUNT(DISTINCT account_ref) AS churned_customers\nFROM gasp_revenue_events\nWHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'", "truth": 0, "ms": 1551, "input_tokens": 891, "output_tokens": 104, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nrr", "run": 2, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS con,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS chu\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST(s.v + e.exp - e.con - e.chu AS DOUBLE PRECISION) / NULLIF(s.v, 0) AS nrr\nFROM start_mrr s CROSS JOIN ev e", "truth": 0.9819277108433735, "ms": 7483, "input_tokens": 3200, "output_tokens": 868, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nrr", "run": 4, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS m\n FROM subs\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS exp,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS con,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS chu\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST((s.m + e.exp - e.con - e.chu) AS DOUBLE PRECISION) / NULLIF(s.m, 0) AS nrr\nFROM start_mrr s, ev e", "truth": 0.9819277108433735, "ms": 7678, "input_tokens": 3200, "output_tokens": 919, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nrr", "run": 1, "sql": "WITH cust_snap AS (\n SELECT MAX(as_of_date) AS d FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01'\n),\ncohort AS (\n SELECT DISTINCT c.customer_id\n FROM gasp_customers c, cust_snap s\n WHERE c.as_of_date = s.d AND c.lifecycle_status = 'active'\n),\nsub_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01'\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(sb.mrr), 0) AS v\n FROM gasp_subscriptions sb, sub_snap s\n WHERE sb.snapshot_day = s.d\n AND sb.status = 'active'\n AND sb.customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'expansion'), 0) AS expansion,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST(sm.v + ev.expansion - ev.contraction - ev.churned AS DOUBLE PRECISION)\n / NULLIF(sm.v, 0) AS nrr\nFROM start_mrr sm, ev", "truth": 0.9819277108433735, "ms": 9143, "input_tokens": 3200, "output_tokens": 1017, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nrr", "run": 3, "sql": "WITH cust_snap AS (\n SELECT MAX(as_of_date) AS d FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01'\n),\nsub_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01'\n),\ncohort AS (\n SELECT DISTINCT c.customer_id\n FROM gasp_customers c, cust_snap cs\n WHERE c.as_of_date = cs.d AND c.lifecycle_status = 'active'\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s, sub_snap ss\n WHERE s.snapshot_day = ss.d\n AND s.status = 'active'\n AND s.customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(CASE WHEN event_type = 'expansion' THEN mrr_change END), 0) AS expansion,\n -COALESCE(SUM(CASE WHEN event_type = 'contraction' THEN mrr_change END), 0) AS contraction,\n -COALESCE(SUM(CASE WHEN event_type = 'churn' THEN mrr_change END), 0) AS churned\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type IN ('expansion', 'contraction', 'churn')\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST(sm.v + ev.expansion - ev.contraction - ev.churned AS DOUBLE PRECISION)\n / NULLIF(sm.v, 0) AS nrr\nFROM start_mrr sm, ev", "truth": 0.9819277108433735, "ms": 8444, "input_tokens": 3200, "output_tokens": 985, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nrr", "run": 5, "sql": "WITH cust_snap AS (\n SELECT MAX(as_of_date) AS d FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01'\n),\nsub_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01'\n),\ncohort AS (\n SELECT DISTINCT c.customer_id\n FROM gasp_customers c, cust_snap cs\n WHERE c.as_of_date = cs.d AND c.lifecycle_status = 'active'\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s, sub_snap ss\n WHERE s.snapshot_day = ss.d\n AND s.status = 'active'\n AND s.customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(CASE WHEN event_type = 'expansion' THEN mrr_change END), 0) AS expansion,\n -COALESCE(SUM(CASE WHEN event_type = 'contraction' THEN mrr_change END), 0) AS contraction,\n -COALESCE(SUM(CASE WHEN event_type = 'churn' THEN mrr_change END), 0) AS churned\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type IN ('expansion', 'contraction', 'churn')\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST(sm.v + ev.expansion - ev.contraction - ev.churned AS DOUBLE PRECISION)\n / NULLIF(sm.v, 0) AS nrr\nFROM start_mrr sm, ev", "truth": 0.9819277108433735, "ms": 8763, "input_tokens": 3200, "output_tokens": 1013, "cache_read": 6427, "value": 0.9819277108433735, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "grr", "run": 1, "sql": "WITH cust_snap AS (\n SELECT MAX(as_of_date) AS d FROM gasp_customers WHERE as_of_date <= DATE '2026-03-01'\n),\ncohort AS (\n SELECT DISTINCT c.customer_id\n FROM gasp_customers c, cust_snap s\n WHERE c.as_of_date = s.d\n AND c.lifecycle_status = 'active'\n),\nsub_snap AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01'\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(sb.mrr), 0) AS m\n FROM gasp_subscriptions sb, sub_snap s\n WHERE sb.snapshot_day = s.d\n AND sb.status = 'active'\n AND sb.customer_id IN (SELECT customer_id FROM cohort)\n),\nev AS (\n SELECT\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction_change,\n COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churn_change\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type IN ('contraction', 'churn')\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT\n CAST(start_mrr.m - (-ev.contraction_change) - (-ev.churn_change) AS DOUBLE PRECISION)\n / NULLIF(start_mrr.m, 0) AS grr\nFROM start_mrr, ev", "truth": 0.9759036144578314, "ms": 11810, "input_tokens": 2324, "output_tokens": 1492, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 1, "sql": "WITH churned AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\nlatest_cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ntotal AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM latest_cust\n WHERE lifecycle_status = 'active'\n)\nSELECT CAST(churned.n AS NUMERIC) / NULLIF(total.n, 0) AS logo_churn_rate\nFROM churned, total", "truth": 0, "ms": 4539, "input_tokens": 1307, "output_tokens": 505, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "grr", "run": 2, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST((s.v - c.v - ch.v) AS DOUBLE PRECISION) / NULLIF(s.v, 0) AS grr\nFROM start_mrr s, contraction c, churn ch", "truth": 0.9759036144578314, "ms": 7177, "input_tokens": 2324, "output_tokens": 925, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "grr", "run": 3, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (s.v - c.v - ch.v)::numeric / NULLIF(s.v, 0) AS grr\nFROM start_mrr s, contraction c, churn ch", "truth": 0.9759036144578314, "ms": 7376, "input_tokens": 2324, "output_tokens": 929, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "grr", "run": 4, "sql": "WITH sub_snap AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust_snap AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust_snap WHERE lifecycle_status = 'active'\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM sub_snap\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT (s.v - c.v - ch.v)::numeric / NULLIF(s.v, 0) AS grr\nFROM start_mrr s, contraction c, churn ch", "truth": 0.9759036144578314, "ms": 8224, "input_tokens": 2324, "output_tokens": 933, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "grr", "run": 5, "sql": "WITH cust AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ncohort AS (\n SELECT customer_id FROM cust WHERE lifecycle_status = 'active'\n),\nsubs AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, customer_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nstart_mrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM subs\n WHERE status = 'active'\n AND customer_id IN (SELECT customer_id FROM cohort)\n),\ncontraction AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction'\n AND account_ref IN (SELECT customer_id FROM cohort)\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n AND account_ref IN (SELECT customer_id FROM cohort)\n)\nSELECT CAST((s.v - c.v - ch.v) AS DOUBLE PRECISION) / NULLIF(s.v, 0) AS grr\nFROM start_mrr s, contraction c, churn ch", "truth": 0.9759036144578314, "ms": 8399, "input_tokens": 2324, "output_tokens": 1021, "cache_read": 6427, "value": 0.9759036144578314, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 2, "sql": "WITH snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n),\nchurned AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND LOWER(TRIM(event_type)) = 'churn'\n),\ntotal AS (\n SELECT COUNT(DISTINCT c.customer_id) AS n\n FROM gasp_customers c\n JOIN snap s ON c.as_of_date = s.d\n WHERE LOWER(TRIM(c.lifecycle_status)) = 'active'\n)\nSELECT CAST(churned.n AS NUMERIC) / NULLIF(total.n, 0) AS logo_churn_rate\nFROM churned, total", "truth": 0, "ms": 5671, "input_tokens": 1307, "output_tokens": 676, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 3, "sql": "WITH churned AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn'\n),\nlatest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ntotal AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM latest\n WHERE lifecycle_status = 'active'\n)\nSELECT CAST(churned.n AS NUMERIC) / NULLIF(total.n, 0) AS logo_churn_rate\nFROM churned, total", "truth": 0, "ms": 4969, "input_tokens": 1307, "output_tokens": 562, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 4, "sql": "WITH snap AS (\n SELECT MAX(as_of_date) AS d\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n),\nchurned AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND LOWER(TRIM(event_type)) = 'churn'\n),\nbase AS (\n SELECT COUNT(DISTINCT c.customer_id) AS n\n FROM gasp_customers c\n JOIN snap s ON c.as_of_date = s.d\n WHERE LOWER(TRIM(c.lifecycle_status)) = 'active'\n)\nSELECT CAST(churned.n AS NUMERIC) / NULLIF(base.n, 0) AS logo_churn_rate\nFROM churned, base", "truth": 0, "ms": 6568, "input_tokens": 1307, "output_tokens": 704, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "logo_churn_rate", "run": 5, "sql": "WITH churned AS (\n SELECT COUNT(DISTINCT account_ref) AS n\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n AND LOWER(TRIM(event_type)) = 'churn'\n),\nlatest AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-01'\n ORDER BY customer_id, as_of_date DESC\n),\ntotal AS (\n SELECT COUNT(DISTINCT customer_id) AS n\n FROM latest\n WHERE LOWER(TRIM(lifecycle_status)) = 'active'\n)\nSELECT CAST(churned.n AS DOUBLE PRECISION) / NULLIF(total.n, 0) AS logo_churn_rate\nFROM churned, total", "truth": 0, "ms": 5870, "input_tokens": 1307, "output_tokens": 667, "cache_read": 6427, "value": 0, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 1, "sql": "WITH latest_subs AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id,\n mrr,\n status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nmrr_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM latest_subs\n WHERE status = 'active'\n),\nlosses AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT\n (l.churned + l.contraction)::numeric / NULLIF(m.mrr, 0) AS revenue_churn_rate\nFROM losses l\nCROSS JOIN mrr_start m", "truth": 0.024096385542168676, "ms": 4620, "input_tokens": 1501, "output_tokens": 562, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 2, "sql": "WITH sub_at AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\ndenom AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM sub_at\n WHERE status = 'active'\n),\nnum AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n - COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS lost\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT num.lost / NULLIF(denom.mrr, 0) AS revenue_churn_rate\nFROM num, denom", "truth": 0.024096385542168676, "ms": 5142, "input_tokens": 1501, "output_tokens": 569, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nps", "run": 1, "sql": "SELECT ((CAST(((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score >= 9) - (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score <= 6)) AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer'), 0)) * 100) AS nps", "truth": -25, "ms": 2670, "input_tokens": 1086, "output_tokens": 312, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nps", "run": 2, "sql": "SELECT (CAST(\n (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score >= 9)\n - (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score <= 6)\n AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer'), 0)) * 100 AS nps", "truth": -25, "ms": 2668, "input_tokens": 1086, "output_tokens": 318, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nps", "run": 3, "sql": "SELECT ((CAST(((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score >= 9) - (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score <= 6)) AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer'), 0)) * 100) AS nps", "truth": -25, "ms": 2619, "input_tokens": 1086, "output_tokens": 312, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 4, "sql": "WITH snap AS (\n SELECT MAX(snapshot_day) AS d\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n),\nmrr_start AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS mrr\n FROM gasp_subscriptions s\n JOIN snap ON s.snapshot_day = snap.d\n WHERE s.status = 'active'\n),\nlost AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n - COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS lost_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT CAST(lost.lost_mrr AS NUMERIC) / NULLIF(mrr_start.mrr, 0) AS revenue_churn_rate\nFROM lost, mrr_start", "truth": 0.024096385542168676, "ms": 5896, "input_tokens": 1501, "output_tokens": 699, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 3, "sql": "WITH sub_at_start AS (\n SELECT DISTINCT ON (subscription_id)\n subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\nmrr_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS mrr\n FROM sub_at_start\n WHERE status = 'active'\n),\nlosses AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned_mrr,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction_mrr\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01'\n AND booked_on <= DATE '2026-03-31'\n)\nSELECT (l.churned_mrr + l.contraction_mrr)::numeric / NULLIF(m.mrr, 0) AS revenue_churn_rate\nFROM losses l\nCROSS JOIN mrr_start m", "truth": 0.024096385542168676, "ms": 6612, "input_tokens": 1501, "output_tokens": 721, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "revenue_churn_rate", "run": 5, "sql": "SELECT (\n (\n -(SELECT COALESCE(SUM(mrr_change), 0)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn')\n -(SELECT COALESCE(SUM(mrr_change), 0)\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction')\n )::numeric\n / NULLIF((\n SELECT COALESCE(SUM(mrr), 0)\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n ), 0)\n) AS revenue_churn_rate", "truth": 0.024096385542168676, "ms": 4662, "input_tokens": 1501, "output_tokens": 522, "cache_read": 6427, "value": 0.024096385542168676, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nps", "run": 5, "sql": "SELECT ((CAST(((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score >= 9) - (SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer' AND nps_score <= 6)) AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31' AND survey_type = 'customer'), 0)) * 100) AS nps", "truth": -25, "ms": 2558, "input_tokens": 1086, "output_tokens": 312, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac", "run": 1, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)) AS cac", "truth": 3000, "ms": 1833, "input_tokens": 1618, "output_tokens": 215, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac", "run": 2, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)) AS cac", "truth": 3000, "ms": 1837, "input_tokens": 1618, "output_tokens": 210, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "nps", "run": 4, "sql": "SELECT (CAST(\n (SELECT COUNT(nps_score) FROM gasp_survey_responses\n WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31'\n AND survey_type = 'customer' AND nps_score >= 9)\n -\n (SELECT COUNT(nps_score) FROM gasp_survey_responses\n WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31'\n AND survey_type = 'customer' AND nps_score <= 6)\n AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(nps_score) FROM gasp_survey_responses\n WHERE survey_date >= DATE '2026-03-01' AND survey_date <= DATE '2026-03-31'\n AND survey_type = 'customer'), 0)) * 100 AS nps", "truth": -25, "ms": 2831, "input_tokens": 1086, "output_tokens": 332, "cache_read": 6427, "value": -25, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac", "run": 3, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)) AS cac", "truth": 3000, "ms": 1881, "input_tokens": 1618, "output_tokens": 210, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac", "run": 4, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)) AS cac", "truth": 3000, "ms": 1978, "input_tokens": 1618, "output_tokens": 215, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac", "run": 5, "sql": "SELECT (CAST((SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31' AND cost_function = 'gtm') AS DOUBLE PRECISION) / NULLIF((SELECT COUNT(DISTINCT account_ref) FROM gasp_revenue_events WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31' AND event_type = 'new'), 0)) AS cac", "truth": 3000, "ms": 3882, "input_tokens": 1618, "output_tokens": 215, "cache_read": 6427, "value": 3000, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv", "run": 5, "sql": "WITH sub_end AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31'\n),\nsub_start AS (\n SELECT MAX(snapshot_day) AS d FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01'\n),\ncust_end AS (\n SELECT MAX(as_of_date) AS d FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31'\n),\nmrr_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT d FROM sub_end) AND status = 'active'\n),\nmrr_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT d FROM sub_start) AND status = 'active'\n),\ncustomers AS (\n SELECT COUNT(DISTINCT customer_id) AS v FROM gasp_customers\n WHERE as_of_date = (SELECT d FROM cust_end) AND lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(revenue), 0) AS v FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ninfra AS (\n SELECT COALESCE(SUM(cost), 0) AS v FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT\n (CAST((SELECT v FROM mrr_end) AS DOUBLE PRECISION) / NULLIF((SELECT v FROM customers), 0))\n * (CAST((SELECT v FROM rev) - (SELECT v FROM infra) AS DOUBLE PRECISION) / NULLIF((SELECT v FROM rev), 0))\n / NULLIF(CAST((SELECT v FROM churn) AS DOUBLE PRECISION) / NULLIF((SELECT v FROM mrr_start), 0), 0)\n AS ltv", "truth": 47870.948012232424, "ms": 9821, "input_tokens": 2945, "output_tokens": 1236, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv", "run": 1, "sql": "WITH sub_end AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31'\n ORDER BY subscription_id, snapshot_day DESC\n),\nsub_start AS (\n SELECT DISTINCT ON (subscription_id) subscription_id, mrr, status\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-01'\n ORDER BY subscription_id, snapshot_day DESC\n),\ncust_end AS (\n SELECT DISTINCT ON (customer_id) customer_id, lifecycle_status\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31'\n ORDER BY customer_id, as_of_date DESC\n),\nm AS (\n SELECT\n (SELECT COALESCE(SUM(mrr), 0) FROM sub_end WHERE status = 'active')::double precision AS mrr_end,\n (SELECT COALESCE(SUM(mrr), 0) FROM sub_start WHERE status = 'active')::double precision AS mrr_start,\n (SELECT COUNT(DISTINCT customer_id) FROM cust_end WHERE lifecycle_status = 'active')::double precision AS customers,\n (SELECT COALESCE(SUM(revenue), 0) FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31')::double precision AS revenue,\n (SELECT COALESCE(SUM(cost), 0) FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure')::double precision AS infra_cost,\n (SELECT -COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'churn')::double precision AS churned,\n (SELECT -COALESCE(SUM(mrr_change), 0) FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n AND event_type = 'contraction')::double precision AS contraction\n)\nSELECT\n (mrr_end / NULLIF(customers, 0))\n * ((revenue - infra_cost) / NULLIF(revenue, 0))\n / NULLIF((churned + contraction) / NULLIF(mrr_start, 0), 0) AS ltv\nFROM m", "truth": 47870.948012232424, "ms": 11467, "input_tokens": 2945, "output_tokens": 1262, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv", "run": 3, "sql": "WITH params AS (\n SELECT DATE '2026-03-01' AS ps, DATE '2026-03-31' AS pe\n),\nsub_end AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s, params p\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions, params p2 WHERE snapshot_day <= p2.pe)\n),\nsub_start AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions, params p2 WHERE snapshot_day <= p2.ps)\n),\ncust AS (\n SELECT COUNT(DISTINCT c.customer_id) AS v\n FROM gasp_customers c\n WHERE c.lifecycle_status = 'active'\n AND c.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers, params p2 WHERE as_of_date <= p2.pe)\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0) AS v\n FROM gasp_financials f, params p\n WHERE f.period_start >= p.ps AND f.period_start <= p.pe\n),\ncost AS (\n SELECT COALESCE(SUM(ce.cost), 0) AS v\n FROM gasp_cost_events ce, params p\n WHERE ce.incurred_on >= p.ps AND ce.incurred_on <= p.pe\n AND ce.cost_function = 'infrastructure'\n),\nchurn AS (\n SELECT -COALESCE(SUM(CASE WHEN r.event_type = 'churn' THEN r.mrr_change END), 0)\n -COALESCE(SUM(CASE WHEN r.event_type = 'contraction' THEN r.mrr_change END), 0) AS v\n FROM gasp_revenue_events r, params p\n WHERE r.booked_on >= p.ps AND r.booked_on <= p.pe\n AND r.event_type IN ('churn', 'contraction')\n)\nSELECT\n (CAST(sub_end.v AS DOUBLE PRECISION) / NULLIF(cust.v, 0))\n * (CAST(rev.v - cost.v AS DOUBLE PRECISION) / NULLIF(rev.v, 0))\n / NULLIF(CAST(churn.v AS DOUBLE PRECISION) / NULLIF(sub_start.v, 0), 0) AS ltv\nFROM sub_end, sub_start, cust, rev, cost, churn", "truth": 47870.948012232424, "ms": 11775, "input_tokens": 2945, "output_tokens": 1291, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv", "run": 2, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nrev AS (\n SELECT COALESCE(SUM(revenue), 0) AS v\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ncost AS (\n SELECT COALESCE(SUM(cost), 0) AS v\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure'\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT\n ( (sub_end.v::double precision / NULLIF(cust.v, 0))\n * ((rev.v - cost.v)::double precision / NULLIF(rev.v, 0)) )\n / NULLIF(churn.v::double precision / NULLIF(sub_start.v, 0), 0) AS ltv\nFROM sub_end, sub_start, cust, rev, cost, churn", "truth": 47870.948012232424, "ms": 11979, "input_tokens": 2945, "output_tokens": 1222, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv", "run": 4, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nrev AS (\n SELECT COALESCE(SUM(revenue), 0) AS v\n FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\ncost AS (\n SELECT COALESCE(SUM(cost), 0) AS v\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND cost_function = 'infrastructure'\n),\nchurn AS (\n SELECT -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0)\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS v\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT\n (CAST(sub_end.v AS DOUBLE PRECISION) / NULLIF(cust.v, 0))\n * (CAST(rev.v - cost.v AS DOUBLE PRECISION) / NULLIF(rev.v, 0))\n / NULLIF(CAST(churn.v AS DOUBLE PRECISION) / NULLIF(sub_start.v, 0), 0) AS ltv\nFROM sub_end, sub_start, cust, rev, cost, churn", "truth": 47870.948012232424, "ms": 12169, "input_tokens": 2945, "output_tokens": 1275, "cache_read": 6427, "value": 47870.948012232424, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 1, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust_end AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nfin AS (\n SELECT COALESCE(SUM(revenue), 0) AS v\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ncosts AS (\n SELECT\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'infrastructure'), 0) AS infra,\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'gtm'), 0) AS gtm\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n),\nrev AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COUNT(DISTINCT account_ref) FILTER (WHERE event_type = 'new') AS new_customers\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n),\nm AS (\n SELECT\n sub_end.v::numeric / NULLIF(cust_end.v, 0) AS arpa,\n (fin.v - costs.infra)::numeric / NULLIF(fin.v, 0) AS gross_margin,\n (rev.churned + rev.contraction)::numeric / NULLIF(sub_start.v, 0) AS churn_rate,\n costs.gtm::numeric / NULLIF(rev.new_customers, 0) AS cac\n FROM sub_end, sub_start, cust_end, fin, costs, rev\n)\nSELECT ((arpa * gross_margin) / NULLIF(churn_rate, 0)) / NULLIF(cac, 0) AS ltv_cac_ratio\nFROM m", "truth": 15.956982670744141, "ms": 10504, "input_tokens": 3209, "output_tokens": 1333, "cache_read": 6427, "value": 15.95698267074414, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac_payback", "run": 1, "sql": "WITH params AS (\n SELECT DATE '2026-03-01' AS ps, DATE '2026-03-31' AS pe\n),\ncac AS (\n SELECT\n CAST((SELECT COALESCE(SUM(c.cost), 0)\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'gtm') AS DOUBLE PRECISION)\n / NULLIF((SELECT COUNT(DISTINCT r.account_ref)\n FROM gasp_revenue_events r, params p\n WHERE r.booked_on BETWEEN p.ps AND p.pe\n AND r.event_type = 'new'), 0) AS v\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n WHERE s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions, params p WHERE snapshot_day <= p.pe)\n AND s.status = 'active'\n),\ntc AS (\n SELECT COUNT(DISTINCT cu.customer_id) AS v\n FROM gasp_customers cu\n WHERE cu.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers, params p WHERE as_of_date <= p.pe)\n AND cu.lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0) AS v\n FROM gasp_financials f, params p\n WHERE f.period_start BETWEEN p.ps AND p.pe\n),\ninfra AS (\n SELECT COALESCE(SUM(c.cost), 0) AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'infrastructure'\n)\nSELECT cac.v / NULLIF(\n (CAST(mrr.v AS DOUBLE PRECISION) / NULLIF(tc.v, 0))\n * (CAST(rev.v - infra.v AS DOUBLE PRECISION) / NULLIF(rev.v, 0)),\n 0) AS cac_payback\nFROM cac, mrr, tc, rev, infra", "truth": 2.600742311770943, "ms": 8471, "input_tokens": 2676, "output_tokens": 1027, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 5, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust_end AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nfin AS (\n SELECT COALESCE(SUM(revenue), 0) AS rev\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\ncosts AS (\n SELECT\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'infrastructure'), 0) AS infra,\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'gtm'), 0) AS gtm\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n),\nrev AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COUNT(DISTINCT account_ref) FILTER (WHERE event_type = 'new') AS new_cust\n FROM gasp_revenue_events\n WHERE booked_on >= DATE '2026-03-01' AND booked_on <= DATE '2026-03-31'\n)\nSELECT\n (\n (sub_end.v::double precision / NULLIF(cust_end.v, 0))\n * ((fin.rev - costs.infra)::double precision / NULLIF(fin.rev, 0))\n / NULLIF((rev.churned + rev.contraction)::double precision / NULLIF(sub_start.v, 0), 0)\n )\n / NULLIF(costs.gtm::double precision / NULLIF(rev.new_cust, 0), 0) AS ltv_cac_ratio\nFROM sub_end, sub_start, cust_end, fin, costs, rev", "truth": 15.956982670744141, "ms": 9576, "input_tokens": 3209, "output_tokens": 1241, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 4, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id) AS v FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nfin AS (\n SELECT COALESCE(SUM(revenue), 0) AS v FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\ncosts AS (\n SELECT\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'infrastructure'), 0) AS infra,\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'gtm'), 0) AS gtm\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nrev AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COUNT(DISTINCT account_ref) FILTER (WHERE event_type = 'new') AS new_cust\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT (\n (sub_end.v::numeric / NULLIF(cust.v, 0))\n * ((fin.v - costs.infra)::numeric / NULLIF(fin.v, 0))\n / NULLIF((rev.churned + rev.contraction)::numeric / NULLIF(sub_start.v, 0), 0)\n) / NULLIF(costs.gtm::numeric / NULLIF(rev.new_cust, 0), 0) AS ltv_cac_ratio\nFROM sub_end, sub_start, cust, fin, costs, rev", "truth": 15.956982670744141, "ms": 10556, "input_tokens": 3209, "output_tokens": 1280, "cache_read": 6427, "value": 15.95698267074414, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac_payback", "run": 2, "sql": "WITH params AS (\n SELECT DATE '2026-03-01' AS ps, DATE '2026-03-31' AS pe\n),\ncac AS (\n SELECT\n (SELECT COALESCE(SUM(c.cost), 0)\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'gtm')::double precision\n / NULLIF((SELECT COUNT(DISTINCT r.account_ref)\n FROM gasp_revenue_events r, params p\n WHERE r.booked_on BETWEEN p.ps AND p.pe\n AND r.event_type = 'new'), 0) AS v\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0)::double precision AS v\n FROM gasp_subscriptions s\n WHERE s.snapshot_day = (SELECT MAX(snapshot_day)\n FROM gasp_subscriptions, params p\n WHERE snapshot_day <= p.pe)\n AND s.status = 'active'\n),\ntc AS (\n SELECT COUNT(DISTINCT cu.customer_id) AS v\n FROM gasp_customers cu\n WHERE cu.as_of_date = (SELECT MAX(as_of_date)\n FROM gasp_customers, params p\n WHERE as_of_date <= p.pe)\n AND cu.lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0)::double precision AS v\n FROM gasp_financials f, params p\n WHERE f.period_start BETWEEN p.ps AND p.pe\n),\ninfra AS (\n SELECT COALESCE(SUM(c.cost), 0)::double precision AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'infrastructure'\n)\nSELECT cac.v / NULLIF(\n (mrr.v / NULLIF(tc.v, 0)) * ((rev.v - infra.v) / NULLIF(rev.v, 0)),\n 0) AS cac_payback\nFROM cac, mrr, tc, rev, infra", "truth": 2.600742311770943, "ms": 9534, "input_tokens": 2676, "output_tokens": 1101, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 3, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active'\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n AND status = 'active'\n),\ncust_end AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n AND lifecycle_status = 'active'\n),\nfin AS (\n SELECT COALESCE(SUM(revenue), 0) AS v\n FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\ninfra AS (\n SELECT COALESCE(SUM(cost), 0) AS v\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND cost_function = 'infrastructure'\n),\ngtm AS (\n SELECT COALESCE(SUM(cost), 0) AS v\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND cost_function = 'gtm'\n),\nrev AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COUNT(DISTINCT account_ref) FILTER (WHERE event_type = 'new') AS new_cust\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT\n (\n (sub_end.v::double precision / NULLIF(cust_end.v, 0))\n * ((fin.v - infra.v)::double precision / NULLIF(fin.v, 0))\n / NULLIF((rev.churned + rev.contraction)::double precision / NULLIF(sub_start.v, 0), 0)\n )\n / NULLIF(gtm.v::double precision / NULLIF(rev.new_cust, 0), 0) AS ltv_cac_ratio\nFROM sub_end, sub_start, cust_end, fin, infra, gtm, rev", "truth": 15.956982670744141, "ms": 14247, "input_tokens": 3209, "output_tokens": 1439, "cache_read": 6427, "value": 15.956982670744141, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "ltv_cac_ratio", "run": 2, "sql": "WITH\nsub_end AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-31')\n),\nsub_start AS (\n SELECT COALESCE(SUM(mrr), 0) AS v FROM gasp_subscriptions\n WHERE status = 'active'\n AND snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions WHERE snapshot_day <= DATE '2026-03-01')\n),\ncust AS (\n SELECT COUNT(DISTINCT customer_id) AS v FROM gasp_customers\n WHERE lifecycle_status = 'active'\n AND as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers WHERE as_of_date <= DATE '2026-03-31')\n),\nfin AS (\n SELECT COALESCE(SUM(revenue), 0) AS v FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\ncosts AS (\n SELECT\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'infrastructure'), 0) AS infra,\n COALESCE(SUM(cost) FILTER (WHERE cost_function = 'gtm'), 0) AS gtm\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\nrev AS (\n SELECT\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'churn'), 0) AS churned,\n -COALESCE(SUM(mrr_change) FILTER (WHERE event_type = 'contraction'), 0) AS contraction,\n COUNT(DISTINCT account_ref) FILTER (WHERE event_type = 'new') AS new_cust\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n)\nSELECT\n (\n (sub_end.v::numeric / NULLIF(cust.v, 0))\n * ((fin.v - costs.infra) / NULLIF(fin.v, 0))\n / NULLIF((rev.churned + rev.contraction) / NULLIF(sub_start.v, 0), 0)\n )\n / NULLIF(costs.gtm / NULLIF(rev.new_cust, 0), 0) AS ltv_cac_ratio\nFROM sub_end, sub_start, cust, fin, costs, rev", "truth": 15.956982670744141, "ms": 15655, "input_tokens": 3209, "output_tokens": 1301, "cache_read": 6427, "value": 15.95698267074414, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac_payback", "run": 3, "sql": "WITH\ncac AS (\n SELECT\n (SELECT COALESCE(SUM(cost), 0)::numeric\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND cost_function = 'gtm')\n / NULLIF((SELECT COUNT(DISTINCT account_ref)\n FROM gasp_revenue_events\n WHERE booked_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND event_type = 'new'), 0) AS v\n),\nmrr AS (\n SELECT COALESCE(SUM(mrr), 0)::numeric AS v\n FROM gasp_subscriptions\n WHERE snapshot_day = (SELECT MAX(snapshot_day)\n FROM gasp_subscriptions\n WHERE snapshot_day <= DATE '2026-03-31')\n AND status = 'active'\n),\ntc AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM gasp_customers\n WHERE as_of_date = (SELECT MAX(as_of_date)\n FROM gasp_customers\n WHERE as_of_date <= DATE '2026-03-31')\n AND lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(revenue), 0)::numeric AS v\n FROM gasp_financials\n WHERE period_start BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n),\ninfra AS (\n SELECT COALESCE(SUM(cost), 0)::numeric AS v\n FROM gasp_cost_events\n WHERE incurred_on BETWEEN DATE '2026-03-01' AND DATE '2026-03-31'\n AND cost_function = 'infrastructure'\n)\nSELECT cac.v / NULLIF(\n (mrr.v / NULLIF(tc.v, 0)) * ((rev.v - infra.v) / NULLIF(rev.v, 0)),\n 0) AS cac_payback\nFROM cac, mrr, tc, rev, infra", "truth": 2.600742311770943, "ms": 8448, "input_tokens": 2676, "output_tokens": 1042, "cache_read": 6427, "value": 2.6007423117709436, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "gross_margin", "run": 1, "sql": "WITH fin AS (\n SELECT revenue, revenue_type\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\nhas_gaap AS (\n SELECT EXISTS (\n SELECT 1 FROM fin\n WHERE lower(revenue_type) LIKE '%recogn%' OR lower(revenue_type) LIKE '%gaap%'\n ) AS flag\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0) AS revenue\n FROM fin f, has_gaap h\n WHERE (NOT h.flag)\n OR lower(f.revenue_type) LIKE '%recogn%'\n OR lower(f.revenue_type) LIKE '%gaap%'\n),\ncogs AS (\n SELECT COALESCE(SUM(cost), 0) AS cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND lower(trim(cost_function)) = 'infrastructure'\n)\nSELECT (rev.revenue - cogs.cost) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 7386, "input_tokens": 1526, "output_tokens": 822, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac_payback", "run": 4, "sql": "WITH params AS (\n SELECT DATE '2026-03-01' AS ps, DATE '2026-03-31' AS pe\n),\ngtm AS (\n SELECT COALESCE(SUM(c.cost), 0) AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe AND c.cost_function = 'gtm'\n),\ninfra AS (\n SELECT COALESCE(SUM(c.cost), 0) AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe AND c.cost_function = 'infrastructure'\n),\nnewc AS (\n SELECT COUNT(DISTINCT r.account_ref) AS v\n FROM gasp_revenue_events r, params p\n WHERE r.booked_on BETWEEN p.ps AND p.pe AND r.event_type = 'new'\n),\nmrr AS (\n SELECT COALESCE(SUM(s.mrr), 0) AS v\n FROM gasp_subscriptions s\n WHERE s.status = 'active'\n AND s.snapshot_day = (SELECT MAX(snapshot_day) FROM gasp_subscriptions, params p WHERE snapshot_day <= p.pe)\n),\ncusts AS (\n SELECT COUNT(DISTINCT cu.customer_id) AS v\n FROM gasp_customers cu\n WHERE cu.lifecycle_status = 'active'\n AND cu.as_of_date = (SELECT MAX(as_of_date) FROM gasp_customers, params p WHERE as_of_date <= p.pe)\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0) AS v\n FROM gasp_financials f, params p\n WHERE f.period_start BETWEEN p.ps AND p.pe\n)\nSELECT\n (gtm.v::double precision / NULLIF(newc.v, 0))\n / NULLIF(\n (mrr.v::double precision / NULLIF(custs.v, 0))\n * ((rev.v - infra.v)::double precision / NULLIF(rev.v, 0)),\n 0) AS cac_payback\nFROM gtm, infra, newc, mrr, custs, rev", "truth": 2.600742311770943, "ms": 10596, "input_tokens": 2676, "output_tokens": 1119, "cache_read": 6427, "value": 2.600742311770943, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "gross_margin", "run": 2, "sql": "WITH fin AS (\n SELECT revenue, revenue_type\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\nrev AS (\n SELECT CASE\n WHEN EXISTS (SELECT 1 FROM fin WHERE lower(revenue_type) LIKE '%recogn%' OR lower(revenue_type) LIKE '%gaap%')\n THEN (SELECT COALESCE(SUM(revenue), 0) FROM fin WHERE lower(revenue_type) LIKE '%recogn%' OR lower(revenue_type) LIKE '%gaap%')\n ELSE (SELECT COALESCE(SUM(revenue), 0) FROM fin)\n END AS revenue\n),\ncogs AS (\n SELECT COALESCE(SUM(cost), 0) AS cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND lower(trim(cost_function)) = 'infrastructure'\n)\nSELECT (rev.revenue - cogs.cost) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 6661, "input_tokens": 1526, "output_tokens": 757, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "cac_payback", "run": 5, "sql": "WITH params AS (\n SELECT DATE '2026-03-01' AS ps, DATE '2026-03-31' AS pe\n),\ngtm_cost AS (\n SELECT COALESCE(SUM(c.cost), 0) AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'gtm'\n),\ninfra_cost AS (\n SELECT COALESCE(SUM(c.cost), 0) AS v\n FROM gasp_cost_events c, params p\n WHERE c.incurred_on BETWEEN p.ps AND p.pe\n AND c.cost_function = 'infrastructure'\n),\nnew_cust AS (\n SELECT COUNT(DISTINCT r.account_ref) AS v\n FROM gasp_revenue_events r, params p\n WHERE r.booked_on BETWEEN p.ps AND p.pe\n AND r.event_type = 'new'\n),\nsub_latest AS (\n SELECT DISTINCT ON (s.subscription_id) s.subscription_id, s.mrr, s.status\n FROM gasp_subscriptions s, params p\n WHERE s.snapshot_day <= p.pe\n ORDER BY s.subscription_id, s.snapshot_day DESC\n),\nmrr AS (\n SELECT COALESCE(SUM(mrr), 0) AS v\n FROM sub_latest\n WHERE status = 'active'\n),\ncust_latest AS (\n SELECT DISTINCT ON (c.customer_id) c.customer_id, c.lifecycle_status\n FROM gasp_customers c, params p\n WHERE c.as_of_date <= p.pe\n ORDER BY c.customer_id, c.as_of_date DESC\n),\ntotal_cust AS (\n SELECT COUNT(DISTINCT customer_id) AS v\n FROM cust_latest\n WHERE lifecycle_status = 'active'\n),\nrev AS (\n SELECT COALESCE(SUM(f.revenue), 0) AS v\n FROM gasp_financials f, params p\n WHERE f.period_start BETWEEN p.ps AND p.pe\n)\nSELECT\n (gtm_cost.v::numeric / NULLIF(new_cust.v, 0))\n / NULLIF(\n (mrr.v::numeric / NULLIF(total_cust.v, 0))\n * ((rev.v - infra_cost.v)::numeric / NULLIF(rev.v, 0)),\n 0\n ) AS cac_payback\nFROM gtm_cost, infra_cost, new_cust, mrr, total_cust, rev", "truth": 2.600742311770943, "ms": 10098, "input_tokens": 2676, "output_tokens": 1246, "cache_read": 6427, "value": 2.6007423117709436, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "gross_margin", "run": 3, "sql": "WITH fin AS (\n SELECT revenue, revenue_type\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\nrev AS (\n SELECT CASE\n WHEN EXISTS (SELECT 1 FROM fin WHERE revenue_type ILIKE '%recogn%' OR revenue_type ILIKE '%gaap%')\n THEN (SELECT COALESCE(SUM(revenue), 0) FROM fin WHERE revenue_type ILIKE '%recogn%' OR revenue_type ILIKE '%gaap%')\n ELSE (SELECT COALESCE(SUM(revenue), 0) FROM fin)\n END AS revenue\n),\ncogs AS (\n SELECT COALESCE(SUM(cost), 0) AS cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND LOWER(TRIM(cost_function)) = 'infrastructure'\n)\nSELECT (rev.revenue - cogs.cost)::numeric / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 7122, "input_tokens": 1526, "output_tokens": 779, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "gross_margin", "run": 4, "sql": "SELECT (\n CAST(\n (SELECT COALESCE(SUM(revenue), 0)\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start <= DATE '2026-03-31')\n -\n (SELECT COALESCE(SUM(cost), 0)\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01'\n AND incurred_on <= DATE '2026-03-31'\n AND cost_function = 'infrastructure')\n AS NUMERIC)\n / NULLIF(\n (SELECT COALESCE(SUM(revenue), 0)\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01'\n AND period_start <= DATE '2026-03-31'), 0)\n) AS gross_margin", "truth": 0.9403669724770642, "ms": 4653, "input_tokens": 1526, "output_tokens": 503, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" }, { "model": "claude-opus-5-5", "condition": "gasp", "metric": "gross_margin", "run": 5, "sql": "WITH fin AS (\n SELECT revenue, lower(trim(coalesce(revenue_type, ''))) AS rt\n FROM gasp_financials\n WHERE period_start >= DATE '2026-03-01' AND period_start <= DATE '2026-03-31'\n),\nrev AS (\n SELECT CASE\n WHEN EXISTS (SELECT 1 FROM fin WHERE rt LIKE '%recogn%' OR rt LIKE '%gaap%')\n THEN (SELECT COALESCE(SUM(revenue), 0) FROM fin WHERE rt LIKE '%recogn%' OR rt LIKE '%gaap%')\n ELSE (SELECT COALESCE(SUM(revenue), 0) FROM fin)\n END AS revenue\n),\ncogs AS (\n SELECT COALESCE(SUM(cost), 0) AS cost\n FROM gasp_cost_events\n WHERE incurred_on >= DATE '2026-03-01' AND incurred_on <= DATE '2026-03-31'\n AND lower(trim(cost_function)) = 'infrastructure'\n)\nSELECT (rev.revenue - cogs.cost) / NULLIF(rev.revenue, 0) AS gross_margin\nFROM rev, cogs", "truth": 0.9403669724770642, "ms": 6732, "input_tokens": 1526, "output_tokens": 770, "cache_read": 6427, "value": 0.9403669724770642, "category": "correct" } ] }