{ "schemaVersion": 2, "protocolId": "duck-yeah-trial-assistance-v2.1", "protocolHash": "f1d1083aefb021ef29a9c6d44bb76640cf9cc9cc50dfca65081cddb6c6ff51db", "protocolSource": "public/duck-yeah/experiment-protocol.json", "generatedAt": "2026-09-29T14:37:51.053Z", "dataKind": "synthetic", "dataKindDetail": "synthetic_business_scenario", "boundary": "Invented business assumptions and outcomes. No MotherDuck internal data, customer results, buying intent, or demonstrated treatment efficacy. Real company-enrichment fixtures and execution receipts are separate.", "execution": { "engine": "motherduck", "engineLabel": "MotherDuck", "verifiedMotherDuck": true, "motherDuckChecks": { "connectionScheme": "md:", "extensionLoaded": true, "databaseType": "motherduck", "experimentTables": [ "account_outcomes", "accounts", "ascertainment", "assignments", "contact_opportunities", "events", "load_manifest", "users" ], "queriesExecuted": 20 }, "database": "duck_yeah_demo", "schema": "duck_yeah_experiment", "duckdbVersion": "v1.5.5", "motherduckExtensionVersion": "v1.5.5-2026-09-355", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:32.551Z", "completedAt": "2026-09-29T14:37:51.051Z", "elapsedMs": 18498.86, "runner": "owner-run scripts/duck-yeah/experiment.py" }, "validation": { "path": "/duck-yeah/experiment-power.json", "generatedAt": "2026-09-29T14:29:09.540Z", "allPass": true, "replicatesPerCondition": 100000, "rng": { "requested": "numpy_pcg64", "used": "numpy_pcg64", "conformant": true, "numpyVersion": "2.5.3", "pinnedNumpyVersion": "2.5.3", "streams": "SeedSequence(seed).spawn in declared condition × dependence-model order", "pythonVersion": "3.13.2" }, "protocolHash": "f1d1083aefb021ef29a9c6d44bb76640cf9cc9cc50dfca65081cddb6c6ff51db", "prefixAllMatch": true }, "dataset": { "generatorVersion": "2026-09-29.3", "tables": { "accounts": { "rows": 33000, "sha256": "df73d4e651b913853819e9f8c1ae06d3a88a3fa9b7f4da3a1eb873593920900d" }, "assignments": { "rows": 27000, "sha256": "67dcec57e89b019971ed2b0c6d09d1c09260e99080493e0dcfe57c249f5717be" }, "users": { "rows": 37845, "sha256": "2b412e2e7e8efd1c2155cb1294df38c64c18758533513c2c917e480380a130f1" }, "events": { "rows": 184678, "sha256": "b3c24f015e2f76215deaaf441305126ed6455f54020bcdf37f07fed88bb8096f" }, "contact_opportunities": { "rows": 66000, "sha256": "7c97d4c7d8f775ce0e637cb0b19605162c6228aa432c7da4349155cad16956ef" }, "ascertainment": { "rows": 132000, "sha256": "ca148426d337388301a02424fb0dc88c527d8ff11bc283f317e4eaef8cfed21c" } }, "combinedSha256": "2f56ff95cae93bdd719f666b729c573bf0edd4bc9e9bd0996b88cdbb8d027a1a", "regenerate": ".duck-yeah-private/venv/bin/python scripts/duck-yeah/experiment.py --engine local (raw CSVs land in .duck-yeah-private/experiment/data/; byte-identical for the same protocol hash)", "publicRawRows": "not published in bulk; see experiment-samples.json for a bounded sample" }, "implementation": { "pipeline": "2026-09-29.1", "stats": "2026-09-29.1", "decision": "2026-09-29.2", "generator": "2026-09-29.3", "interval": { "method": "newcombe", "level": 0.95, "z": 1.9599639845, "continuityCorrection": false } }, "baseline": { "id": "synthetic-history-v2.0", "dataKind": "synthetic_historical_baseline", "seed": 2026092901, "period": { "startAt": "2026-01-01T00:00:00Z", "enrollmentWeeks": 12, "completeFollowupDays": 28, "firstAssignedAt": null, "note": "synthetic calendar" }, "assumptions": { "rates": { "sqa28": 0.1, "activation7": 0.25, "optout28": 0.01, "sharing7": 0.12 }, "salesMinutesPerAccount": 0.6, "accountsPerWeek": 500, "eligibleAccounts": 6000, "note": "Generator assumptions. The cohort/segment figures below are generated observations around them, not the assumptions restated." }, "volume": { "weekly": [ { "week": 1, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 128, "sqa28": 40, "sales_minutes": 299.14 }, { "week": 2, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 118, "sqa28": 47, "sales_minutes": 300.68 }, { "week": 3, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 123, "sqa28": 48, "sales_minutes": 300.69 }, { "week": 4, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 127, "sqa28": 45, "sales_minutes": 300.92 }, { "week": 5, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 132, "sqa28": 44, "sales_minutes": 301.74 }, { "week": 6, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 136, "sqa28": 41, "sales_minutes": 300.16 }, { "week": 7, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 113, "sqa28": 57, "sales_minutes": 301.18 }, { "week": 8, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 128, "sqa28": 57, "sales_minutes": 299.13 }, { "week": 9, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 116, "sqa28": 55, "sales_minutes": 298.67 }, { "week": 10, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 99, "sqa28": 47, "sales_minutes": 298.95 }, { "week": 11, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 136, "sqa28": 46, "sales_minutes": 299.42 }, { "week": 12, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 129, "sqa28": 48, "sales_minutes": 300.81 } ], "meanWeeklyEligible": 500.0, "distinctCompanies": 6000, "unresolvedAscertainment": 0, "immature": 0 }, "feasibility": { "observedMeanWeeklyEligible": 500.0, "weeksFor8000AtObservedVolume": 16.0, "accountsIn42DaysAtObservedVolume": 3000.0, "protocolStatement": { "weeksFor8000AtBaseline": 16, "accountsIn42DaysAtBaseline": 3000, "fullScenarioVolumeIsHypothetical": true, "message": "At the modeled historical volume of 500 eligible accounts per week, 8,000 accounts would take 16 weeks. The six-week cap supplies only 3,000. Full-size cases assume a hypothetical 1,333.33 accounts per week and do not support a feasible launch recommendation at this baseline volume." }, "objection": "At this volume the planned 8,000 needs about 16 weeks against a 6-week cap. The three full-size cases ASSUME higher volume for illustration; generating rows does not fix a real traffic shortage." }, "funnel": { "denominator": "all eligible accounts", "stages": [ { "id": "eligible", "label": "Eligible accounts", "count": 6000, "rate": 1.0 }, { "id": "imported7", "label": "Imported own data", "count": 3552, "rate": 0.592 }, { "id": "first_query7", "label": "First successful query", "count": 2686, "rate": 0.4476666667 }, { "id": "activated7", "label": "Activated (repeat query, later UTC date)", "count": 1485, "rate": 0.2475 }, { "id": "shared7", "label": "Colleague queried", "count": 754, "rate": 0.1256666667 }, { "id": "sqa28", "label": "SQA within 28 days", "count": 575, "rate": 0.0958333333 } ], "largestNestedDrop": { "from": "eligible", "to": "imported7", "accounts": 2448 } }, "segments": [ { "dimension": "acquisition_source", "segment": "content", "eligible": 1802, "imported7": 1076, "first_query7": 823, "activated7": 448, "shared7": 212, "sqa28": 178, "sqa28_among_activated": 36, "optout28": 26 }, { "dimension": "acquisition_source", "segment": "direct", "eligible": 1203, "imported7": 719, "first_query7": 538, "activated7": 292, "shared7": 144, "sqa28": 113, "sqa28_among_activated": 25, "optout28": 13 }, { "dimension": "acquisition_source", "segment": "product_community", "eligible": 2995, "imported7": 1757, "first_query7": 1325, "activated7": 745, "shared7": 398, "sqa28": 284, "sqa28_among_activated": 71, "optout28": 29 }, { "dimension": "company_size", "segment": "1-10", "eligible": 3014, "imported7": 1776, "first_query7": 1341, "activated7": 761, "shared7": 378, "sqa28": 295, "sqa28_among_activated": 71, "optout28": 35 }, { "dimension": "company_size", "segment": "11-50", "eligible": 2074, "imported7": 1236, "first_query7": 933, "activated7": 495, "shared7": 256, "sqa28": 195, "sqa28_among_activated": 45, "optout28": 25 }, { "dimension": "company_size", "segment": "51-200", "eligible": 912, "imported7": 540, "first_query7": 412, "activated7": 229, "shared7": 120, "sqa28": 85, "sqa28_among_activated": 16, "optout28": 8 }, { "dimension": "task_need", "segment": "import", "eligible": 2112, "imported7": 1267, "first_query7": 940, "activated7": 500, "shared7": 252, "sqa28": 189, "sqa28_among_activated": 41, "optout28": 25 }, { "dimension": "task_need", "segment": "sharing", "eligible": 2087, "imported7": 1222, "first_query7": 926, "activated7": 544, "shared7": 255, "sqa28": 207, "sqa28_among_activated": 51, "optout28": 26 }, { "dimension": "task_need", "segment": "unknown", "eligible": 1801, "imported7": 1063, "first_query7": 820, "activated7": 441, "shared7": 247, "sqa28": 179, "sqa28_among_activated": 40, "optout28": 17 } ], "cohorts": { "denominator": "all eligible accounts", "timeToValue": [ { "day": 1, "at_risk": 6000, "activated_cum": 0, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 2, "at_risk": 6000, "activated_cum": 0, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 3, "at_risk": 6000, "activated_cum": 318, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 5, "at_risk": 6000, "activated_cum": 906, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 7, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 14, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 153, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 21, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 379, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 28, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 575, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 } ] }, "hypothesis": "Task-specific trial help and an optional qualified Sales handoff increase SQA28 while preserving activation, contact tolerance, and Sales capacity.", "priorityHypothesis": "In this synthetic history, 59% of eligible accounts imported data but only 25% activated within 7 days; the largest nested drop is eligible → imported7 (2,448 accounts). SQA28 reached 9.6% overall (8.9% of activated accounts vs 9.8% of the rest — a descriptive association built into the synthetic propensity, not a causal effect). That motivates testing task-specific help around import, repeat use and sharing, judged on downstream SQA with activation and opt-out guardrails. It is an illustrative priority for this exercise, not a statement of MotherDuck's actual priorities.", "priorityBoundary": "Analyze this separately versioned history before presenting the candidate play. Descriptive segment gaps are hypotheses, not causal effects or proof this is MotherDuck's best priority.", "analysisOrder": [ "identity_and_maturity", "weekly_eligible_volume", "account_funnel", "source_and_task_segments", "time_to_value", "cost_and_sales_capacity", "candidate_priority_and_sample_feasibility" ], "receipts": [ "query:derive_account_outcomes", "query:baseline_volume", "query:baseline_funnel", "query:baseline_time_to_value" ] }, "scenarios": [ { "id": "beneficial", "seed": 2026092911, "label": "Assumed downstream benefit", "description": "Exercise every success gate; the sampled decision is not guaranteed.", "generativeAssumptions": { "controlRates": { "sqa28": 0.1, "activation7": 0.25, "optout28": 0.01, "sharing7": 0.12 }, "treatmentRates": { "sqa28": 0.13, "activation7": 0.28, "optout28": 0.01, "sharing7": 0.18 }, "salesMinutes": { "control": 0.6, "treatment": 0.9 }, "accounts": 8000, "enrollmentDays": 42, "note": "Inputs to the synthetic generator — not observed results and not the decision." }, "enrollment": { "control": 4000, "treatment": 4000, "total": 8000, "targetPerArm": 4000, "targetTotal": 8000, "plannedSampleReached": true, "firstAssignedAt": "2026-05-01T00:00:00.000Z", "lastAssignedAt": "2026-06-11T23:52:26.000Z", "enrollmentSpanDays": 41.995, "enrollmentCapDays": 42, "blocks": 800, "partialBlocks": 0, "timeline": "modeled: account i assigned at trialStartAt + floor(i·days·86400/accounts) s" }, "maturity": { "status": "mature", "analysisAt": "2026-07-09T23:52:26.000Z", "immatureAccounts": 0, "ascertainedAfterCutoff": 0, "rule": "final analysis at last assignment + 28 days" }, "integrity": { "status": "pass", "failures": [], "identicalEventRetriesCollapsed": 0, "checks": [ { "id": "ascertainment_after_cutoff_ignored", "violations": 0, "zeroTolerance": false }, { "id": "ascertainment_conflict", "violations": 0, "zeroTolerance": true }, { "id": "assignment_replay_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "completed_block_imbalance", "violations": 0, "zeroTolerance": true }, { "id": "contact_after_optout", "violations": 0, "zeroTolerance": true }, { "id": "contradictory_ascertainment_statuses", "violations": 0, "zeroTolerance": true }, { "id": "documented_nonreceipt", "violations": 0, "zeroTolerance": false }, { "id": "duplicate_account_in_denominator", "violations": 0, "zeroTolerance": true }, { "id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "zeroTolerance": true }, { "id": "event_for_unassigned_account", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_missing", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_unplanned", "violations": 0, "zeroTolerance": true }, { "id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "zeroTolerance": true }, { "id": "future_ascertainment_contradiction", "violations": 0, "zeroTolerance": true }, { "id": "future_evidence_in_routing", "violations": 0, "zeroTolerance": true }, { "id": "identical_event_retries_collapsed", "violations": 0, "zeroTolerance": false }, { "id": "ledger_reconciliation_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "missing_required_outcome_ascertainment", "violations": 0, "zeroTolerance": true }, { "id": "more_than_two_proactive_contacts", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_ledger_incomplete", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_status_inconsistent", "violations": 0, "zeroTolerance": true }, { "id": "outcome_before_assignment", "violations": 0, "zeroTolerance": true }, { "id": "partial_block_arm_difference_over_5", "violations": 0, "zeroTolerance": true }, { "id": "unresolved_cross_arm_company_collision", "violations": 0, "zeroTolerance": true } ] }, "metrics": { "sqa28": { "control": { "events": 390, "total": 4000, "rate": 0.0975, "confirmedZero": 3610, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 478, "total": 4000, "rate": 0.1195, "confirmedZero": 3522, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.022, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0083726675, "upper": 0.0356447876 }, "denominator": "all randomized eligible accounts (ITT)", "role": "primary", "gate": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +0.84 pts is above zero and the observed gain +2.20 pts meets +1.50 pts" } }, "activation7": { "control": { "events": 1008, "total": 4000, "rate": 0.252, "confirmedZero": 2992, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 1154, "total": 4000, "rate": 0.2885, "confirmedZero": 2846, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0365, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0170399321, "upper": 0.0559238233 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound +1.70 pts is above −3.00 pts" } }, "optout28": { "control": { "events": 40, "total": 4000, "rate": 0.01, "confirmedZero": 3960, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 40, "total": 4000, "rate": 0.01, "confirmedZero": 3960, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0044588177, "upper": 0.0044588177 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.45 pts is below +1.00 pts" } }, "sharing7": { "control": { "events": 493, "total": 4000, "rate": 0.12325, "confirmedZero": 3507, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 752, "total": 4000, "rate": 0.188, "confirmedZero": 3248, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.06475, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0489164844, "upper": 0.0805760613 }, "denominator": "all randomized eligible accounts (ITT)", "role": "descriptive_secondary", "gate": { "status": "descriptive_only", "code": "not_a_gate", "reason": "Descriptive secondary; never a success gate." } }, "salesMinutesPerAssignedAccount": { "control": { "minutes": 2398.1349, "total": 4000, "mean": 0.599533725 }, "treatment": { "minutes": 3597.9164, "total": 4000, "mean": 0.8994791 }, "delta": 0.299945375, "cap": 0.5, "role": "operating_cap", "gate": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.300 Sales minutes per assigned account is within the 0.5 cap" } } }, "costs": { "dataKind": "assumed_synthetic_costs", "currency": "USD", "arms": { "control": { "assigned": 4000, "contacts": 7993, "sales_minutes": 2398.1349, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.965, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2398.1349, "total": 26598.0999, "per_assigned": 6.649524975, "per_activated7": 26.387003869, "per_sqa28": 68.2002561538 }, "treatment": { "assigned": 4000, "contacts": 7998, "sales_minutes": 3597.9164, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.99, "content_ops": 240.0, "inference": 8.0, "sales_labor": 3597.9164, "total": 27925.9064, "per_assigned": 6.9814766, "per_activated7": 24.1992256499, "per_sqa28": 58.4223983264 } }, "extraSalesMinutesPerAccount": 0.299945375, "incremental": { "costPerAssignedDelta": 0.331951625, "sqaDelta": 0.022, "costPerIncrementalSqa": 15.0887102273, "costPerIncrementalSqaRange": { "fromSqaUpper": 9.3127676653, "fromSqaLower": 39.6470569292 }, "uncertain": false, "note": "Point estimate with the range implied by the SQA interval." }, "note": "Invented allocations from the frozen protocol; not vendor prices, not measured charges." }, "decision": { "code": "advance", "status": "advance", "label": "Advance to a bounded expansion", "reasons": [ "SQA lower bound +0.84 pts is above zero and the observed gain +2.20 pts meets +1.50 pts", "activation lower bound +1.70 pts is above −3.00 pts", "opt-out upper bound +0.45 pts is below +1.00 pts", "+0.300 Sales minutes per assigned account is within the 0.5 cap" ], "nextStep": "Attempt a bounded, monitored expansion. This does not prove the true SQA lift exceeds 1.5 points or permit unlimited scaling.", "allGates": { "sqa28": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +0.84 pts is above zero and the observed gain +2.20 pts meets +1.50 pts" }, "activation7": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound +1.70 pts is above −3.00 pts" }, "optout28": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.45 pts is below +1.00 pts" }, "salesMinutesPerAssignedAccount": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.300 Sales minutes per assigned account is within the 0.5 cap" } }, "scope": "synthetic_case_only" }, "cohorts": { "denominator": "all assigned accounts (non-converters stay at risk)", "atRisk": { "control": 4000, "treatment": 4000 }, "activation7": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 202, "treatment": 237, "controlRate": 0.0505, "treatmentRate": 0.05925 }, { "day": 4, "control": 403, "treatment": 506, "controlRate": 0.10075, "treatmentRate": 0.1265 }, { "day": 5, "control": 598, "treatment": 727, "controlRate": 0.1495, "treatmentRate": 0.18175 }, { "day": 6, "control": 796, "treatment": 921, "controlRate": 0.199, "treatmentRate": 0.23025 }, { "day": 7, "control": 1008, "treatment": 1154, "controlRate": 0.252, "treatmentRate": 0.2885 } ], "sqa28": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 4, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 5, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 6, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 7, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 8, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 9, "control": 13, "treatment": 23, "controlRate": 0.00325, "treatmentRate": 0.00575 }, { "day": 10, "control": 30, "treatment": 53, "controlRate": 0.0075, "treatmentRate": 0.01325 }, { "day": 11, "control": 53, "treatment": 77, "controlRate": 0.01325, "treatmentRate": 0.01925 }, { "day": 12, "control": 74, "treatment": 105, "controlRate": 0.0185, "treatmentRate": 0.02625 }, { "day": 13, "control": 91, "treatment": 132, "controlRate": 0.02275, "treatmentRate": 0.033 }, { "day": 14, "control": 105, "treatment": 152, "controlRate": 0.02625, "treatmentRate": 0.038 }, { "day": 15, "control": 123, "treatment": 174, "controlRate": 0.03075, "treatmentRate": 0.0435 }, { "day": 16, "control": 139, "treatment": 196, "controlRate": 0.03475, "treatmentRate": 0.049 }, { "day": 17, "control": 172, "treatment": 214, "controlRate": 0.043, "treatmentRate": 0.0535 }, { "day": 18, "control": 189, "treatment": 238, "controlRate": 0.04725, "treatmentRate": 0.0595 }, { "day": 19, "control": 209, "treatment": 266, "controlRate": 0.05225, "treatmentRate": 0.0665 }, { "day": 20, "control": 230, "treatment": 288, "controlRate": 0.0575, "treatmentRate": 0.072 }, { "day": 21, "control": 253, "treatment": 312, "controlRate": 0.06325, "treatmentRate": 0.078 }, { "day": 22, "control": 269, "treatment": 335, "controlRate": 0.06725, "treatmentRate": 0.08375 }, { "day": 23, "control": 285, "treatment": 357, "controlRate": 0.07125, "treatmentRate": 0.08925 }, { "day": 24, "control": 313, "treatment": 381, "controlRate": 0.07825, "treatmentRate": 0.09525 }, { "day": 25, "control": 334, "treatment": 408, "controlRate": 0.0835, "treatmentRate": 0.102 }, { "day": 26, "control": 351, "treatment": 438, "controlRate": 0.08775, "treatmentRate": 0.1095 }, { "day": 27, "control": 368, "treatment": 458, "controlRate": 0.092, "treatmentRate": 0.1145 }, { "day": 28, "control": 390, "treatment": 478, "controlRate": 0.0975, "treatmentRate": 0.1195 } ] }, "funnel": { "denominator": "all assigned accounts in the arm", "note": "SQA is a parallel downstream endpoint, not nested under activation.", "stages": [ { "id": "assigned", "label": "Randomized eligible accounts", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted", "label": "Received ≥1 proactive contact", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted_twice", "label": "Received both contacts", "control": 3993, "treatment": 3998, "controlRate": 0.99825, "treatmentRate": 0.9995 }, { "id": "imported7", "label": "Imported own data (7 days)", "control": 2395, "treatment": 2564, "controlRate": 0.59875, "treatmentRate": 0.641 }, { "id": "first_query7", "label": "First successful query (7 days)", "control": 1816, "treatment": 1951, "controlRate": 0.454, "treatmentRate": 0.48775 }, { "id": "activated7", "label": "Activated: repeat query on a later UTC date (7 days)", "control": 1008, "treatment": 1154, "controlRate": 0.252, "treatmentRate": 0.2885 }, { "id": "shared7", "label": "Colleague queried (7 days, descriptive)", "control": 493, "treatment": 752, "controlRate": 0.12325, "treatmentRate": 0.188 }, { "id": "sqa28", "label": "SQA within 28 days (parallel endpoint)", "control": 390, "treatment": 478, "controlRate": 0.0975, "treatmentRate": 0.1195 }, { "id": "optout28", "label": "Opted out within 28 days", "control": 40, "treatment": 40, "controlRate": 0.01, "treatmentRate": 0.01 } ], "sqaWithoutActivation": { "control": 281, "treatment": 343 } }, "segments": [ { "dimension": "acquisition_source", "segment": "content", "kind": "prespecified_descriptive", "control": { "assigned": 1297, "sqa28": 123, "activation7": 320, "optout28": 12 }, "treatment": { "assigned": 1178, "sqa28": 143, "activation7": 371, "optout28": 12 }, "sqa28": { "delta": 0.0265579573, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0020969722, "upper": 0.051332532 } } }, { "dimension": "acquisition_source", "segment": "direct", "kind": "prespecified_descriptive", "control": { "assigned": 787, "sqa28": 70, "activation7": 197, "optout28": 9 }, "treatment": { "assigned": 794, "sqa28": 95, "activation7": 224, "optout28": 12 }, "sqa28": { "delta": 0.030701993, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0004972971, "upper": 0.0610100802 } } }, { "dimension": "acquisition_source", "segment": "product_community", "kind": "prespecified_descriptive", "control": { "assigned": 1916, "sqa28": 197, "activation7": 491, "optout28": 19 }, "treatment": { "assigned": 2028, "sqa28": 240, "activation7": 559, "optout28": 16 }, "sqa28": { "delta": 0.0155248237, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0041089097, "upper": 0.0350904184 } } }, { "dimension": "company_size", "segment": "1-10", "kind": "prespecified_descriptive", "control": { "assigned": 2009, "sqa28": 214, "activation7": 510, "optout28": 21 }, "treatment": { "assigned": 2003, "sqa28": 252, "activation7": 570, "optout28": 19 }, "sqa28": { "delta": 0.019290626, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.000553274, "upper": 0.0391616122 } } }, { "dimension": "company_size", "segment": "11-50", "kind": "prespecified_descriptive", "control": { "assigned": 1379, "sqa28": 127, "activation7": 344, "optout28": 14 }, "treatment": { "assigned": 1371, "sqa28": 157, "activation7": 388, "optout28": 13 }, "sqa28": { "delta": 0.0224192311, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0003577486, "upper": 0.0452706925 } } }, { "dimension": "company_size", "segment": "51-200", "kind": "prespecified_descriptive", "control": { "assigned": 612, "sqa28": 49, "activation7": 154, "optout28": 5 }, "treatment": { "assigned": 626, "sqa28": 69, "activation7": 196, "optout28": 8 }, "sqa28": { "delta": 0.0301582827, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0026887375, "upper": 0.0631178642 } } }, { "dimension": "task_need", "segment": "import", "kind": "prespecified_descriptive", "control": { "assigned": 1428, "sqa28": 142, "activation7": 358, "optout28": 17 }, "treatment": { "assigned": 1366, "sqa28": 167, "activation7": 387, "optout28": 13 }, "sqa28": { "delta": 0.0228149825, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.000472572, "upper": 0.0462491969 } } }, { "dimension": "task_need", "segment": "sharing", "kind": "prespecified_descriptive", "control": { "assigned": 1359, "sqa28": 125, "activation7": 338, "optout28": 12 }, "treatment": { "assigned": 1402, "sqa28": 175, "activation7": 395, "optout28": 15 }, "sqa28": { "delta": 0.0328422867, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0096351692, "upper": 0.0560535031 } } }, { "dimension": "task_need", "segment": "unknown", "kind": "prespecified_descriptive", "control": { "assigned": 1213, "sqa28": 123, "activation7": 312, "optout28": 11 }, "treatment": { "assigned": 1232, "sqa28": 136, "activation7": 372, "optout28": 12 }, "sqa28": { "delta": 0.0089881265, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0154943387, "upper": 0.0334549957 } } } ], "exampleAccountTrace": { "selectionRule": "first treatment account by enrollment order with a documented SQA", "accountId": "account-000004", "arm": "treatment", "enrollmentIndex": 3, "blockIndex": 0, "token": "T4", "assignedAt": "2026-05-01T00:22:40.000Z", "attributes": { "acquisitionSource": "product_community", "taskNeed": "sharing", "companySize": "11-50" }, "events": [ { "event_id": "beneficial:account-000004:trial_started", "event_ts": "2026-05-01T00:22:40.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "beneficial:account-000004:contact_sent:0", "event_ts": "2026-05-02T00:22:40.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000004-u1", "evidence_event_id": "beneficial:account-000004:trial_started", "sales_minutes": null }, { "event_id": "beneficial:account-000004:data_imported:0", "event_ts": "2026-05-02T02:22:40.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "beneficial:account-000004:own_data_query:0", "event_ts": "2026-05-02T03:22:40.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "beneficial:account-000004:contact_sent:1", "event_ts": "2026-05-04T00:22:40.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000004-u1", "evidence_event_id": "beneficial:account-000004:trial_started", "sales_minutes": null }, { "event_id": "beneficial:account-000004:own_data_query:1", "event_ts": "2026-05-04T03:22:40.000Z", "elapsed_days": 3.125, "event_type": "own_data_query", "detail": "repeat", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "beneficial:account-000004:sqa_qualified:0", "event_ts": "2026-05-26T05:22:40.000Z", "elapsed_days": 25.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "beneficial:account-000004:sales_effort_logged:0", "event_ts": "2026-05-28T23:22:40.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9585 } ] }, "receipts": [ "query:derive_account_outcomes", "query:integrity", "query:maturity", "query:metrics", "query:funnel", "query:cohort_curves", "query:segments", "query:costs", "query:trace" ] }, { "id": "upstream_only", "seed": 2026092912, "label": "Upstream improvement without assumed SQA lift", "description": "A tempting activation improvement cannot substitute for downstream benefit; retain whatever sampled decision occurs.", "generativeAssumptions": { "controlRates": { "sqa28": 0.1, "activation7": 0.25, "optout28": 0.01, "sharing7": 0.12 }, "treatmentRates": { "sqa28": 0.1, "activation7": 0.33, "optout28": 0.01, "sharing7": 0.22 }, "salesMinutes": { "control": 0.6, "treatment": 0.9 }, "accounts": 8000, "enrollmentDays": 42, "note": "Inputs to the synthetic generator — not observed results and not the decision." }, "enrollment": { "control": 4000, "treatment": 4000, "total": 8000, "targetPerArm": 4000, "targetTotal": 8000, "plannedSampleReached": true, "firstAssignedAt": "2026-05-01T00:00:00.000Z", "lastAssignedAt": "2026-06-11T23:52:26.000Z", "enrollmentSpanDays": 41.995, "enrollmentCapDays": 42, "blocks": 800, "partialBlocks": 0, "timeline": "modeled: account i assigned at trialStartAt + floor(i·days·86400/accounts) s" }, "maturity": { "status": "mature", "analysisAt": "2026-07-09T23:52:26.000Z", "immatureAccounts": 0, "ascertainedAfterCutoff": 0, "rule": "final analysis at last assignment + 28 days" }, "integrity": { "status": "pass", "failures": [], "identicalEventRetriesCollapsed": 0, "checks": [ { "id": "ascertainment_after_cutoff_ignored", "violations": 0, "zeroTolerance": false }, { "id": "ascertainment_conflict", "violations": 0, "zeroTolerance": true }, { "id": "assignment_replay_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "completed_block_imbalance", "violations": 0, "zeroTolerance": true }, { "id": "contact_after_optout", "violations": 0, "zeroTolerance": true }, { "id": "contradictory_ascertainment_statuses", "violations": 0, "zeroTolerance": true }, { "id": "documented_nonreceipt", "violations": 0, "zeroTolerance": false }, { "id": "duplicate_account_in_denominator", "violations": 0, "zeroTolerance": true }, { "id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "zeroTolerance": true }, { "id": "event_for_unassigned_account", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_missing", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_unplanned", "violations": 0, "zeroTolerance": true }, { "id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "zeroTolerance": true }, { "id": "future_ascertainment_contradiction", "violations": 0, "zeroTolerance": true }, { "id": "future_evidence_in_routing", "violations": 0, "zeroTolerance": true }, { "id": "identical_event_retries_collapsed", "violations": 0, "zeroTolerance": false }, { "id": "ledger_reconciliation_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "missing_required_outcome_ascertainment", "violations": 0, "zeroTolerance": true }, { "id": "more_than_two_proactive_contacts", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_ledger_incomplete", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_status_inconsistent", "violations": 0, "zeroTolerance": true }, { "id": "outcome_before_assignment", "violations": 0, "zeroTolerance": true }, { "id": "partial_block_arm_difference_over_5", "violations": 0, "zeroTolerance": true }, { "id": "unresolved_cross_arm_company_collision", "violations": 0, "zeroTolerance": true } ] }, "metrics": { "sqa28": { "control": { "events": 394, "total": 4000, "rate": 0.0985, "confirmedZero": 3606, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 407, "total": 4000, "rate": 0.10175, "confirmedZero": 3593, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.00325, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0099193081, "upper": 0.0164227012 }, "denominator": "all randomized eligible accounts (ITT)", "role": "primary", "gate": { "status": "not_established", "code": "sqa_lower_not_above_zero", "reason": "SQA lower bound −0.99 pts does not exclude zero" } }, "activation7": { "control": { "events": 1019, "total": 4000, "rate": 0.25475, "confirmedZero": 2981, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 1363, "total": 4000, "rate": 0.34075, "confirmedZero": 2637, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.086, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0660073798, "upper": 0.1058990674 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound +6.60 pts is above −3.00 pts" } }, "optout28": { "control": { "events": 39, "total": 4000, "rate": 0.00975, "confirmedZero": 3961, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 37, "total": 4000, "rate": 0.00925, "confirmedZero": 3963, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": -0.0005, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0048602576, "upper": 0.0038442108 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.38 pts is below +1.00 pts" } }, "sharing7": { "control": { "events": 499, "total": 4000, "rate": 0.12475, "confirmedZero": 3501, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 855, "total": 4000, "rate": 0.21375, "confirmedZero": 3145, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.089, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0726656131, "upper": 0.1053100027 }, "denominator": "all randomized eligible accounts (ITT)", "role": "descriptive_secondary", "gate": { "status": "descriptive_only", "code": "not_a_gate", "reason": "Descriptive secondary; never a success gate." } }, "salesMinutesPerAssignedAccount": { "control": { "minutes": 2404.5432, "total": 4000, "mean": 0.6011358 }, "treatment": { "minutes": 3601.2896, "total": 4000, "mean": 0.9003224 }, "delta": 0.2991866, "cap": 0.5, "role": "operating_cap", "gate": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.299 Sales minutes per assigned account is within the 0.5 cap" } } }, "costs": { "dataKind": "assumed_synthetic_costs", "currency": "USD", "arms": { "control": { "assigned": 4000, "contacts": 7996, "sales_minutes": 2404.5432, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.98, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2404.5432, "total": 26604.5232, "per_assigned": 6.6511308, "per_activated7": 26.1084624141, "per_sqa28": 67.5241705584 }, "treatment": { "assigned": 4000, "contacts": 7997, "sales_minutes": 3601.2896, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.985, "content_ops": 240.0, "inference": 8.0, "sales_labor": 3601.2896, "total": 27929.2746, "per_assigned": 6.98231865, "per_activated7": 20.4910305209, "per_sqa28": 68.6222963145 } }, "extraSalesMinutesPerAccount": 0.2991866, "incremental": { "costPerAssignedDelta": 0.33118785, "sqaDelta": 0.00325, "costPerIncrementalSqa": 101.9039538462, "costPerIncrementalSqaRange": { "fromSqaUpper": 20.166466322, "fromSqaLower": null }, "uncertain": true, "note": "Carries the SQA interval's uncertainty; an unbounded end means the interval includes zero." }, "note": "Invented allocations from the frozen protocol; not vendor prices, not measured charges." }, "decision": { "code": "inconclusive", "status": "inconclusive", "label": "Inconclusive — do not expand", "reasons": [ "SQA lower bound −0.99 pts does not exclude zero" ], "nextStep": "Do not expand. Report the estimates and intervals; this is not proof of no effect.", "allGates": { "sqa28": { "status": "not_established", "code": "sqa_lower_not_above_zero", "reason": "SQA lower bound −0.99 pts does not exclude zero" }, "activation7": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound +6.60 pts is above −3.00 pts" }, "optout28": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.38 pts is below +1.00 pts" }, "salesMinutesPerAssignedAccount": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.299 Sales minutes per assigned account is within the 0.5 cap" } }, "scope": "synthetic_case_only" }, "cohorts": { "denominator": "all assigned accounts (non-converters stay at risk)", "atRisk": { "control": 4000, "treatment": 4000 }, "activation7": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 226, "treatment": 285, "controlRate": 0.0565, "treatmentRate": 0.07125 }, { "day": 4, "control": 411, "treatment": 545, "controlRate": 0.10275, "treatmentRate": 0.13625 }, { "day": 5, "control": 601, "treatment": 820, "controlRate": 0.15025, "treatmentRate": 0.205 }, { "day": 6, "control": 821, "treatment": 1069, "controlRate": 0.20525, "treatmentRate": 0.26725 }, { "day": 7, "control": 1019, "treatment": 1363, "controlRate": 0.25475, "treatmentRate": 0.34075 } ], "sqa28": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 4, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 5, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 6, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 7, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 8, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 9, "control": 14, "treatment": 29, "controlRate": 0.0035, "treatmentRate": 0.00725 }, { "day": 10, "control": 34, "treatment": 45, "controlRate": 0.0085, "treatmentRate": 0.01125 }, { "day": 11, "control": 51, "treatment": 76, "controlRate": 0.01275, "treatmentRate": 0.019 }, { "day": 12, "control": 72, "treatment": 98, "controlRate": 0.018, "treatmentRate": 0.0245 }, { "day": 13, "control": 93, "treatment": 123, "controlRate": 0.02325, "treatmentRate": 0.03075 }, { "day": 14, "control": 113, "treatment": 144, "controlRate": 0.02825, "treatmentRate": 0.036 }, { "day": 15, "control": 132, "treatment": 156, "controlRate": 0.033, "treatmentRate": 0.039 }, { "day": 16, "control": 152, "treatment": 178, "controlRate": 0.038, "treatmentRate": 0.0445 }, { "day": 17, "control": 174, "treatment": 202, "controlRate": 0.0435, "treatmentRate": 0.0505 }, { "day": 18, "control": 200, "treatment": 218, "controlRate": 0.05, "treatmentRate": 0.0545 }, { "day": 19, "control": 217, "treatment": 237, "controlRate": 0.05425, "treatmentRate": 0.05925 }, { "day": 20, "control": 241, "treatment": 257, "controlRate": 0.06025, "treatmentRate": 0.06425 }, { "day": 21, "control": 256, "treatment": 275, "controlRate": 0.064, "treatmentRate": 0.06875 }, { "day": 22, "control": 272, "treatment": 293, "controlRate": 0.068, "treatmentRate": 0.07325 }, { "day": 23, "control": 296, "treatment": 309, "controlRate": 0.074, "treatmentRate": 0.07725 }, { "day": 24, "control": 314, "treatment": 331, "controlRate": 0.0785, "treatmentRate": 0.08275 }, { "day": 25, "control": 331, "treatment": 347, "controlRate": 0.08275, "treatmentRate": 0.08675 }, { "day": 26, "control": 357, "treatment": 364, "controlRate": 0.08925, "treatmentRate": 0.091 }, { "day": 27, "control": 378, "treatment": 387, "controlRate": 0.0945, "treatmentRate": 0.09675 }, { "day": 28, "control": 394, "treatment": 407, "controlRate": 0.0985, "treatmentRate": 0.10175 } ] }, "funnel": { "denominator": "all assigned accounts in the arm", "note": "SQA is a parallel downstream endpoint, not nested under activation.", "stages": [ { "id": "assigned", "label": "Randomized eligible accounts", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted", "label": "Received ≥1 proactive contact", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted_twice", "label": "Received both contacts", "control": 3996, "treatment": 3997, "controlRate": 0.999, "treatmentRate": 0.99925 }, { "id": "imported7", "label": "Imported own data (7 days)", "control": 2464, "treatment": 2722, "controlRate": 0.616, "treatmentRate": 0.6805 }, { "id": "first_query7", "label": "First successful query (7 days)", "control": 1820, "treatment": 2120, "controlRate": 0.455, "treatmentRate": 0.53 }, { "id": "activated7", "label": "Activated: repeat query on a later UTC date (7 days)", "control": 1019, "treatment": 1363, "controlRate": 0.25475, "treatmentRate": 0.34075 }, { "id": "shared7", "label": "Colleague queried (7 days, descriptive)", "control": 499, "treatment": 855, "controlRate": 0.12475, "treatmentRate": 0.21375 }, { "id": "sqa28", "label": "SQA within 28 days (parallel endpoint)", "control": 394, "treatment": 407, "controlRate": 0.0985, "treatmentRate": 0.10175 }, { "id": "optout28", "label": "Opted out within 28 days", "control": 39, "treatment": 37, "controlRate": 0.00975, "treatmentRate": 0.00925 } ], "sqaWithoutActivation": { "control": 290, "treatment": 262 } }, "segments": [ { "dimension": "acquisition_source", "segment": "content", "kind": "prespecified_descriptive", "control": { "assigned": 1182, "sqa28": 117, "activation7": 299, "optout28": 11 }, "treatment": { "assigned": 1193, "sqa28": 129, "activation7": 407, "optout28": 13 }, "sqa28": { "delta": 0.0091459912, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0154489143, "upper": 0.0337441443 } } }, { "dimension": "acquisition_source", "segment": "direct", "kind": "prespecified_descriptive", "control": { "assigned": 768, "sqa28": 79, "activation7": 207, "optout28": 7 }, "treatment": { "assigned": 818, "sqa28": 59, "activation7": 300, "optout28": 5 }, "sqa28": { "delta": -0.030737444, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0590537166, "upper": -0.0029157482 } } }, { "dimension": "acquisition_source", "segment": "product_community", "kind": "prespecified_descriptive", "control": { "assigned": 2050, "sqa28": 198, "activation7": 513, "optout28": 21 }, "treatment": { "assigned": 1989, "sqa28": 219, "activation7": 656, "optout28": 19 }, "sqa28": { "delta": 0.0135202148, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0052672335, "upper": 0.032380793 } } }, { "dimension": "company_size", "segment": "1-10", "kind": "prespecified_descriptive", "control": { "assigned": 2040, "sqa28": 204, "activation7": 505, "optout28": 22 }, "treatment": { "assigned": 1984, "sqa28": 194, "activation7": 690, "optout28": 14 }, "sqa28": { "delta": -0.0022177419, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0206903261, "upper": 0.0162951208 } } }, { "dimension": "company_size", "segment": "11-50", "kind": "prespecified_descriptive", "control": { "assigned": 1374, "sqa28": 130, "activation7": 372, "optout28": 11 }, "treatment": { "assigned": 1432, "sqa28": 156, "activation7": 483, "optout28": 17 }, "sqa28": { "delta": 0.0143242826, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0081370534, "upper": 0.0367306004 } } }, { "dimension": "company_size", "segment": "51-200", "kind": "prespecified_descriptive", "control": { "assigned": 586, "sqa28": 60, "activation7": 142, "optout28": 6 }, "treatment": { "assigned": 584, "sqa28": 57, "activation7": 190, "optout28": 6 }, "sqa28": { "delta": -0.0047863388, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0394539915, "upper": 0.0298673101 } } }, { "dimension": "task_need", "segment": "import", "kind": "prespecified_descriptive", "control": { "assigned": 1402, "sqa28": 146, "activation7": 347, "optout28": 17 }, "treatment": { "assigned": 1418, "sqa28": 146, "activation7": 495, "optout28": 15 }, "sqa28": { "delta": -0.001175029, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0237526888, "upper": 0.021373724 } } }, { "dimension": "task_need", "segment": "sharing", "kind": "prespecified_descriptive", "control": { "assigned": 1394, "sqa28": 137, "activation7": 349, "optout28": 11 }, "treatment": { "assigned": 1427, "sqa28": 143, "activation7": 473, "optout28": 10 }, "sqa28": { "delta": 0.0019318955, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0202361879, "upper": 0.0240519366 } } }, { "dimension": "task_need", "segment": "unknown", "kind": "prespecified_descriptive", "control": { "assigned": 1204, "sqa28": 111, "activation7": 323, "optout28": 11 }, "treatment": { "assigned": 1155, "sqa28": 118, "activation7": 395, "optout28": 12 }, "sqa28": { "delta": 0.0099718111, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0139743976, "upper": 0.0340707643 } } } ], "exampleAccountTrace": { "selectionRule": "first treatment account by enrollment order with a documented SQA", "accountId": "account-000007", "arm": "treatment", "enrollmentIndex": 6, "blockIndex": 0, "token": "T1", "assignedAt": "2026-05-01T00:45:21.000Z", "attributes": { "acquisitionSource": "direct", "taskNeed": "unknown", "companySize": "1-10" }, "events": [ { "event_id": "upstream_only:account-000007:trial_started", "event_ts": "2026-05-01T00:45:21.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=unknown", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "upstream_only:account-000007:contact_sent:0", "event_ts": "2026-05-02T00:45:21.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_general", "user_id": "account-000007-u1", "evidence_event_id": "upstream_only:account-000007:trial_started", "sales_minutes": null }, { "event_id": "upstream_only:account-000007:data_imported:0", "event_ts": "2026-05-02T02:45:21.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "upstream_only:account-000007:own_data_query:0", "event_ts": "2026-05-02T03:45:21.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "upstream_only:account-000007:contact_sent:1", "event_ts": "2026-05-04T00:45:21.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_general", "user_id": "account-000007-u1", "evidence_event_id": "upstream_only:account-000007:trial_started", "sales_minutes": null }, { "event_id": "upstream_only:account-000007:sqa_qualified:0", "event_ts": "2026-05-17T05:45:21.000Z", "elapsed_days": 16.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "upstream_only:account-000007:sales_effort_logged:0", "event_ts": "2026-05-28T23:45:21.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9516 } ] }, "receipts": [ "query:derive_account_outcomes", "query:integrity", "query:maturity", "query:metrics", "query:funnel", "query:cohort_curves", "query:segments", "query:costs", "query:trace" ] }, { "id": "harm", "seed": 2026092913, "label": "Assumed guardrail harm", "description": "Possible SQA improvement cannot override activation/contact harm or an exceeded Sales budget.", "generativeAssumptions": { "controlRates": { "sqa28": 0.1, "activation7": 0.25, "optout28": 0.01, "sharing7": 0.12 }, "treatmentRates": { "sqa28": 0.13, "activation7": 0.19, "optout28": 0.03, "sharing7": 0.15 }, "salesMinutes": { "control": 0.6, "treatment": 1.3 }, "accounts": 8000, "enrollmentDays": 42, "note": "Inputs to the synthetic generator — not observed results and not the decision." }, "enrollment": { "control": 4000, "treatment": 4000, "total": 8000, "targetPerArm": 4000, "targetTotal": 8000, "plannedSampleReached": true, "firstAssignedAt": "2026-05-01T00:00:00.000Z", "lastAssignedAt": "2026-06-11T23:52:26.000Z", "enrollmentSpanDays": 41.995, "enrollmentCapDays": 42, "blocks": 800, "partialBlocks": 0, "timeline": "modeled: account i assigned at trialStartAt + floor(i·days·86400/accounts) s" }, "maturity": { "status": "mature", "analysisAt": "2026-07-09T23:52:26.000Z", "immatureAccounts": 0, "ascertainedAfterCutoff": 0, "rule": "final analysis at last assignment + 28 days" }, "integrity": { "status": "pass", "failures": [], "identicalEventRetriesCollapsed": 0, "checks": [ { "id": "ascertainment_after_cutoff_ignored", "violations": 0, "zeroTolerance": false }, { "id": "ascertainment_conflict", "violations": 0, "zeroTolerance": true }, { "id": "assignment_replay_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "completed_block_imbalance", "violations": 0, "zeroTolerance": true }, { "id": "contact_after_optout", "violations": 0, "zeroTolerance": true }, { "id": "contradictory_ascertainment_statuses", "violations": 0, "zeroTolerance": true }, { "id": "documented_nonreceipt", "violations": 0, "zeroTolerance": false }, { "id": "duplicate_account_in_denominator", "violations": 0, "zeroTolerance": true }, { "id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "zeroTolerance": true }, { "id": "event_for_unassigned_account", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_missing", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_unplanned", "violations": 0, "zeroTolerance": true }, { "id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "zeroTolerance": true }, { "id": "future_ascertainment_contradiction", "violations": 0, "zeroTolerance": true }, { "id": "future_evidence_in_routing", "violations": 0, "zeroTolerance": true }, { "id": "identical_event_retries_collapsed", "violations": 0, "zeroTolerance": false }, { "id": "ledger_reconciliation_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "missing_required_outcome_ascertainment", "violations": 0, "zeroTolerance": true }, { "id": "more_than_two_proactive_contacts", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_ledger_incomplete", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_status_inconsistent", "violations": 0, "zeroTolerance": true }, { "id": "outcome_before_assignment", "violations": 0, "zeroTolerance": true }, { "id": "partial_block_arm_difference_over_5", "violations": 0, "zeroTolerance": true }, { "id": "unresolved_cross_arm_company_collision", "violations": 0, "zeroTolerance": true } ] }, "metrics": { "sqa28": { "control": { "events": 397, "total": 4000, "rate": 0.09925, "confirmedZero": 3603, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 515, "total": 4000, "rate": 0.12875, "confirmedZero": 3485, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0295, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0155830869, "upper": 0.0434360634 }, "denominator": "all randomized eligible accounts (ITT)", "role": "primary", "gate": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +1.56 pts is above zero and the observed gain +2.95 pts meets +1.50 pts" } }, "activation7": { "control": { "events": 996, "total": 4000, "rate": 0.249, "confirmedZero": 3004, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 763, "total": 4000, "rate": 0.19075, "confirmedZero": 3237, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": -0.05825, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0763342319, "upper": -0.0401231582 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "stop", "code": "activation_harm", "reason": "activation upper bound −4.01 pts is below the −3.00 pts tolerated loss" } }, "optout28": { "control": { "events": 36, "total": 4000, "rate": 0.009, "confirmedZero": 3964, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 114, "total": 4000, "rate": 0.0285, "confirmedZero": 3886, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0195, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0136617009, "upper": 0.0256534329 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "stop", "code": "optout_harm", "reason": "opt-out lower bound +1.37 pts is above the +1.00 pts tolerated increase" } }, "sharing7": { "control": { "events": 462, "total": 4000, "rate": 0.1155, "confirmedZero": 3538, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 578, "total": 4000, "rate": 0.1445, "confirmedZero": 3422, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.029, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0142689348, "upper": 0.0437394262 }, "denominator": "all randomized eligible accounts (ITT)", "role": "descriptive_secondary", "gate": { "status": "descriptive_only", "code": "not_a_gate", "reason": "Descriptive secondary; never a success gate." } }, "salesMinutesPerAssignedAccount": { "control": { "minutes": 2398.0449, "total": 4000, "mean": 0.599511225 }, "treatment": { "minutes": 5200.5454, "total": 4000, "mean": 1.30013635 }, "delta": 0.700625125, "cap": 0.5, "role": "operating_cap", "gate": { "status": "stop", "code": "sales_cap_exceeded", "reason": "+0.701 Sales minutes per assigned account exceeds the 0.5 cap" } } }, "costs": { "dataKind": "assumed_synthetic_costs", "currency": "USD", "arms": { "control": { "assigned": 4000, "contacts": 7999, "sales_minutes": 2398.0449, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.995, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2398.0449, "total": 26598.0399, "per_assigned": 6.649509975, "per_activated7": 26.7048593373, "per_sqa28": 66.9975816121 }, "treatment": { "assigned": 4000, "contacts": 7992, "sales_minutes": 5200.5454, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.96, "content_ops": 240.0, "inference": 8.0, "sales_labor": 5200.5454, "total": 29528.5054, "per_assigned": 7.38212635, "per_activated7": 38.7005313237, "per_sqa28": 57.3369036893 } }, "extraSalesMinutesPerAccount": 0.700625125, "incremental": { "costPerAssignedDelta": 0.732616375, "sqaDelta": 0.0295, "costPerIncrementalSqa": 24.8344533898, "costPerIncrementalSqaRange": { "fromSqaUpper": 16.8665463303, "fromSqaLower": 47.013558835 }, "uncertain": false, "note": "Point estimate with the range implied by the SQA interval." }, "note": "Invented allocations from the frozen protocol; not vendor prices, not measured charges." }, "decision": { "code": "stop", "status": "stop", "label": "Stop this version", "reasons": [ "activation upper bound −4.01 pts is below the −3.00 pts tolerated loss", "opt-out lower bound +1.37 pts is above the +1.00 pts tolerated increase", "+0.701 Sales minutes per assigned account exceeds the 0.5 cap" ], "nextStep": "Stop this version and keep every recorded reason. A changed play is a new, separately frozen experiment.", "allGates": { "sqa28": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +1.56 pts is above zero and the observed gain +2.95 pts meets +1.50 pts" }, "activation7": { "status": "stop", "code": "activation_harm", "reason": "activation upper bound −4.01 pts is below the −3.00 pts tolerated loss" }, "optout28": { "status": "stop", "code": "optout_harm", "reason": "opt-out lower bound +1.37 pts is above the +1.00 pts tolerated increase" }, "salesMinutesPerAssignedAccount": { "status": "stop", "code": "sales_cap_exceeded", "reason": "+0.701 Sales minutes per assigned account exceeds the 0.5 cap" } }, "scope": "synthetic_case_only" }, "cohorts": { "denominator": "all assigned accounts (non-converters stay at risk)", "atRisk": { "control": 4000, "treatment": 4000 }, "activation7": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 202, "treatment": 175, "controlRate": 0.0505, "treatmentRate": 0.04375 }, { "day": 4, "control": 397, "treatment": 333, "controlRate": 0.09925, "treatmentRate": 0.08325 }, { "day": 5, "control": 592, "treatment": 468, "controlRate": 0.148, "treatmentRate": 0.117 }, { "day": 6, "control": 795, "treatment": 618, "controlRate": 0.19875, "treatmentRate": 0.1545 }, { "day": 7, "control": 996, "treatment": 763, "controlRate": 0.249, "treatmentRate": 0.19075 } ], "sqa28": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 4, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 5, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 6, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 7, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 8, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 9, "control": 20, "treatment": 34, "controlRate": 0.005, "treatmentRate": 0.0085 }, { "day": 10, "control": 42, "treatment": 59, "controlRate": 0.0105, "treatmentRate": 0.01475 }, { "day": 11, "control": 62, "treatment": 79, "controlRate": 0.0155, "treatmentRate": 0.01975 }, { "day": 12, "control": 81, "treatment": 93, "controlRate": 0.02025, "treatmentRate": 0.02325 }, { "day": 13, "control": 106, "treatment": 118, "controlRate": 0.0265, "treatmentRate": 0.0295 }, { "day": 14, "control": 119, "treatment": 141, "controlRate": 0.02975, "treatmentRate": 0.03525 }, { "day": 15, "control": 142, "treatment": 161, "controlRate": 0.0355, "treatmentRate": 0.04025 }, { "day": 16, "control": 157, "treatment": 189, "controlRate": 0.03925, "treatmentRate": 0.04725 }, { "day": 17, "control": 180, "treatment": 219, "controlRate": 0.045, "treatmentRate": 0.05475 }, { "day": 18, "control": 197, "treatment": 244, "controlRate": 0.04925, "treatmentRate": 0.061 }, { "day": 19, "control": 217, "treatment": 272, "controlRate": 0.05425, "treatmentRate": 0.068 }, { "day": 20, "control": 233, "treatment": 300, "controlRate": 0.05825, "treatmentRate": 0.075 }, { "day": 21, "control": 258, "treatment": 316, "controlRate": 0.0645, "treatmentRate": 0.079 }, { "day": 22, "control": 273, "treatment": 340, "controlRate": 0.06825, "treatmentRate": 0.085 }, { "day": 23, "control": 296, "treatment": 369, "controlRate": 0.074, "treatmentRate": 0.09225 }, { "day": 24, "control": 314, "treatment": 399, "controlRate": 0.0785, "treatmentRate": 0.09975 }, { "day": 25, "control": 337, "treatment": 433, "controlRate": 0.08425, "treatmentRate": 0.10825 }, { "day": 26, "control": 365, "treatment": 460, "controlRate": 0.09125, "treatmentRate": 0.115 }, { "day": 27, "control": 381, "treatment": 488, "controlRate": 0.09525, "treatmentRate": 0.122 }, { "day": 28, "control": 397, "treatment": 515, "controlRate": 0.09925, "treatmentRate": 0.12875 } ] }, "funnel": { "denominator": "all assigned accounts in the arm", "note": "SQA is a parallel downstream endpoint, not nested under activation.", "stages": [ { "id": "assigned", "label": "Randomized eligible accounts", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted", "label": "Received ≥1 proactive contact", "control": 4000, "treatment": 4000, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted_twice", "label": "Received both contacts", "control": 3999, "treatment": 3992, "controlRate": 0.99975, "treatmentRate": 0.998 }, { "id": "imported7", "label": "Imported own data (7 days)", "control": 2449, "treatment": 2157, "controlRate": 0.61225, "treatmentRate": 0.53925 }, { "id": "first_query7", "label": "First successful query (7 days)", "control": 1828, "treatment": 1562, "controlRate": 0.457, "treatmentRate": 0.3905 }, { "id": "activated7", "label": "Activated: repeat query on a later UTC date (7 days)", "control": 996, "treatment": 763, "controlRate": 0.249, "treatmentRate": 0.19075 }, { "id": "shared7", "label": "Colleague queried (7 days, descriptive)", "control": 462, "treatment": 578, "controlRate": 0.1155, "treatmentRate": 0.1445 }, { "id": "sqa28", "label": "SQA within 28 days (parallel endpoint)", "control": 397, "treatment": 515, "controlRate": 0.09925, "treatmentRate": 0.12875 }, { "id": "optout28", "label": "Opted out within 28 days", "control": 36, "treatment": 114, "controlRate": 0.009, "treatmentRate": 0.0285 } ], "sqaWithoutActivation": { "control": 302, "treatment": 416 } }, "segments": [ { "dimension": "acquisition_source", "segment": "content", "kind": "prespecified_descriptive", "control": { "assigned": 1212, "sqa28": 121, "activation7": 314, "optout28": 15 }, "treatment": { "assigned": 1157, "sqa28": 140, "activation7": 213, "optout28": 33 }, "sqa28": { "delta": 0.0211676094, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0040874089, "upper": 0.0465976424 } } }, { "dimension": "acquisition_source", "segment": "direct", "kind": "prespecified_descriptive", "control": { "assigned": 802, "sqa28": 79, "activation7": 185, "optout28": 9 }, "treatment": { "assigned": 778, "sqa28": 106, "activation7": 157, "optout28": 25 }, "sqa28": { "delta": 0.037743046, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0059915599, "upper": 0.0697171416 } } }, { "dimension": "acquisition_source", "segment": "product_community", "kind": "prespecified_descriptive", "control": { "assigned": 1986, "sqa28": 197, "activation7": 497, "optout28": 12 }, "treatment": { "assigned": 2065, "sqa28": 269, "activation7": 393, "optout28": 56 }, "sqa28": { "delta": 0.0310719833, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0114394133, "upper": 0.0506829805 } } }, { "dimension": "company_size", "segment": "1-10", "kind": "prespecified_descriptive", "control": { "assigned": 2014, "sqa28": 208, "activation7": 515, "optout28": 20 }, "treatment": { "assigned": 2031, "sqa28": 256, "activation7": 405, "optout28": 53 }, "sqa28": { "delta": 0.022769222, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0031195168, "upper": 0.0424340079 } } }, { "dimension": "company_size", "segment": "11-50", "kind": "prespecified_descriptive", "control": { "assigned": 1385, "sqa28": 116, "activation7": 355, "optout28": 11 }, "treatment": { "assigned": 1368, "sqa28": 184, "activation7": 253, "optout28": 45 }, "sqa28": { "delta": 0.0507484113, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0275187375, "upper": 0.0741224949 } } }, { "dimension": "company_size", "segment": "51-200", "kind": "prespecified_descriptive", "control": { "assigned": 601, "sqa28": 73, "activation7": 126, "optout28": 5 }, "treatment": { "assigned": 601, "sqa28": 75, "activation7": 105, "optout28": 16 }, "sqa28": { "delta": 0.003327787, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0340084766, "upper": 0.0406728734 } } }, { "dimension": "task_need", "segment": "import", "kind": "prespecified_descriptive", "control": { "assigned": 1415, "sqa28": 128, "activation7": 342, "optout28": 13 }, "treatment": { "assigned": 1395, "sqa28": 191, "activation7": 282, "optout28": 33 }, "sqa28": { "delta": 0.0464581988, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0230287975, "upper": 0.0700063416 } } }, { "dimension": "task_need", "segment": "sharing", "kind": "prespecified_descriptive", "control": { "assigned": 1411, "sqa28": 145, "activation7": 351, "optout28": 10 }, "treatment": { "assigned": 1392, "sqa28": 178, "activation7": 269, "optout28": 45 }, "sqa28": { "delta": 0.0251095661, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.00144633, "upper": 0.0488460229 } } }, { "dimension": "task_need", "segment": "unknown", "kind": "prespecified_descriptive", "control": { "assigned": 1174, "sqa28": 124, "activation7": 303, "optout28": 13 }, "treatment": { "assigned": 1213, "sqa28": 146, "activation7": 212, "optout28": 36 }, "sqa28": { "delta": 0.0147409312, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0107517105, "upper": 0.040180206 } } } ], "exampleAccountTrace": { "selectionRule": "first treatment account by enrollment order with a documented SQA", "accountId": "account-000008", "arm": "treatment", "enrollmentIndex": 7, "blockIndex": 0, "token": "T4", "assignedAt": "2026-05-01T00:52:55.000Z", "attributes": { "acquisitionSource": "direct", "taskNeed": "sharing", "companySize": "1-10" }, "events": [ { "event_id": "harm:account-000008:trial_started", "event_ts": "2026-05-01T00:52:55.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000008-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "harm:account-000008:contact_sent:0", "event_ts": "2026-05-02T00:52:55.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000008-u1", "evidence_event_id": "harm:account-000008:trial_started", "sales_minutes": null }, { "event_id": "harm:account-000008:contact_sent:1", "event_ts": "2026-05-04T00:52:55.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000008-u1", "evidence_event_id": "harm:account-000008:trial_started", "sales_minutes": null }, { "event_id": "harm:account-000008:sqa_qualified:0", "event_ts": "2026-05-24T05:52:55.000Z", "elapsed_days": 23.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000008-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "harm:account-000008:sales_effort_logged:0", "event_ts": "2026-05-28T23:52:55.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 1.2111 } ] }, "receipts": [ "query:derive_account_outcomes", "query:integrity", "query:maturity", "query:metrics", "query:funnel", "query:cohort_curves", "query:segments", "query:costs", "query:trace" ] }, { "id": "insufficient_enrollment", "seed": 2026092914, "label": "Baseline volume cannot reach the planned sample", "description": "Use baseline 500/week volume; insufficient enrollment prevents advancement even if the effect looks promising.", "generativeAssumptions": { "controlRates": { "sqa28": 0.1, "activation7": 0.25, "optout28": 0.01, "sharing7": 0.12 }, "treatmentRates": { "sqa28": 0.13, "activation7": 0.28, "optout28": 0.01, "sharing7": 0.18 }, "salesMinutes": { "control": 0.6, "treatment": 0.9 }, "accounts": 3000, "enrollmentDays": 42, "note": "Inputs to the synthetic generator — not observed results and not the decision." }, "enrollment": { "control": 1500, "treatment": 1500, "total": 3000, "targetPerArm": 4000, "targetTotal": 8000, "plannedSampleReached": false, "firstAssignedAt": "2026-05-01T00:00:00.000Z", "lastAssignedAt": "2026-06-11T23:39:50.000Z", "enrollmentSpanDays": 41.986, "enrollmentCapDays": 42, "blocks": 300, "partialBlocks": 0, "timeline": "modeled: account i assigned at trialStartAt + floor(i·days·86400/accounts) s" }, "maturity": { "status": "mature", "analysisAt": "2026-07-09T23:39:50.000Z", "immatureAccounts": 0, "ascertainedAfterCutoff": 0, "rule": "final analysis at last assignment + 28 days" }, "integrity": { "status": "pass", "failures": [], "identicalEventRetriesCollapsed": 0, "checks": [ { "id": "ascertainment_after_cutoff_ignored", "violations": 0, "zeroTolerance": false }, { "id": "ascertainment_conflict", "violations": 0, "zeroTolerance": true }, { "id": "assignment_replay_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "completed_block_imbalance", "violations": 0, "zeroTolerance": true }, { "id": "contact_after_optout", "violations": 0, "zeroTolerance": true }, { "id": "contradictory_ascertainment_statuses", "violations": 0, "zeroTolerance": true }, { "id": "documented_nonreceipt", "violations": 0, "zeroTolerance": false }, { "id": "duplicate_account_in_denominator", "violations": 0, "zeroTolerance": true }, { "id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "zeroTolerance": true }, { "id": "event_for_unassigned_account", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_missing", "violations": 0, "zeroTolerance": true }, { "id": "exposure_log_unplanned", "violations": 0, "zeroTolerance": true }, { "id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "zeroTolerance": true }, { "id": "future_ascertainment_contradiction", "violations": 0, "zeroTolerance": true }, { "id": "future_evidence_in_routing", "violations": 0, "zeroTolerance": true }, { "id": "identical_event_retries_collapsed", "violations": 0, "zeroTolerance": false }, { "id": "ledger_reconciliation_mismatch", "violations": 0, "zeroTolerance": true }, { "id": "missing_required_outcome_ascertainment", "violations": 0, "zeroTolerance": true }, { "id": "more_than_two_proactive_contacts", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_ledger_incomplete", "violations": 0, "zeroTolerance": true }, { "id": "opportunity_status_inconsistent", "violations": 0, "zeroTolerance": true }, { "id": "outcome_before_assignment", "violations": 0, "zeroTolerance": true }, { "id": "partial_block_arm_difference_over_5", "violations": 0, "zeroTolerance": true }, { "id": "unresolved_cross_arm_company_collision", "violations": 0, "zeroTolerance": true } ] }, "metrics": { "sqa28": { "control": { "events": 166, "total": 1500, "rate": 0.1106666667, "confirmedZero": 1334, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 209, "total": 1500, "rate": 0.1393333333, "confirmedZero": 1291, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0286666667, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0049872724, "upper": 0.0523753578 }, "denominator": "all randomized eligible accounts (ITT)", "role": "primary", "gate": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +0.50 pts is above zero and the observed gain +2.87 pts meets +1.50 pts" } }, "activation7": { "control": { "events": 391, "total": 1500, "rate": 0.2606666667, "confirmedZero": 1109, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 410, "total": 1500, "rate": 0.2733333333, "confirmedZero": 1090, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.0126666667, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0189896196, "upper": 0.0442898621 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound −1.90 pts is above −3.00 pts" } }, "optout28": { "control": { "events": 18, "total": 1500, "rate": 0.012, "confirmedZero": 1482, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 14, "total": 1500, "rate": 0.0093333333, "confirmedZero": 1486, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": -0.0026666667, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0105178094, "upper": 0.0049927943 }, "denominator": "all randomized eligible accounts (ITT)", "role": "noninferiority_guardrail", "gate": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.50 pts is below +1.00 pts" } }, "sharing7": { "control": { "events": 175, "total": 1500, "rate": 0.1166666667, "confirmedZero": 1325, "missing": 0, "pending": 0, "conflicts": 0 }, "treatment": { "events": 277, "total": 1500, "rate": 0.1846666667, "confirmedZero": 1223, "missing": 0, "pending": 0, "conflicts": 0 }, "delta": 0.068, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.042479747, "upper": 0.0935119387 }, "denominator": "all randomized eligible accounts (ITT)", "role": "descriptive_secondary", "gate": { "status": "descriptive_only", "code": "not_a_gate", "reason": "Descriptive secondary; never a success gate." } }, "salesMinutesPerAssignedAccount": { "control": { "minutes": 900.7931, "total": 1500, "mean": 0.6005287333 }, "treatment": { "minutes": 1349.0462, "total": 1500, "mean": 0.8993641333 }, "delta": 0.2988354, "cap": 0.5, "role": "operating_cap", "gate": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.299 Sales minutes per assigned account is within the 0.5 cap" } } }, "costs": { "dataKind": "assumed_synthetic_costs", "currency": "USD", "arms": { "control": { "assigned": 1500, "contacts": 2998, "sales_minutes": 900.7931, "acquisition": 9000.0, "platform_allocation": 15.0, "delivery": 14.99, "content_ops": 120.0, "inference": 0.0, "sales_labor": 900.7931, "total": 10050.7831, "per_assigned": 6.7005220667, "per_activated7": 25.7053276215, "per_sqa28": 60.5468861446 }, "treatment": { "assigned": 1500, "contacts": 2999, "sales_minutes": 1349.0462, "acquisition": 9000.0, "platform_allocation": 15.0, "delivery": 14.995, "content_ops": 240.0, "inference": 3.0, "sales_labor": 1349.0462, "total": 10622.0412, "per_assigned": 7.0813608, "per_activated7": 25.907417561, "per_sqa28": 50.8231636364 } }, "extraSalesMinutesPerAccount": 0.2988354, "incremental": { "costPerAssignedDelta": 0.3808387333, "sqaDelta": 0.0286666667, "costPerIncrementalSqa": 13.285072093, "costPerIncrementalSqaRange": { "fromSqaUpper": 7.2713342575, "fromSqaLower": 76.3621274675 }, "uncertain": false, "note": "Point estimate with the range implied by the SQA interval." }, "note": "Invented allocations from the frozen protocol; not vendor prices, not measured charges." }, "decision": { "code": "inconclusive_insufficient_enrollment", "status": "inconclusive_insufficient_enrollment", "label": "Inconclusive — planned sample not enrolled", "reasons": [ "3,000 accounts were assigned by the 42-day cap, below the planned 8,000; no stop condition applied", "SQA lower bound +0.50 pts is above zero and the observed gain +2.87 pts meets +1.50 pts", "activation lower bound −1.90 pts is above −3.00 pts", "opt-out upper bound +0.50 pts is below +1.00 pts", "+0.299 Sales minutes per assigned account is within the 0.5 cap" ], "nextStep": "Do not expand. Available volume could not supply the planned sample; propose a feasibility study with a weaker claim, or a different test.", "allGates": { "sqa28": { "status": "pass", "code": "sqa_pass", "reason": "SQA lower bound +0.50 pts is above zero and the observed gain +2.87 pts meets +1.50 pts" }, "activation7": { "status": "pass", "code": "activation_pass", "reason": "activation lower bound −1.90 pts is above −3.00 pts" }, "optout28": { "status": "pass", "code": "optout_pass", "reason": "opt-out upper bound +0.50 pts is below +1.00 pts" }, "salesMinutesPerAssignedAccount": { "status": "pass", "code": "sales_cap_pass", "reason": "+0.299 Sales minutes per assigned account is within the 0.5 cap" } }, "scope": "synthetic_case_only" }, "cohorts": { "denominator": "all assigned accounts (non-converters stay at risk)", "atRisk": { "control": 1500, "treatment": 1500 }, "activation7": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 83, "treatment": 68, "controlRate": 0.0553333333, "treatmentRate": 0.0453333333 }, { "day": 4, "control": 158, "treatment": 152, "controlRate": 0.1053333333, "treatmentRate": 0.1013333333 }, { "day": 5, "control": 234, "treatment": 247, "controlRate": 0.156, "treatmentRate": 0.1646666667 }, { "day": 6, "control": 321, "treatment": 326, "controlRate": 0.214, "treatmentRate": 0.2173333333 }, { "day": 7, "control": 391, "treatment": 410, "controlRate": 0.2606666667, "treatmentRate": 0.2733333333 } ], "sqa28": [ { "day": 0, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 1, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 2, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 3, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 4, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 5, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 6, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 7, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 8, "control": 0, "treatment": 0, "controlRate": 0.0, "treatmentRate": 0.0 }, { "day": 9, "control": 7, "treatment": 13, "controlRate": 0.0046666667, "treatmentRate": 0.0086666667 }, { "day": 10, "control": 18, "treatment": 25, "controlRate": 0.012, "treatmentRate": 0.0166666667 }, { "day": 11, "control": 31, "treatment": 38, "controlRate": 0.0206666667, "treatmentRate": 0.0253333333 }, { "day": 12, "control": 37, "treatment": 50, "controlRate": 0.0246666667, "treatmentRate": 0.0333333333 }, { "day": 13, "control": 45, "treatment": 65, "controlRate": 0.03, "treatmentRate": 0.0433333333 }, { "day": 14, "control": 54, "treatment": 76, "controlRate": 0.036, "treatmentRate": 0.0506666667 }, { "day": 15, "control": 64, "treatment": 84, "controlRate": 0.0426666667, "treatmentRate": 0.056 }, { "day": 16, "control": 72, "treatment": 92, "controlRate": 0.048, "treatmentRate": 0.0613333333 }, { "day": 17, "control": 76, "treatment": 102, "controlRate": 0.0506666667, "treatmentRate": 0.068 }, { "day": 18, "control": 86, "treatment": 112, "controlRate": 0.0573333333, "treatmentRate": 0.0746666667 }, { "day": 19, "control": 95, "treatment": 126, "controlRate": 0.0633333333, "treatmentRate": 0.084 }, { "day": 20, "control": 103, "treatment": 137, "controlRate": 0.0686666667, "treatmentRate": 0.0913333333 }, { "day": 21, "control": 107, "treatment": 146, "controlRate": 0.0713333333, "treatmentRate": 0.0973333333 }, { "day": 22, "control": 114, "treatment": 155, "controlRate": 0.076, "treatmentRate": 0.1033333333 }, { "day": 23, "control": 123, "treatment": 167, "controlRate": 0.082, "treatmentRate": 0.1113333333 }, { "day": 24, "control": 127, "treatment": 177, "controlRate": 0.0846666667, "treatmentRate": 0.118 }, { "day": 25, "control": 139, "treatment": 181, "controlRate": 0.0926666667, "treatmentRate": 0.1206666667 }, { "day": 26, "control": 149, "treatment": 192, "controlRate": 0.0993333333, "treatmentRate": 0.128 }, { "day": 27, "control": 157, "treatment": 198, "controlRate": 0.1046666667, "treatmentRate": 0.132 }, { "day": 28, "control": 166, "treatment": 209, "controlRate": 0.1106666667, "treatmentRate": 0.1393333333 } ] }, "funnel": { "denominator": "all assigned accounts in the arm", "note": "SQA is a parallel downstream endpoint, not nested under activation.", "stages": [ { "id": "assigned", "label": "Randomized eligible accounts", "control": 1500, "treatment": 1500, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted", "label": "Received ≥1 proactive contact", "control": 1500, "treatment": 1500, "controlRate": 1.0, "treatmentRate": 1.0 }, { "id": "contacted_twice", "label": "Received both contacts", "control": 1498, "treatment": 1499, "controlRate": 0.9986666667, "treatmentRate": 0.9993333333 }, { "id": "imported7", "label": "Imported own data (7 days)", "control": 918, "treatment": 945, "controlRate": 0.612, "treatmentRate": 0.63 }, { "id": "first_query7", "label": "First successful query (7 days)", "control": 694, "treatment": 697, "controlRate": 0.4626666667, "treatmentRate": 0.4646666667 }, { "id": "activated7", "label": "Activated: repeat query on a later UTC date (7 days)", "control": 391, "treatment": 410, "controlRate": 0.2606666667, "treatmentRate": 0.2733333333 }, { "id": "shared7", "label": "Colleague queried (7 days, descriptive)", "control": 175, "treatment": 277, "controlRate": 0.1166666667, "treatmentRate": 0.1846666667 }, { "id": "sqa28", "label": "SQA within 28 days (parallel endpoint)", "control": 166, "treatment": 209, "controlRate": 0.1106666667, "treatmentRate": 0.1393333333 }, { "id": "optout28", "label": "Opted out within 28 days", "control": 18, "treatment": 14, "controlRate": 0.012, "treatmentRate": 0.0093333333 } ], "sqaWithoutActivation": { "control": 129, "treatment": 152 } }, "segments": [ { "dimension": "acquisition_source", "segment": "content", "kind": "prespecified_descriptive", "control": { "assigned": 459, "sqa28": 59, "activation7": 131, "optout28": 5 }, "treatment": { "assigned": 446, "sqa28": 61, "activation7": 117, "optout28": 3 }, "sqa28": { "delta": 0.0082309954, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0361226253, "upper": 0.0527859609 } } }, { "dimension": "acquisition_source", "segment": "direct", "kind": "prespecified_descriptive", "control": { "assigned": 268, "sqa28": 31, "activation7": 62, "optout28": 1 }, "treatment": { "assigned": 322, "sqa28": 42, "activation7": 84, "optout28": 2 }, "sqa28": { "delta": 0.0147631408, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.03978871, "upper": 0.0675207309 } } }, { "dimension": "acquisition_source", "segment": "product_community", "kind": "prespecified_descriptive", "control": { "assigned": 773, "sqa28": 76, "activation7": 198, "optout28": 12 }, "treatment": { "assigned": 732, "sqa28": 106, "activation7": 209, "optout28": 9 }, "sqa28": { "delta": 0.0464905025, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0134957355, "upper": 0.0798168506 } } }, { "dimension": "company_size", "segment": "1-10", "kind": "prespecified_descriptive", "control": { "assigned": 756, "sqa28": 80, "activation7": 196, "optout28": 11 }, "treatment": { "assigned": 765, "sqa28": 98, "activation7": 202, "optout28": 9 }, "sqa28": { "delta": 0.0222844693, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0101309767, "upper": 0.0547162369 } } }, { "dimension": "company_size", "segment": "11-50", "kind": "prespecified_descriptive", "control": { "assigned": 524, "sqa28": 56, "activation7": 131, "optout28": 6 }, "treatment": { "assigned": 512, "sqa28": 74, "activation7": 136, "optout28": 3 }, "sqa28": { "delta": 0.037661021, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0027893194, "upper": 0.0783470597 } } }, { "dimension": "company_size", "segment": "51-200", "kind": "prespecified_descriptive", "control": { "assigned": 220, "sqa28": 30, "activation7": 64, "optout28": 1 }, "treatment": { "assigned": 223, "sqa28": 37, "activation7": 72, "optout28": 2 }, "sqa28": { "delta": 0.0295556461, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0376868952, "upper": 0.0965748349 } } }, { "dimension": "task_need", "segment": "import", "kind": "prespecified_descriptive", "control": { "assigned": 518, "sqa28": 50, "activation7": 129, "optout28": 6 }, "treatment": { "assigned": 532, "sqa28": 76, "activation7": 139, "optout28": 4 }, "sqa28": { "delta": 0.0463320463, "ci": { "method": "newcombe", "level": 0.95, "lower": 0.0069643284, "upper": 0.085718687 } } }, { "dimension": "task_need", "segment": "sharing", "kind": "prespecified_descriptive", "control": { "assigned": 539, "sqa28": 64, "activation7": 152, "optout28": 9 }, "treatment": { "assigned": 524, "sqa28": 69, "activation7": 136, "optout28": 4 }, "sqa28": { "delta": 0.0129409849, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0269813265, "upper": 0.0530571289 } } }, { "dimension": "task_need", "segment": "unknown", "kind": "prespecified_descriptive", "control": { "assigned": 443, "sqa28": 52, "activation7": 110, "optout28": 3 }, "treatment": { "assigned": 444, "sqa28": 64, "activation7": 135, "optout28": 6 }, "sqa28": { "delta": 0.0267626543, "ci": { "method": "newcombe", "level": 0.95, "lower": -0.0178154221, "upper": 0.071384242 } } } ], "exampleAccountTrace": { "selectionRule": "first treatment account by enrollment order with a documented SQA", "accountId": "account-000006", "arm": "treatment", "enrollmentIndex": 5, "blockIndex": 0, "token": "T2", "assignedAt": "2026-05-01T01:40:48.000Z", "attributes": { "acquisitionSource": "product_community", "taskNeed": "sharing", "companySize": "1-10" }, "events": [ { "event_id": "insufficient_enrollment:account-000006:trial_started", "event_ts": "2026-05-01T01:40:48.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:contact_sent:0", "event_ts": "2026-05-02T01:40:48.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000006-u1", "evidence_event_id": "insufficient_enrollment:account-000006:trial_started", "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:data_imported:0", "event_ts": "2026-05-02T03:40:48.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:own_data_query:0", "event_ts": "2026-05-02T04:40:48.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:own_data_query:1", "event_ts": "2026-05-03T04:40:48.000Z", "elapsed_days": 2.125, "event_type": "own_data_query", "detail": "repeat", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:contact_sent:1", "event_ts": "2026-05-04T01:40:48.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000006-u1", "evidence_event_id": "insufficient_enrollment:account-000006:trial_started", "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:sqa_qualified:0", "event_ts": "2026-05-14T06:40:48.000Z", "elapsed_days": 13.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "event_id": "insufficient_enrollment:account-000006:sales_effort_logged:0", "event_ts": "2026-05-29T00:40:48.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9213 } ] }, "receipts": [ "query:derive_account_outcomes", "query:integrity", "query:maturity", "query:metrics", "query:funnel", "query:cohort_curves", "query:segments", "query:costs", "query:trace" ] } ], "operationalFixtures": { "status": "not_in_this_artifact", "owner": "root", "records": [], "note": "Real-company Clay/test-queue fixtures are published separately with their own receipts and never enter these denominators." }, "queries": [ { "id": "derive_account_outcomes", "sqlFile": "scripts/duck-yeah/sql/experiment_derive_account_outcomes.sql", "sql": "-- Duck Yeah v2 · derive one row per account from the raw synthetic rows.\n-- Runs on the engine (MotherDuck for the published run). Every published\n-- rate is aggregated from this table, which is itself built only from\n-- accounts / assignments / users / events / ascertainment.\n--\n-- Windows: elapsed time since assignment, event_ts >= assigned_at AND\n-- event_ts < assigned_at + N days (protocol windows.outcomeWindow).\n-- Identical event-ID retries collapse via DISTINCT (idempotent); conflicting\n-- payloads are an integrity failure reported by experiment_integrity.sql.\n--\n-- Ascertainment is read AS OF the analysis cutoff: records dated after the\n-- cutoff are ignored for status (counted, and flagged if they contradict).\n-- Status per account × gated outcome:\n-- pending window not closed at the cutoff\n-- conflict contradictory records (observed AND confirmed_absent), an\n-- observed record with no qualifying event, an absence record\n-- despite a qualifying event, an absence confirmed before the\n-- window closed, or an unknown status value\n-- event qualifying event AND an 'observed' record\n-- zero no qualifying event AND a timely 'confirmed_absent' record\n-- missing anything else (no record by the cutoff)\n-- SQA qualifies only with exactly the three positive structured tokens\n-- company_fit;team_use_case;next_step_agreed — no substring matching.\nCREATE OR REPLACE TABLE duck_yeah_experiment.account_outcomes AS\nWITH acct AS (\n SELECT a.dataset, a.account_id, a.assigned_at, a.acquisition_source, a.task_need, a.company_size,\n coalesce(s.arm, 'historical') AS arm, s.enrollment_index\n FROM duck_yeah_experiment.accounts a\n LEFT JOIN duck_yeah_experiment.assignments s USING (dataset, account_id)\n),\ncutoff AS ( -- protocol windows.finalAnalysisAt: last assignment + 28 days\n SELECT dataset, max(assigned_at) + INTERVAL 28 DAY AS analysis_at FROM acct GROUP BY dataset\n),\nev AS (\n SELECT DISTINCT e.dataset, e.event_id, e.account_id, e.user_id, e.event_ts, e.event_type, e.detail,\n e.evidence_event_id, e.sales_minutes, a.assigned_at\n FROM duck_yeah_experiment.events e\n JOIN acct a USING (dataset, account_id)\n),\nimp AS (\n SELECT dataset, account_id, min(event_ts) AS imported_at\n FROM ev WHERE event_type = 'data_imported'\n AND event_ts >= assigned_at AND event_ts < assigned_at + INTERVAL 7 DAY\n GROUP BY ALL\n),\nq AS ( -- successful queries against the account's own imported dataset, inside 7 days\n SELECT ev.dataset, ev.account_id, ev.event_ts, CAST(ev.event_ts AS DATE) AS utc_date\n FROM ev JOIN imp USING (dataset, account_id)\n WHERE ev.event_type = 'own_data_query' AND ev.event_ts >= imp.imported_at\n AND ev.event_ts >= ev.assigned_at AND ev.event_ts < ev.assigned_at + INTERVAL 7 DAY\n),\nq_first AS (SELECT dataset, account_id, min(event_ts) AS first_query_at, min(utc_date) AS first_date FROM q GROUP BY ALL),\nact AS ( -- activation = first successful query on a separate, later UTC date\n SELECT q.dataset, q.account_id, min(q.event_ts) AS activated_at\n FROM q JOIN q_first USING (dataset, account_id)\n WHERE q.utc_date > q_first.first_date\n GROUP BY ALL\n),\nshr AS ( -- sharing7: a linked colleague's successful query inside 7 days\n SELECT ev.dataset, ev.account_id, min(ev.event_ts) AS shared_at\n FROM ev JOIN duck_yeah_experiment.users u USING (dataset, account_id, user_id)\n WHERE ev.event_type = 'colleague_query' AND u.role = 'linked_colleague'\n AND ev.event_ts >= ev.assigned_at AND ev.event_ts < ev.assigned_at + INTERVAL 7 DAY\n GROUP BY ALL\n),\nsqa AS ( -- exactly the three positive structured criteria, inside 28 days\n SELECT dataset, account_id, min(event_ts) AS sqa_at\n FROM ev WHERE event_type = 'sqa_qualified' AND detail IS NOT NULL\n AND list_sort(list_distinct(string_split(detail, ';'))) = ['company_fit', 'next_step_agreed', 'team_use_case']\n AND event_ts >= assigned_at AND event_ts < assigned_at + INTERVAL 28 DAY\n GROUP BY ALL\n),\nopt AS (\n SELECT dataset, account_id, min(event_ts) AS optout_at\n FROM ev WHERE event_type = 'contact_opt_out'\n AND event_ts >= assigned_at AND event_ts < assigned_at + INTERVAL 28 DAY\n GROUP BY ALL\n),\ncon AS (\n SELECT dataset, account_id, count(*) AS contacts\n FROM ev WHERE event_type = 'contact_sent' GROUP BY ALL\n),\nsales AS (\n SELECT dataset, account_id, sum(sales_minutes) AS sales_minutes\n FROM ev WHERE event_type = 'sales_effort_logged' AND sales_minutes IS NOT NULL GROUP BY ALL\n),\nj AS (\n SELECT acct.*, c.analysis_at, imp.imported_at, q_first.first_query_at, act.activated_at, shr.shared_at,\n sqa.sqa_at, opt.optout_at, coalesce(con.contacts, 0) AS contacts, sales.sales_minutes,\n acct.assigned_at + INTERVAL 28 DAY > c.analysis_at AS immature\n FROM acct\n JOIN cutoff c USING (dataset)\n LEFT JOIN imp USING (dataset, account_id)\n LEFT JOIN q_first USING (dataset, account_id)\n LEFT JOIN act USING (dataset, account_id)\n LEFT JOIN shr USING (dataset, account_id)\n LEFT JOIN sqa USING (dataset, account_id)\n LEFT JOIN opt USING (dataset, account_id)\n LEFT JOIN con USING (dataset, account_id)\n LEFT JOIN sales USING (dataset, account_id)\n),\noutcomes AS (\n SELECT * FROM (VALUES ('sqa28', 28), ('activation7', 7), ('optout28', 28), ('sharing7', 7)) t(outcome, window_days)\n),\nasc_ AS (\n SELECT x.dataset, x.account_id, x.outcome,\n count(*) FILTER (WHERE x.recorded_at <= j.analysis_at AND x.status = 'observed') AS obs_in,\n count(*) FILTER (WHERE x.recorded_at <= j.analysis_at AND x.status = 'confirmed_absent'\n AND x.recorded_at >= j.assigned_at + to_days(o.window_days)) AS abs_in,\n count(*) FILTER (WHERE x.recorded_at <= j.analysis_at AND x.status = 'confirmed_absent'\n AND x.recorded_at < j.assigned_at + to_days(o.window_days)) AS abs_premature,\n count(*) FILTER (WHERE x.recorded_at <= j.analysis_at\n AND (x.status IS NULL OR x.status NOT IN ('observed', 'confirmed_absent'))) AS other_in,\n count(*) FILTER (WHERE x.recorded_at > j.analysis_at) AS after_cutoff,\n count(*) FILTER (WHERE x.recorded_at > j.analysis_at AND x.status = 'observed') AS obs_after,\n count(*) FILTER (WHERE x.recorded_at > j.analysis_at AND x.status = 'confirmed_absent') AS abs_after\n FROM duck_yeah_experiment.ascertainment x\n JOIN j USING (dataset, account_id)\n JOIN outcomes o USING (outcome)\n GROUP BY ALL\n),\nlong AS (\n SELECT j.dataset, j.account_id, o.outcome, j.immature,\n CASE o.outcome WHEN 'sqa28' THEN j.sqa_at IS NOT NULL WHEN 'activation7' THEN j.activated_at IS NOT NULL\n WHEN 'optout28' THEN j.optout_at IS NOT NULL ELSE j.shared_at IS NOT NULL END AS has_event,\n coalesce(a.obs_in, 0) AS obs_in, coalesce(a.abs_in, 0) AS abs_in, coalesce(a.abs_premature, 0) AS abs_premature,\n coalesce(a.other_in, 0) AS other_in, coalesce(a.after_cutoff, 0) AS after_cutoff,\n coalesce(a.obs_after, 0) AS obs_after, coalesce(a.abs_after, 0) AS abs_after\n FROM j CROSS JOIN outcomes o\n LEFT JOIN asc_ a ON a.dataset = j.dataset AND a.account_id = j.account_id AND a.outcome = o.outcome\n),\nresolved AS (\n SELECT *,\n CASE WHEN immature THEN 'pending'\n WHEN other_in > 0 OR abs_premature > 0 OR (obs_in > 0 AND abs_in > 0)\n OR (has_event AND abs_in > 0) OR (NOT has_event AND obs_in > 0) THEN 'conflict'\n WHEN has_event AND obs_in > 0 THEN 'event'\n WHEN NOT has_event AND abs_in > 0 THEN 'zero'\n ELSE 'missing' END AS status,\n -- a post-cutoff record contradicting the evidence (or itself)\n (obs_after > 0 AND abs_after > 0) OR (has_event AND abs_after > 0) OR (NOT has_event AND obs_after > 0)\n OR (obs_after > 0 AND abs_in > 0) OR (abs_after > 0 AND obs_in > 0) AS future_contradiction\n FROM long\n),\npiv AS (\n SELECT dataset, account_id,\n max(CASE WHEN outcome = 'sqa28' THEN status END) AS sqa28_status,\n max(CASE WHEN outcome = 'activation7' THEN status END) AS activation7_status,\n max(CASE WHEN outcome = 'optout28' THEN status END) AS optout28_status,\n max(CASE WHEN outcome = 'sharing7' THEN status END) AS sharing7_status,\n sum(after_cutoff) AS asc_after_cutoff,\n count(*) FILTER (WHERE future_contradiction) AS asc_future_contradictions,\n count(*) FILTER (WHERE obs_in > 0 AND abs_in + abs_premature > 0) AS asc_contradictory_statuses\n FROM resolved GROUP BY ALL\n)\nSELECT j.*, piv.sqa28_status, piv.activation7_status, piv.optout28_status, piv.sharing7_status,\n piv.asc_after_cutoff, piv.asc_future_contradictions, piv.asc_contradictory_statuses\nFROM j JOIN piv USING (dataset, account_id);\n", "sha256": "35c8e0dfcefd89e2b74b296a6645c3545b48b9c62573f83eb48c29062da90d20", "elapsedMs": 791.24, "rowCount": 1, "columns": [ "rows" ], "rows": null, "rowsPublishedIn": "not published (derived table)" }, { "id": "integrity", "sqlFile": "scripts/duck-yeah/sql/experiment_integrity.sql", "sql": "-- Duck Yeah v2 · integrity checks against the complete assignment ledger.\n-- One row per scenario × check; `violations` > 0 on a zero-tolerance check\n-- blocks a definitive decision. `informational` rows never block.\n-- $assignment_seed is the protocol's frozen assignment seed.\nWITH asg AS (\n SELECT s.*, a.assigned_at, a.company_domain\n FROM duck_yeah_experiment.assignments s\n JOIN duck_yeah_experiment.accounts a USING (dataset, account_id)\n),\nscen AS (SELECT DISTINCT dataset FROM duck_yeah_experiment.assignments),\ntokens AS (SELECT unnest(['C0','C1','C2','C3','C4','T0','T1','T2','T3','T4']) AS token),\nblocks AS (SELECT DISTINCT dataset, block_index FROM asg),\nreplay AS ( -- protocol sha256_shuffled_blocks_v1, recomputed on the engine\n SELECT b.dataset, b.block_index, t.token,\n row_number() OVER (PARTITION BY b.dataset, b.block_index\n ORDER BY sha256(CAST($assignment_seed AS VARCHAR) || '|' || b.dataset || '|' ||\n CAST(b.block_index AS VARCHAR) || '|' || t.token), t.token) - 1 AS block_position\n FROM blocks b CROSS JOIN tokens t\n),\narrival AS (\n SELECT dataset, account_id,\n row_number() OVER (PARTITION BY dataset ORDER BY assigned_at, account_id) - 1 AS expected_index\n FROM asg\n),\nev AS (SELECT e.*, a.assigned_at FROM duck_yeah_experiment.events e\n LEFT JOIN duck_yeah_experiment.accounts a USING (dataset, account_id)),\nchecks AS (\n SELECT dataset, 'assignment_replay_mismatch' AS check_id, count(*) AS violations\n FROM asg LEFT JOIN replay r USING (dataset, block_index, block_position)\n JOIN arrival USING (dataset, account_id)\n WHERE r.token IS DISTINCT FROM asg.token\n OR asg.arm <> CASE WHEN asg.token LIKE 'C%' THEN 'control' ELSE 'treatment' END\n OR asg.enrollment_index <> arrival.expected_index\n OR asg.block_index <> asg.enrollment_index // 10\n OR asg.block_position <> asg.enrollment_index % 10\n GROUP BY ALL\n UNION ALL\n SELECT dataset, 'completed_block_imbalance', count(*) FROM (\n SELECT dataset, block_index, count(*) AS n, count(*) FILTER (WHERE arm = 'control') AS c\n FROM asg GROUP BY ALL) WHERE n = 10 AND c <> 5 GROUP BY ALL\n UNION ALL\n SELECT dataset, 'partial_block_arm_difference_over_5', count(*) FROM (\n SELECT dataset, abs(count(*) FILTER (WHERE arm = 'control') - count(*) FILTER (WHERE arm = 'treatment')) AS d\n FROM asg GROUP BY ALL) WHERE d > 5 GROUP BY ALL\n UNION ALL\n SELECT dataset, 'duplicate_account_in_denominator', count(*) FROM (\n SELECT dataset, account_id FROM duck_yeah_experiment.assignments GROUP BY ALL HAVING count(*) > 1) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'unresolved_cross_arm_company_collision', count(*) FROM (\n SELECT dataset, company_domain FROM asg GROUP BY ALL HAVING count(DISTINCT arm) > 1) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'duplicate_event_id_with_conflicting_payload', count(*) FROM (\n SELECT dataset, event_id FROM duck_yeah_experiment.events GROUP BY ALL\n HAVING count(DISTINCT (account_id, user_id, event_ts, event_type, detail, evidence_event_id, sales_minutes)) > 1) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'outcome_before_assignment', count(*) FROM ev\n WHERE event_ts < assigned_at AND dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT c.dataset, 'future_evidence_in_routing', count(*)\n FROM duck_yeah_experiment.events c\n LEFT JOIN duck_yeah_experiment.events e ON e.dataset = c.dataset AND e.event_id = c.evidence_event_id\n WHERE c.event_type = 'contact_sent' AND c.dataset IN (SELECT dataset FROM scen)\n AND (e.event_id IS NULL OR e.event_ts > c.event_ts OR e.account_id <> c.account_id)\n GROUP BY ALL\n UNION ALL\n SELECT dataset, 'missing_required_outcome_ascertainment', count(*) FROM (\n SELECT dataset, account_id, 'sqa28' AS s, sqa28_status AS st FROM duck_yeah_experiment.account_outcomes\n UNION ALL SELECT dataset, account_id, 'activation7', activation7_status FROM duck_yeah_experiment.account_outcomes\n UNION ALL SELECT dataset, account_id, 'optout28', optout28_status FROM duck_yeah_experiment.account_outcomes)\n WHERE st = 'missing' AND dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'ascertainment_conflict', count(*) FROM (\n SELECT dataset, account_id, sqa28_status AS st FROM duck_yeah_experiment.account_outcomes\n UNION ALL SELECT dataset, account_id, activation7_status FROM duck_yeah_experiment.account_outcomes\n UNION ALL SELECT dataset, account_id, optout28_status FROM duck_yeah_experiment.account_outcomes)\n WHERE st = 'conflict' AND dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'contradictory_ascertainment_statuses', sum(asc_contradictory_statuses)\n FROM duck_yeah_experiment.account_outcomes WHERE dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'future_ascertainment_contradiction', sum(asc_future_contradictions)\n FROM duck_yeah_experiment.account_outcomes WHERE dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'ascertainment_after_cutoff_ignored', sum(asc_after_cutoff)\n FROM duck_yeah_experiment.account_outcomes WHERE dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL -- every assigned account has exactly the two planned opportunities (days 1 and 3)\n SELECT s.dataset, 'opportunity_ledger_incomplete', count(*)\n FROM duck_yeah_experiment.assignments s\n LEFT JOIN (SELECT dataset, account_id, count(*) AS n, count(DISTINCT opportunity_day) AS d,\n bool_and(opportunity_day IN (1, 3)) AS days_ok\n FROM duck_yeah_experiment.contact_opportunities GROUP BY ALL) o USING (dataset, account_id)\n WHERE o.n IS NULL OR o.n <> 2 OR o.d <> 2 OR NOT o.days_ok\n GROUP BY ALL\n UNION ALL -- a 'sent' opportunity whose contact log is absent\n SELECT o.dataset, 'exposure_log_missing', count(*)\n FROM duck_yeah_experiment.contact_opportunities o\n LEFT JOIN (SELECT DISTINCT dataset, event_id, account_id FROM duck_yeah_experiment.events WHERE event_type = 'contact_sent') e\n ON e.dataset = o.dataset AND e.event_id = o.contact_event_id AND e.account_id = o.account_id\n WHERE o.status = 'sent' AND e.event_id IS NULL AND o.dataset IN (SELECT dataset FROM scen)\n GROUP BY ALL\n UNION ALL -- a contact log not backed by a 'sent' opportunity\n SELECT e.dataset, 'exposure_log_unplanned', count(*)\n FROM (SELECT DISTINCT dataset, event_id, account_id FROM duck_yeah_experiment.events WHERE event_type = 'contact_sent') e\n LEFT JOIN duck_yeah_experiment.contact_opportunities o\n ON o.dataset = e.dataset AND o.contact_event_id = e.event_id AND o.account_id = e.account_id AND o.status = 'sent'\n WHERE o.opportunity_id IS NULL AND e.dataset IN (SELECT dataset FROM scen)\n GROUP BY ALL\n UNION ALL -- ledger status must agree with opt-out timing; unknown statuses are invalid\n SELECT o.dataset, 'opportunity_status_inconsistent', count(*)\n FROM duck_yeah_experiment.contact_opportunities o\n LEFT JOIN (SELECT dataset, account_id, min(event_ts) AS opt_at FROM duck_yeah_experiment.events\n WHERE event_type = 'contact_opt_out' GROUP BY ALL) x USING (dataset, account_id)\n WHERE o.dataset IN (SELECT dataset FROM scen)\n AND (o.status NOT IN ('sent', 'suppressed_after_optout', 'documented_nonreceipt')\n OR (o.status = 'suppressed_after_optout' AND (x.opt_at IS NULL OR x.opt_at > o.planned_at))\n OR (o.status = 'sent' AND x.opt_at IS NOT NULL AND x.opt_at <= o.planned_at)\n OR (o.status <> 'sent' AND o.contact_event_id IS NOT NULL))\n GROUP BY ALL\n UNION ALL\n SELECT dataset, 'documented_nonreceipt', count(*) FROM duck_yeah_experiment.contact_opportunities\n WHERE status = 'documented_nonreceipt' AND dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT c.dataset, 'contact_after_optout', count(*)\n FROM duck_yeah_experiment.events c\n JOIN duck_yeah_experiment.events o ON o.dataset = c.dataset AND o.account_id = c.account_id\n AND o.event_type = 'contact_opt_out' AND c.event_ts > o.event_ts\n WHERE c.event_type = 'contact_sent' AND c.dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT dataset, 'more_than_two_proactive_contacts', count(*) FROM (\n SELECT dataset, account_id FROM (SELECT DISTINCT * FROM duck_yeah_experiment.events)\n WHERE event_type = 'contact_sent' GROUP BY ALL HAVING count(*) > 2)\n WHERE dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n UNION ALL\n SELECT a.dataset, 'fixture_or_visitor_record_in_modeled_cohort', count(*)\n FROM duck_yeah_experiment.accounts a\n WHERE a.dataset IN (SELECT dataset FROM scen)\n AND (NOT regexp_full_match(a.account_id, 'account-[0-9]{6}') OR a.company_domain NOT LIKE '%.test')\n GROUP BY ALL\n UNION ALL\n SELECT dataset, 'ledger_reconciliation_mismatch', count(*)\n FROM duck_yeah_experiment.accounts a\n FULL OUTER JOIN duck_yeah_experiment.assignments s USING (dataset, account_id)\n WHERE (s.account_id IS NULL AND dataset IN (SELECT dataset FROM scen)) OR a.account_id IS NULL\n GROUP BY ALL\n UNION ALL\n SELECT e.dataset, 'event_for_unassigned_account', count(*)\n FROM duck_yeah_experiment.events e\n LEFT JOIN duck_yeah_experiment.assignments s USING (dataset, account_id)\n WHERE e.dataset IN (SELECT dataset FROM scen) AND s.account_id IS NULL GROUP BY ALL\n UNION ALL\n SELECT dataset, 'identical_event_retries_collapsed',\n count(*) - count(DISTINCT (event_id, account_id, user_id, event_ts, event_type, detail, evidence_event_id, sales_minutes))\n FROM duck_yeah_experiment.events WHERE dataset IN (SELECT dataset FROM scen) GROUP BY ALL\n)\nSELECT scen.dataset, k.check_id,\n coalesce(c.violations, 0) AS violations,\n k.check_id IN ('identical_event_retries_collapsed', 'ascertainment_after_cutoff_ignored', 'documented_nonreceipt') AS informational\nFROM scen\nCROSS JOIN (SELECT unnest(['assignment_replay_mismatch','completed_block_imbalance','partial_block_arm_difference_over_5',\n 'duplicate_account_in_denominator','unresolved_cross_arm_company_collision','duplicate_event_id_with_conflicting_payload',\n 'outcome_before_assignment','future_evidence_in_routing','missing_required_outcome_ascertainment','contact_after_optout',\n 'more_than_two_proactive_contacts','fixture_or_visitor_record_in_modeled_cohort','ledger_reconciliation_mismatch',\n 'event_for_unassigned_account','identical_event_retries_collapsed','ascertainment_conflict',\n 'contradictory_ascertainment_statuses','future_ascertainment_contradiction','ascertainment_after_cutoff_ignored',\n 'opportunity_ledger_incomplete','exposure_log_missing','exposure_log_unplanned','opportunity_status_inconsistent',\n 'documented_nonreceipt']) AS check_id) k\nLEFT JOIN checks c ON c.dataset = scen.dataset AND c.check_id = k.check_id\nORDER BY scen.dataset, k.check_id;\n", "sha256": "f2fdbe25acef778aaf12f9c82c1a4a2960243c1297d378673ac63456afd81072", "elapsedMs": 510.52, "rowCount": 96, "columns": [ "dataset", "check_id", "violations", "informational" ], "rows": [ { "dataset": "beneficial", "check_id": "ascertainment_after_cutoff_ignored", "violations": 0, "informational": true }, { "dataset": "beneficial", "check_id": "ascertainment_conflict", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "assignment_replay_mismatch", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "completed_block_imbalance", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "contact_after_optout", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "contradictory_ascertainment_statuses", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "documented_nonreceipt", "violations": 0, "informational": true }, { "dataset": "beneficial", "check_id": "duplicate_account_in_denominator", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "event_for_unassigned_account", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "exposure_log_missing", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "exposure_log_unplanned", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "future_ascertainment_contradiction", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "future_evidence_in_routing", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "identical_event_retries_collapsed", "violations": 0, "informational": true }, { "dataset": "beneficial", "check_id": "ledger_reconciliation_mismatch", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "missing_required_outcome_ascertainment", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "more_than_two_proactive_contacts", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "opportunity_ledger_incomplete", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "opportunity_status_inconsistent", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "outcome_before_assignment", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "partial_block_arm_difference_over_5", "violations": 0, "informational": false }, { "dataset": "beneficial", "check_id": "unresolved_cross_arm_company_collision", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "ascertainment_after_cutoff_ignored", "violations": 0, "informational": true }, { "dataset": "harm", "check_id": "ascertainment_conflict", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "assignment_replay_mismatch", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "completed_block_imbalance", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "contact_after_optout", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "contradictory_ascertainment_statuses", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "documented_nonreceipt", "violations": 0, "informational": true }, { "dataset": "harm", "check_id": "duplicate_account_in_denominator", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "event_for_unassigned_account", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "exposure_log_missing", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "exposure_log_unplanned", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "future_ascertainment_contradiction", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "future_evidence_in_routing", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "identical_event_retries_collapsed", "violations": 0, "informational": true }, { "dataset": "harm", "check_id": "ledger_reconciliation_mismatch", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "missing_required_outcome_ascertainment", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "more_than_two_proactive_contacts", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "opportunity_ledger_incomplete", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "opportunity_status_inconsistent", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "outcome_before_assignment", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "partial_block_arm_difference_over_5", "violations": 0, "informational": false }, { "dataset": "harm", "check_id": "unresolved_cross_arm_company_collision", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "ascertainment_after_cutoff_ignored", "violations": 0, "informational": true }, { "dataset": "insufficient_enrollment", "check_id": "ascertainment_conflict", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "assignment_replay_mismatch", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "completed_block_imbalance", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "contact_after_optout", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "contradictory_ascertainment_statuses", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "documented_nonreceipt", "violations": 0, "informational": true }, { "dataset": "insufficient_enrollment", "check_id": "duplicate_account_in_denominator", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "event_for_unassigned_account", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "exposure_log_missing", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "exposure_log_unplanned", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "future_ascertainment_contradiction", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "future_evidence_in_routing", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "identical_event_retries_collapsed", "violations": 0, "informational": true }, { "dataset": "insufficient_enrollment", "check_id": "ledger_reconciliation_mismatch", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "missing_required_outcome_ascertainment", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "more_than_two_proactive_contacts", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "opportunity_ledger_incomplete", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "opportunity_status_inconsistent", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "outcome_before_assignment", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "partial_block_arm_difference_over_5", "violations": 0, "informational": false }, { "dataset": "insufficient_enrollment", "check_id": "unresolved_cross_arm_company_collision", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "ascertainment_after_cutoff_ignored", "violations": 0, "informational": true }, { "dataset": "upstream_only", "check_id": "ascertainment_conflict", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "assignment_replay_mismatch", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "completed_block_imbalance", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "contact_after_optout", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "contradictory_ascertainment_statuses", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "documented_nonreceipt", "violations": 0, "informational": true }, { "dataset": "upstream_only", "check_id": "duplicate_account_in_denominator", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "duplicate_event_id_with_conflicting_payload", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "event_for_unassigned_account", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "exposure_log_missing", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "exposure_log_unplanned", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "fixture_or_visitor_record_in_modeled_cohort", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "future_ascertainment_contradiction", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "future_evidence_in_routing", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "identical_event_retries_collapsed", "violations": 0, "informational": true }, { "dataset": "upstream_only", "check_id": "ledger_reconciliation_mismatch", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "missing_required_outcome_ascertainment", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "more_than_two_proactive_contacts", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "opportunity_ledger_incomplete", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "opportunity_status_inconsistent", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "outcome_before_assignment", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "partial_block_arm_difference_over_5", "violations": 0, "informational": false }, { "dataset": "upstream_only", "check_id": "unresolved_cross_arm_company_collision", "violations": 0, "informational": false } ], "rowsPublishedIn": null }, { "id": "maturity", "sqlFile": "scripts/duck-yeah/sql/experiment_maturity.sql", "sql": "-- Duck Yeah v2 · enrollment and maturity per scenario.\n-- Final analysis = last assignment + 28 days (protocol windows.finalAnalysisAt).\n-- `immature` counts accounts whose 28-day window had not closed at that\n-- cutoff; they would be pending, never zeros.\nSELECT dataset,\n count(*) AS assigned,\n count(*) FILTER (WHERE arm = 'control') AS control,\n count(*) FILTER (WHERE arm = 'treatment') AS treatment,\n min(assigned_at) AS first_assigned_at,\n max(assigned_at) AS last_assigned_at,\n any_value(analysis_at) AS analysis_at,\n round(date_diff('second', min(assigned_at), max(assigned_at)) / 86400.0, 3) AS enrollment_span_days,\n count(*) FILTER (WHERE immature) AS immature,\n coalesce(sum(asc_after_cutoff), 0) AS ascertained_after_cutoff,\n count(DISTINCT enrollment_index // 10) AS blocks,\n -- a block is complete iff it reached position 9\n count(DISTINCT enrollment_index // 10)\n - count(DISTINCT enrollment_index // 10) FILTER (WHERE enrollment_index % 10 = 9) AS partial_blocks\nFROM duck_yeah_experiment.account_outcomes\nWHERE arm IN ('control', 'treatment')\nGROUP BY dataset ORDER BY dataset;\n", "sha256": "b7c92a926f7bb08494de4d18000bcb953793fa9be34f841015cff45baf9e1088", "elapsedMs": 67.95, "rowCount": 4, "columns": [ "dataset", "assigned", "control", "treatment", "first_assigned_at", "last_assigned_at", "analysis_at", "enrollment_span_days", "immature", "ascertained_after_cutoff", "blocks", "partial_blocks" ], "rows": [ { "dataset": "beneficial", "assigned": 8000, "control": 4000, "treatment": 4000, "first_assigned_at": "2026-05-01T00:00:00.000Z", "last_assigned_at": "2026-06-11T23:52:26.000Z", "analysis_at": "2026-07-09T23:52:26.000Z", "enrollment_span_days": 41.995, "immature": 0, "ascertained_after_cutoff": 0, "blocks": 800, "partial_blocks": 0 }, { "dataset": "harm", "assigned": 8000, "control": 4000, "treatment": 4000, "first_assigned_at": "2026-05-01T00:00:00.000Z", "last_assigned_at": "2026-06-11T23:52:26.000Z", "analysis_at": "2026-07-09T23:52:26.000Z", "enrollment_span_days": 41.995, "immature": 0, "ascertained_after_cutoff": 0, "blocks": 800, "partial_blocks": 0 }, { "dataset": "insufficient_enrollment", "assigned": 3000, "control": 1500, "treatment": 1500, "first_assigned_at": "2026-05-01T00:00:00.000Z", "last_assigned_at": "2026-06-11T23:39:50.000Z", "analysis_at": "2026-07-09T23:39:50.000Z", "enrollment_span_days": 41.986, "immature": 0, "ascertained_after_cutoff": 0, "blocks": 300, "partial_blocks": 0 }, { "dataset": "upstream_only", "assigned": 8000, "control": 4000, "treatment": 4000, "first_assigned_at": "2026-05-01T00:00:00.000Z", "last_assigned_at": "2026-06-11T23:52:26.000Z", "analysis_at": "2026-07-09T23:52:26.000Z", "enrollment_span_days": 41.995, "immature": 0, "ascertained_after_cutoff": 0, "blocks": 800, "partial_blocks": 0 } ], "rowsPublishedIn": null }, { "id": "metrics", "sqlFile": "scripts/duck-yeah/sql/experiment_metrics.sql", "sql": "-- Duck Yeah v2 · ITT outcome counts per scenario × arm × outcome.\n-- Denominator `assigned` = every randomized eligible account, including\n-- accounts never contacted or that ignored the help. Unknowns are NOT\n-- failures: missing / pending / conflict are counted apart from zeros and\n-- block a definitive decision. Sales minutes: arm total ÷ all assigned.\nWITH o AS (\n SELECT * FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')\n),\nlong AS (\n SELECT dataset, arm, 'sqa28' AS outcome, sqa28_status AS status FROM o\n UNION ALL SELECT dataset, arm, 'activation7', activation7_status FROM o\n UNION ALL SELECT dataset, arm, 'optout28', optout28_status FROM o\n UNION ALL SELECT dataset, arm, 'sharing7', sharing7_status FROM o\n)\nSELECT dataset, arm, outcome,\n count(*) AS assigned,\n count(*) FILTER (WHERE status = 'event') AS events,\n count(*) FILTER (WHERE status = 'zero') AS confirmed_zero,\n count(*) FILTER (WHERE status = 'missing') AS missing,\n count(*) FILTER (WHERE status = 'pending') AS pending,\n count(*) FILTER (WHERE status = 'conflict') AS conflicts,\n NULL::DOUBLE AS minutes_total,\n NULL::BIGINT AS minutes_missing\nFROM long GROUP BY ALL\nUNION ALL\nSELECT dataset, arm, 'sales_minutes', count(*), NULL, NULL, NULL, NULL, NULL,\n sum(sales_minutes), count(*) FILTER (WHERE sales_minutes IS NULL)\nFROM o GROUP BY ALL\nORDER BY dataset, outcome, arm;\n", "sha256": "7f51f37a1e27bb7fd7a90d1f9b1399bdd6645680cce3e12d6c74276fd54c0e06", "elapsedMs": 72.99, "rowCount": 40, "columns": [ "dataset", "arm", "outcome", "assigned", "events", "confirmed_zero", "missing", "pending", "conflicts", "minutes_total", "minutes_missing" ], "rows": [ { "dataset": "beneficial", "arm": "control", "outcome": "activation7", "assigned": 4000, "events": 1008, "confirmed_zero": 2992, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "treatment", "outcome": "activation7", "assigned": 4000, "events": 1154, "confirmed_zero": 2846, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "control", "outcome": "optout28", "assigned": 4000, "events": 40, "confirmed_zero": 3960, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "treatment", "outcome": "optout28", "assigned": 4000, "events": 40, "confirmed_zero": 3960, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "control", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 2398.1349, "minutes_missing": 0 }, { "dataset": "beneficial", "arm": "treatment", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 3597.9164, "minutes_missing": 0 }, { "dataset": "beneficial", "arm": "control", "outcome": "sharing7", "assigned": 4000, "events": 493, "confirmed_zero": 3507, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "treatment", "outcome": "sharing7", "assigned": 4000, "events": 752, "confirmed_zero": 3248, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "control", "outcome": "sqa28", "assigned": 4000, "events": 390, "confirmed_zero": 3610, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "beneficial", "arm": "treatment", "outcome": "sqa28", "assigned": 4000, "events": 478, "confirmed_zero": 3522, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "control", "outcome": "activation7", "assigned": 4000, "events": 996, "confirmed_zero": 3004, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "treatment", "outcome": "activation7", "assigned": 4000, "events": 763, "confirmed_zero": 3237, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "control", "outcome": "optout28", "assigned": 4000, "events": 36, "confirmed_zero": 3964, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "treatment", "outcome": "optout28", "assigned": 4000, "events": 114, "confirmed_zero": 3886, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "control", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 2398.0449, "minutes_missing": 0 }, { "dataset": "harm", "arm": "treatment", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 5200.5454, "minutes_missing": 0 }, { "dataset": "harm", "arm": "control", "outcome": "sharing7", "assigned": 4000, "events": 462, "confirmed_zero": 3538, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "treatment", "outcome": "sharing7", "assigned": 4000, "events": 578, "confirmed_zero": 3422, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "control", "outcome": "sqa28", "assigned": 4000, "events": 397, "confirmed_zero": 3603, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "harm", "arm": "treatment", "outcome": "sqa28", "assigned": 4000, "events": 515, "confirmed_zero": 3485, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "control", "outcome": "activation7", "assigned": 1500, "events": 391, "confirmed_zero": 1109, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "treatment", "outcome": "activation7", "assigned": 1500, "events": 410, "confirmed_zero": 1090, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "control", "outcome": "optout28", "assigned": 1500, "events": 18, "confirmed_zero": 1482, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "treatment", "outcome": "optout28", "assigned": 1500, "events": 14, "confirmed_zero": 1486, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "control", "outcome": "sales_minutes", "assigned": 1500, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 900.7931, "minutes_missing": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "outcome": "sales_minutes", "assigned": 1500, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 1349.0462, "minutes_missing": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "outcome": "sharing7", "assigned": 1500, "events": 175, "confirmed_zero": 1325, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "treatment", "outcome": "sharing7", "assigned": 1500, "events": 277, "confirmed_zero": 1223, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "control", "outcome": "sqa28", "assigned": 1500, "events": 166, "confirmed_zero": 1334, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "insufficient_enrollment", "arm": "treatment", "outcome": "sqa28", "assigned": 1500, "events": 209, "confirmed_zero": 1291, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "control", "outcome": "activation7", "assigned": 4000, "events": 1019, "confirmed_zero": 2981, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "treatment", "outcome": "activation7", "assigned": 4000, "events": 1363, "confirmed_zero": 2637, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "control", "outcome": "optout28", "assigned": 4000, "events": 39, "confirmed_zero": 3961, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "treatment", "outcome": "optout28", "assigned": 4000, "events": 37, "confirmed_zero": 3963, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "control", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 2404.5432, "minutes_missing": 0 }, { "dataset": "upstream_only", "arm": "treatment", "outcome": "sales_minutes", "assigned": 4000, "events": null, "confirmed_zero": null, "missing": null, "pending": null, "conflicts": null, "minutes_total": 3601.2896, "minutes_missing": 0 }, { "dataset": "upstream_only", "arm": "control", "outcome": "sharing7", "assigned": 4000, "events": 499, "confirmed_zero": 3501, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "treatment", "outcome": "sharing7", "assigned": 4000, "events": 855, "confirmed_zero": 3145, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "control", "outcome": "sqa28", "assigned": 4000, "events": 394, "confirmed_zero": 3606, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null }, { "dataset": "upstream_only", "arm": "treatment", "outcome": "sqa28", "assigned": 4000, "events": 407, "confirmed_zero": 3593, "missing": 0, "pending": 0, "conflicts": 0, "minutes_total": null, "minutes_missing": null } ], "rowsPublishedIn": null }, { "id": "funnel", "sqlFile": "scripts/duck-yeah/sql/experiment_funnel.sql", "sql": "-- Duck Yeah v2 · account funnel per scenario × arm. Every stage uses ALL\n-- assigned accounts as the denominator (no conditional ratios here; any\n-- stage-to-stage ratio shown elsewhere is descriptive). SQA is a parallel\n-- downstream endpoint — not a subset of activation — so it is listed last\n-- but not nested.\nSELECT dataset, arm,\n count(*) AS assigned,\n count(*) FILTER (WHERE contacts >= 1) AS contacted,\n count(*) FILTER (WHERE contacts >= 2) AS contacted_twice,\n count(*) FILTER (WHERE imported_at IS NOT NULL) AS imported7,\n count(*) FILTER (WHERE first_query_at IS NOT NULL) AS first_query7,\n count(*) FILTER (WHERE activation7_status = 'event') AS activated7,\n count(*) FILTER (WHERE sharing7_status = 'event') AS shared7,\n count(*) FILTER (WHERE sqa28_status = 'event') AS sqa28,\n count(*) FILTER (WHERE sqa28_status = 'event' AND activation7_status <> 'event') AS sqa28_without_activation7,\n count(*) FILTER (WHERE optout28_status = 'event') AS optout28\nFROM duck_yeah_experiment.account_outcomes\nWHERE arm IN ('control', 'treatment')\nGROUP BY ALL ORDER BY dataset, arm;\n", "sha256": "eaa08e547c57654b2c58c7c33a7d2fbda7de05f7f753f2ab56780916bb9f4980", "elapsedMs": 47.4, "rowCount": 8, "columns": [ "dataset", "arm", "assigned", "contacted", "contacted_twice", "imported7", "first_query7", "activated7", "shared7", "sqa28", "sqa28_without_activation7", "optout28" ], "rows": [ { "dataset": "beneficial", "arm": "control", "assigned": 4000, "contacted": 4000, "contacted_twice": 3993, "imported7": 2395, "first_query7": 1816, "activated7": 1008, "shared7": 493, "sqa28": 390, "sqa28_without_activation7": 281, "optout28": 40 }, { "dataset": "beneficial", "arm": "treatment", "assigned": 4000, "contacted": 4000, "contacted_twice": 3998, "imported7": 2564, "first_query7": 1951, "activated7": 1154, "shared7": 752, "sqa28": 478, "sqa28_without_activation7": 343, "optout28": 40 }, { "dataset": "harm", "arm": "control", "assigned": 4000, "contacted": 4000, "contacted_twice": 3999, "imported7": 2449, "first_query7": 1828, "activated7": 996, "shared7": 462, "sqa28": 397, "sqa28_without_activation7": 302, "optout28": 36 }, { "dataset": "harm", "arm": "treatment", "assigned": 4000, "contacted": 4000, "contacted_twice": 3992, "imported7": 2157, "first_query7": 1562, "activated7": 763, "shared7": 578, "sqa28": 515, "sqa28_without_activation7": 416, "optout28": 114 }, { "dataset": "insufficient_enrollment", "arm": "control", "assigned": 1500, "contacted": 1500, "contacted_twice": 1498, "imported7": 918, "first_query7": 694, "activated7": 391, "shared7": 175, "sqa28": 166, "sqa28_without_activation7": 129, "optout28": 18 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "assigned": 1500, "contacted": 1500, "contacted_twice": 1499, "imported7": 945, "first_query7": 697, "activated7": 410, "shared7": 277, "sqa28": 209, "sqa28_without_activation7": 152, "optout28": 14 }, { "dataset": "upstream_only", "arm": "control", "assigned": 4000, "contacted": 4000, "contacted_twice": 3996, "imported7": 2464, "first_query7": 1820, "activated7": 1019, "shared7": 499, "sqa28": 394, "sqa28_without_activation7": 290, "optout28": 39 }, { "dataset": "upstream_only", "arm": "treatment", "assigned": 4000, "contacted": 4000, "contacted_twice": 3997, "imported7": 2722, "first_query7": 2120, "activated7": 1363, "shared7": 855, "sqa28": 407, "sqa28_without_activation7": 262, "optout28": 37 } ], "rowsPublishedIn": null }, { "id": "cohort_curves", "sqlFile": "scripts/duck-yeah/sql/experiment_cohort_curves.sql", "sql": "-- Duck Yeah v2 · cumulative activation (days 0–7) and SQA (days 0–28) by\n-- elapsed day since assignment. The at-risk denominator is EVERY assigned\n-- account in the arm (non-converters stay in it), so no \"faster activation\"\n-- claim can come from converters only. Day d counts events strictly before\n-- assignment + d days.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')),\ndays AS (SELECT unnest(range(0, 29)) AS day)\nSELECT o.dataset, o.arm, d.day,\n count(*) AS at_risk,\n CASE WHEN d.day <= 7 THEN count(*) FILTER (WHERE o.activation7_status = 'event'\n AND o.activated_at < o.assigned_at + to_days(CAST(d.day AS INTEGER))) END AS activated_cum,\n count(*) FILTER (WHERE o.sqa28_status = 'event' AND o.sqa_at < o.assigned_at + to_days(CAST(d.day AS INTEGER))) AS sqa_cum\nFROM o CROSS JOIN days d\nGROUP BY o.dataset, o.arm, d.day ORDER BY o.dataset, o.arm, d.day;\n", "sha256": "85b956f419f667091cef8786091e31cc534a7525605bde5721bcebe2e4a2e1e4", "elapsedMs": 154.84, "rowCount": 232, "columns": [ "dataset", "arm", "day", "at_risk", "activated_cum", "sqa_cum" ], "rows": [ { "dataset": "beneficial", "arm": "control", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 3, "at_risk": 4000, "activated_cum": 202, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 4, "at_risk": 4000, "activated_cum": 403, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 5, "at_risk": 4000, "activated_cum": 598, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 6, "at_risk": 4000, "activated_cum": 796, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 7, "at_risk": 4000, "activated_cum": 1008, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "control", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 13 }, { "dataset": "beneficial", "arm": "control", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 30 }, { "dataset": "beneficial", "arm": "control", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 53 }, { "dataset": "beneficial", "arm": "control", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 74 }, { "dataset": "beneficial", "arm": "control", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 91 }, { "dataset": "beneficial", "arm": "control", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 105 }, { "dataset": "beneficial", "arm": "control", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 123 }, { "dataset": "beneficial", "arm": "control", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 139 }, { "dataset": "beneficial", "arm": "control", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 172 }, { "dataset": "beneficial", "arm": "control", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 189 }, { "dataset": "beneficial", "arm": "control", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 209 }, { "dataset": "beneficial", "arm": "control", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 230 }, { "dataset": "beneficial", "arm": "control", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 253 }, { "dataset": "beneficial", "arm": "control", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 269 }, { "dataset": "beneficial", "arm": "control", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 285 }, { "dataset": "beneficial", "arm": "control", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 313 }, { "dataset": "beneficial", "arm": "control", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 334 }, { "dataset": "beneficial", "arm": "control", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 351 }, { "dataset": "beneficial", "arm": "control", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 368 }, { "dataset": "beneficial", "arm": "control", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 390 }, { "dataset": "beneficial", "arm": "treatment", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 3, "at_risk": 4000, "activated_cum": 237, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 4, "at_risk": 4000, "activated_cum": 506, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 5, "at_risk": 4000, "activated_cum": 727, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 6, "at_risk": 4000, "activated_cum": 921, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 7, "at_risk": 4000, "activated_cum": 1154, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "beneficial", "arm": "treatment", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 23 }, { "dataset": "beneficial", "arm": "treatment", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 53 }, { "dataset": "beneficial", "arm": "treatment", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 77 }, { "dataset": "beneficial", "arm": "treatment", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 105 }, { "dataset": "beneficial", "arm": "treatment", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 132 }, { "dataset": "beneficial", "arm": "treatment", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 152 }, { "dataset": "beneficial", "arm": "treatment", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 174 }, { "dataset": "beneficial", "arm": "treatment", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 196 }, { "dataset": "beneficial", "arm": "treatment", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 214 }, { "dataset": "beneficial", "arm": "treatment", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 238 }, { "dataset": "beneficial", "arm": "treatment", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 266 }, { "dataset": "beneficial", "arm": "treatment", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 288 }, { "dataset": "beneficial", "arm": "treatment", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 312 }, { "dataset": "beneficial", "arm": "treatment", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 335 }, { "dataset": "beneficial", "arm": "treatment", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 357 }, { "dataset": "beneficial", "arm": "treatment", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 381 }, { "dataset": "beneficial", "arm": "treatment", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 408 }, { "dataset": "beneficial", "arm": "treatment", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 438 }, { "dataset": "beneficial", "arm": "treatment", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 458 }, { "dataset": "beneficial", "arm": "treatment", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 478 }, { "dataset": "harm", "arm": "control", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 3, "at_risk": 4000, "activated_cum": 202, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 4, "at_risk": 4000, "activated_cum": 397, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 5, "at_risk": 4000, "activated_cum": 592, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 6, "at_risk": 4000, "activated_cum": 795, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 7, "at_risk": 4000, "activated_cum": 996, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "harm", "arm": "control", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 20 }, { "dataset": "harm", "arm": "control", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 42 }, { "dataset": "harm", "arm": "control", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 62 }, { "dataset": "harm", "arm": "control", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 81 }, { "dataset": "harm", "arm": "control", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 106 }, { "dataset": "harm", "arm": "control", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 119 }, { "dataset": "harm", "arm": "control", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 142 }, { "dataset": "harm", "arm": "control", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 157 }, { "dataset": "harm", "arm": "control", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 180 }, { "dataset": "harm", "arm": "control", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 197 }, { "dataset": "harm", "arm": "control", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 217 }, { "dataset": "harm", "arm": "control", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 233 }, { "dataset": "harm", "arm": "control", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 258 }, { "dataset": "harm", "arm": "control", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 273 }, { "dataset": "harm", "arm": "control", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 296 }, { "dataset": "harm", "arm": "control", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 314 }, { "dataset": "harm", "arm": "control", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 337 }, { "dataset": "harm", "arm": "control", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 365 }, { "dataset": "harm", "arm": "control", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 381 }, { "dataset": "harm", "arm": "control", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 397 }, { "dataset": "harm", "arm": "treatment", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 3, "at_risk": 4000, "activated_cum": 175, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 4, "at_risk": 4000, "activated_cum": 333, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 5, "at_risk": 4000, "activated_cum": 468, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 6, "at_risk": 4000, "activated_cum": 618, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 7, "at_risk": 4000, "activated_cum": 763, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "harm", "arm": "treatment", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 34 }, { "dataset": "harm", "arm": "treatment", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 59 }, { "dataset": "harm", "arm": "treatment", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 79 }, { "dataset": "harm", "arm": "treatment", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 93 }, { "dataset": "harm", "arm": "treatment", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 118 }, { "dataset": "harm", "arm": "treatment", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 141 }, { "dataset": "harm", "arm": "treatment", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 161 }, { "dataset": "harm", "arm": "treatment", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 189 }, { "dataset": "harm", "arm": "treatment", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 219 }, { "dataset": "harm", "arm": "treatment", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 244 }, { "dataset": "harm", "arm": "treatment", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 272 }, { "dataset": "harm", "arm": "treatment", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 300 }, { "dataset": "harm", "arm": "treatment", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 316 }, { "dataset": "harm", "arm": "treatment", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 340 }, { "dataset": "harm", "arm": "treatment", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 369 }, { "dataset": "harm", "arm": "treatment", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 399 }, { "dataset": "harm", "arm": "treatment", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 433 }, { "dataset": "harm", "arm": "treatment", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 460 }, { "dataset": "harm", "arm": "treatment", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 488 }, { "dataset": "harm", "arm": "treatment", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 515 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 0, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 1, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 2, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 3, "at_risk": 1500, "activated_cum": 83, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 4, "at_risk": 1500, "activated_cum": 158, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 5, "at_risk": 1500, "activated_cum": 234, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 6, "at_risk": 1500, "activated_cum": 321, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 7, "at_risk": 1500, "activated_cum": 391, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 8, "at_risk": 1500, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 9, "at_risk": 1500, "activated_cum": null, "sqa_cum": 7 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 10, "at_risk": 1500, "activated_cum": null, "sqa_cum": 18 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 11, "at_risk": 1500, "activated_cum": null, "sqa_cum": 31 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 12, "at_risk": 1500, "activated_cum": null, "sqa_cum": 37 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 13, "at_risk": 1500, "activated_cum": null, "sqa_cum": 45 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 14, "at_risk": 1500, "activated_cum": null, "sqa_cum": 54 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 15, "at_risk": 1500, "activated_cum": null, "sqa_cum": 64 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 16, "at_risk": 1500, "activated_cum": null, "sqa_cum": 72 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 17, "at_risk": 1500, "activated_cum": null, "sqa_cum": 76 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 18, "at_risk": 1500, "activated_cum": null, "sqa_cum": 86 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 19, "at_risk": 1500, "activated_cum": null, "sqa_cum": 95 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 20, "at_risk": 1500, "activated_cum": null, "sqa_cum": 103 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 21, "at_risk": 1500, "activated_cum": null, "sqa_cum": 107 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 22, "at_risk": 1500, "activated_cum": null, "sqa_cum": 114 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 23, "at_risk": 1500, "activated_cum": null, "sqa_cum": 123 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 24, "at_risk": 1500, "activated_cum": null, "sqa_cum": 127 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 25, "at_risk": 1500, "activated_cum": null, "sqa_cum": 139 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 26, "at_risk": 1500, "activated_cum": null, "sqa_cum": 149 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 27, "at_risk": 1500, "activated_cum": null, "sqa_cum": 157 }, { "dataset": "insufficient_enrollment", "arm": "control", "day": 28, "at_risk": 1500, "activated_cum": null, "sqa_cum": 166 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 0, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 1, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 2, "at_risk": 1500, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 3, "at_risk": 1500, "activated_cum": 68, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 4, "at_risk": 1500, "activated_cum": 152, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 5, "at_risk": 1500, "activated_cum": 247, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 6, "at_risk": 1500, "activated_cum": 326, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 7, "at_risk": 1500, "activated_cum": 410, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 8, "at_risk": 1500, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 9, "at_risk": 1500, "activated_cum": null, "sqa_cum": 13 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 10, "at_risk": 1500, "activated_cum": null, "sqa_cum": 25 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 11, "at_risk": 1500, "activated_cum": null, "sqa_cum": 38 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 12, "at_risk": 1500, "activated_cum": null, "sqa_cum": 50 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 13, "at_risk": 1500, "activated_cum": null, "sqa_cum": 65 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 14, "at_risk": 1500, "activated_cum": null, "sqa_cum": 76 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 15, "at_risk": 1500, "activated_cum": null, "sqa_cum": 84 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 16, "at_risk": 1500, "activated_cum": null, "sqa_cum": 92 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 17, "at_risk": 1500, "activated_cum": null, "sqa_cum": 102 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 18, "at_risk": 1500, "activated_cum": null, "sqa_cum": 112 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 19, "at_risk": 1500, "activated_cum": null, "sqa_cum": 126 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 20, "at_risk": 1500, "activated_cum": null, "sqa_cum": 137 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 21, "at_risk": 1500, "activated_cum": null, "sqa_cum": 146 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 22, "at_risk": 1500, "activated_cum": null, "sqa_cum": 155 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 23, "at_risk": 1500, "activated_cum": null, "sqa_cum": 167 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 24, "at_risk": 1500, "activated_cum": null, "sqa_cum": 177 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 25, "at_risk": 1500, "activated_cum": null, "sqa_cum": 181 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 26, "at_risk": 1500, "activated_cum": null, "sqa_cum": 192 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 27, "at_risk": 1500, "activated_cum": null, "sqa_cum": 198 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "day": 28, "at_risk": 1500, "activated_cum": null, "sqa_cum": 209 }, { "dataset": "upstream_only", "arm": "control", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 3, "at_risk": 4000, "activated_cum": 226, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 4, "at_risk": 4000, "activated_cum": 411, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 5, "at_risk": 4000, "activated_cum": 601, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 6, "at_risk": 4000, "activated_cum": 821, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 7, "at_risk": 4000, "activated_cum": 1019, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "control", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 14 }, { "dataset": "upstream_only", "arm": "control", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 34 }, { "dataset": "upstream_only", "arm": "control", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 51 }, { "dataset": "upstream_only", "arm": "control", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 72 }, { "dataset": "upstream_only", "arm": "control", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 93 }, { "dataset": "upstream_only", "arm": "control", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 113 }, { "dataset": "upstream_only", "arm": "control", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 132 }, { "dataset": "upstream_only", "arm": "control", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 152 }, { "dataset": "upstream_only", "arm": "control", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 174 }, { "dataset": "upstream_only", "arm": "control", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 200 }, { "dataset": "upstream_only", "arm": "control", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 217 }, { "dataset": "upstream_only", "arm": "control", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 241 }, { "dataset": "upstream_only", "arm": "control", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 256 }, { "dataset": "upstream_only", "arm": "control", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 272 }, { "dataset": "upstream_only", "arm": "control", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 296 }, { "dataset": "upstream_only", "arm": "control", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 314 }, { "dataset": "upstream_only", "arm": "control", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 331 }, { "dataset": "upstream_only", "arm": "control", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 357 }, { "dataset": "upstream_only", "arm": "control", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 378 }, { "dataset": "upstream_only", "arm": "control", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 394 }, { "dataset": "upstream_only", "arm": "treatment", "day": 0, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 1, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 2, "at_risk": 4000, "activated_cum": 0, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 3, "at_risk": 4000, "activated_cum": 285, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 4, "at_risk": 4000, "activated_cum": 545, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 5, "at_risk": 4000, "activated_cum": 820, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 6, "at_risk": 4000, "activated_cum": 1069, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 7, "at_risk": 4000, "activated_cum": 1363, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 8, "at_risk": 4000, "activated_cum": null, "sqa_cum": 0 }, { "dataset": "upstream_only", "arm": "treatment", "day": 9, "at_risk": 4000, "activated_cum": null, "sqa_cum": 29 }, { "dataset": "upstream_only", "arm": "treatment", "day": 10, "at_risk": 4000, "activated_cum": null, "sqa_cum": 45 }, { "dataset": "upstream_only", "arm": "treatment", "day": 11, "at_risk": 4000, "activated_cum": null, "sqa_cum": 76 }, { "dataset": "upstream_only", "arm": "treatment", "day": 12, "at_risk": 4000, "activated_cum": null, "sqa_cum": 98 }, { "dataset": "upstream_only", "arm": "treatment", "day": 13, "at_risk": 4000, "activated_cum": null, "sqa_cum": 123 }, { "dataset": "upstream_only", "arm": "treatment", "day": 14, "at_risk": 4000, "activated_cum": null, "sqa_cum": 144 }, { "dataset": "upstream_only", "arm": "treatment", "day": 15, "at_risk": 4000, "activated_cum": null, "sqa_cum": 156 }, { "dataset": "upstream_only", "arm": "treatment", "day": 16, "at_risk": 4000, "activated_cum": null, "sqa_cum": 178 }, { "dataset": "upstream_only", "arm": "treatment", "day": 17, "at_risk": 4000, "activated_cum": null, "sqa_cum": 202 }, { "dataset": "upstream_only", "arm": "treatment", "day": 18, "at_risk": 4000, "activated_cum": null, "sqa_cum": 218 }, { "dataset": "upstream_only", "arm": "treatment", "day": 19, "at_risk": 4000, "activated_cum": null, "sqa_cum": 237 }, { "dataset": "upstream_only", "arm": "treatment", "day": 20, "at_risk": 4000, "activated_cum": null, "sqa_cum": 257 }, { "dataset": "upstream_only", "arm": "treatment", "day": 21, "at_risk": 4000, "activated_cum": null, "sqa_cum": 275 }, { "dataset": "upstream_only", "arm": "treatment", "day": 22, "at_risk": 4000, "activated_cum": null, "sqa_cum": 293 }, { "dataset": "upstream_only", "arm": "treatment", "day": 23, "at_risk": 4000, "activated_cum": null, "sqa_cum": 309 }, { "dataset": "upstream_only", "arm": "treatment", "day": 24, "at_risk": 4000, "activated_cum": null, "sqa_cum": 331 }, { "dataset": "upstream_only", "arm": "treatment", "day": 25, "at_risk": 4000, "activated_cum": null, "sqa_cum": 347 }, { "dataset": "upstream_only", "arm": "treatment", "day": 26, "at_risk": 4000, "activated_cum": null, "sqa_cum": 364 }, { "dataset": "upstream_only", "arm": "treatment", "day": 27, "at_risk": 4000, "activated_cum": null, "sqa_cum": 387 }, { "dataset": "upstream_only", "arm": "treatment", "day": 28, "at_risk": 4000, "activated_cum": null, "sqa_cum": 407 } ], "rowsPublishedIn": null }, { "id": "segments", "sqlFile": "scripts/duck-yeah/sql/experiment_segments.sql", "sql": "-- Duck Yeah v2 · prespecified DESCRIPTIVE segment cuts (protocol\n-- analysis.segments): acquisition source, declared task need, company size.\n-- No subgroup can replace the registered primary population; intervals added\n-- downstream are labelled descriptive.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')),\nlong AS (\n SELECT dataset, arm, 'acquisition_source' AS dimension, acquisition_source AS segment, * EXCLUDE (dataset, arm) FROM o\n UNION ALL BY NAME\n SELECT dataset, arm, 'task_need' AS dimension, task_need AS segment, * EXCLUDE (dataset, arm) FROM o\n UNION ALL BY NAME\n SELECT dataset, arm, 'company_size' AS dimension, company_size AS segment, * EXCLUDE (dataset, arm) FROM o\n)\nSELECT dataset, dimension, segment, arm,\n count(*) AS assigned,\n count(*) FILTER (WHERE sqa28_status = 'event') AS sqa28,\n count(*) FILTER (WHERE activation7_status = 'event') AS activation7,\n count(*) FILTER (WHERE optout28_status = 'event') AS optout28\nFROM long GROUP BY ALL ORDER BY dataset, dimension, segment, arm;\n", "sha256": "6fc55a1d28c72b0ec11cc6485edafbc27c2c158a7835a0ed49e440d47acc4038", "elapsedMs": 72.49, "rowCount": 72, "columns": [ "dataset", "dimension", "segment", "arm", "assigned", "sqa28", "activation7", "optout28" ], "rows": [ { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "content", "arm": "control", "assigned": 1297, "sqa28": 123, "activation7": 320, "optout28": 12 }, { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "content", "arm": "treatment", "assigned": 1178, "sqa28": 143, "activation7": 371, "optout28": 12 }, { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "direct", "arm": "control", "assigned": 787, "sqa28": 70, "activation7": 197, "optout28": 9 }, { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "direct", "arm": "treatment", "assigned": 794, "sqa28": 95, "activation7": 224, "optout28": 12 }, { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "product_community", "arm": "control", "assigned": 1916, "sqa28": 197, "activation7": 491, "optout28": 19 }, { "dataset": "beneficial", "dimension": "acquisition_source", "segment": "product_community", "arm": "treatment", "assigned": 2028, "sqa28": 240, "activation7": 559, "optout28": 16 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "1-10", "arm": "control", "assigned": 2009, "sqa28": 214, "activation7": 510, "optout28": 21 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "1-10", "arm": "treatment", "assigned": 2003, "sqa28": 252, "activation7": 570, "optout28": 19 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "11-50", "arm": "control", "assigned": 1379, "sqa28": 127, "activation7": 344, "optout28": 14 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "11-50", "arm": "treatment", "assigned": 1371, "sqa28": 157, "activation7": 388, "optout28": 13 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "51-200", "arm": "control", "assigned": 612, "sqa28": 49, "activation7": 154, "optout28": 5 }, { "dataset": "beneficial", "dimension": "company_size", "segment": "51-200", "arm": "treatment", "assigned": 626, "sqa28": 69, "activation7": 196, "optout28": 8 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "import", "arm": "control", "assigned": 1428, "sqa28": 142, "activation7": 358, "optout28": 17 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "import", "arm": "treatment", "assigned": 1366, "sqa28": 167, "activation7": 387, "optout28": 13 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "sharing", "arm": "control", "assigned": 1359, "sqa28": 125, "activation7": 338, "optout28": 12 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "sharing", "arm": "treatment", "assigned": 1402, "sqa28": 175, "activation7": 395, "optout28": 15 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "unknown", "arm": "control", "assigned": 1213, "sqa28": 123, "activation7": 312, "optout28": 11 }, { "dataset": "beneficial", "dimension": "task_need", "segment": "unknown", "arm": "treatment", "assigned": 1232, "sqa28": 136, "activation7": 372, "optout28": 12 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "content", "arm": "control", "assigned": 1212, "sqa28": 121, "activation7": 314, "optout28": 15 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "content", "arm": "treatment", "assigned": 1157, "sqa28": 140, "activation7": 213, "optout28": 33 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "direct", "arm": "control", "assigned": 802, "sqa28": 79, "activation7": 185, "optout28": 9 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "direct", "arm": "treatment", "assigned": 778, "sqa28": 106, "activation7": 157, "optout28": 25 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "product_community", "arm": "control", "assigned": 1986, "sqa28": 197, "activation7": 497, "optout28": 12 }, { "dataset": "harm", "dimension": "acquisition_source", "segment": "product_community", "arm": "treatment", "assigned": 2065, "sqa28": 269, "activation7": 393, "optout28": 56 }, { "dataset": "harm", "dimension": "company_size", "segment": "1-10", "arm": "control", "assigned": 2014, "sqa28": 208, "activation7": 515, "optout28": 20 }, { "dataset": "harm", "dimension": "company_size", "segment": "1-10", "arm": "treatment", "assigned": 2031, "sqa28": 256, "activation7": 405, "optout28": 53 }, { "dataset": "harm", "dimension": "company_size", "segment": "11-50", "arm": "control", "assigned": 1385, "sqa28": 116, "activation7": 355, "optout28": 11 }, { "dataset": "harm", "dimension": "company_size", "segment": "11-50", "arm": "treatment", "assigned": 1368, "sqa28": 184, "activation7": 253, "optout28": 45 }, { "dataset": "harm", "dimension": "company_size", "segment": "51-200", "arm": "control", "assigned": 601, "sqa28": 73, "activation7": 126, "optout28": 5 }, { "dataset": "harm", "dimension": "company_size", "segment": "51-200", "arm": "treatment", "assigned": 601, "sqa28": 75, "activation7": 105, "optout28": 16 }, { "dataset": "harm", "dimension": "task_need", "segment": "import", "arm": "control", "assigned": 1415, "sqa28": 128, "activation7": 342, "optout28": 13 }, { "dataset": "harm", "dimension": "task_need", "segment": "import", "arm": "treatment", "assigned": 1395, "sqa28": 191, "activation7": 282, "optout28": 33 }, { "dataset": "harm", "dimension": "task_need", "segment": "sharing", "arm": "control", "assigned": 1411, "sqa28": 145, "activation7": 351, "optout28": 10 }, { "dataset": "harm", "dimension": "task_need", "segment": "sharing", "arm": "treatment", "assigned": 1392, "sqa28": 178, "activation7": 269, "optout28": 45 }, { "dataset": "harm", "dimension": "task_need", "segment": "unknown", "arm": "control", "assigned": 1174, "sqa28": 124, "activation7": 303, "optout28": 13 }, { "dataset": "harm", "dimension": "task_need", "segment": "unknown", "arm": "treatment", "assigned": 1213, "sqa28": 146, "activation7": 212, "optout28": 36 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "content", "arm": "control", "assigned": 459, "sqa28": 59, "activation7": 131, "optout28": 5 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "content", "arm": "treatment", "assigned": 446, "sqa28": 61, "activation7": 117, "optout28": 3 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "direct", "arm": "control", "assigned": 268, "sqa28": 31, "activation7": 62, "optout28": 1 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "direct", "arm": "treatment", "assigned": 322, "sqa28": 42, "activation7": 84, "optout28": 2 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "product_community", "arm": "control", "assigned": 773, "sqa28": 76, "activation7": 198, "optout28": 12 }, { "dataset": "insufficient_enrollment", "dimension": "acquisition_source", "segment": "product_community", "arm": "treatment", "assigned": 732, "sqa28": 106, "activation7": 209, "optout28": 9 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "1-10", "arm": "control", "assigned": 756, "sqa28": 80, "activation7": 196, "optout28": 11 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "1-10", "arm": "treatment", "assigned": 765, "sqa28": 98, "activation7": 202, "optout28": 9 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "11-50", "arm": "control", "assigned": 524, "sqa28": 56, "activation7": 131, "optout28": 6 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "11-50", "arm": "treatment", "assigned": 512, "sqa28": 74, "activation7": 136, "optout28": 3 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "51-200", "arm": "control", "assigned": 220, "sqa28": 30, "activation7": 64, "optout28": 1 }, { "dataset": "insufficient_enrollment", "dimension": "company_size", "segment": "51-200", "arm": "treatment", "assigned": 223, "sqa28": 37, "activation7": 72, "optout28": 2 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "import", "arm": "control", "assigned": 518, "sqa28": 50, "activation7": 129, "optout28": 6 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "import", "arm": "treatment", "assigned": 532, "sqa28": 76, "activation7": 139, "optout28": 4 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "sharing", "arm": "control", "assigned": 539, "sqa28": 64, "activation7": 152, "optout28": 9 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "sharing", "arm": "treatment", "assigned": 524, "sqa28": 69, "activation7": 136, "optout28": 4 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "unknown", "arm": "control", "assigned": 443, "sqa28": 52, "activation7": 110, "optout28": 3 }, { "dataset": "insufficient_enrollment", "dimension": "task_need", "segment": "unknown", "arm": "treatment", "assigned": 444, "sqa28": 64, "activation7": 135, "optout28": 6 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "content", "arm": "control", "assigned": 1182, "sqa28": 117, "activation7": 299, "optout28": 11 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "content", "arm": "treatment", "assigned": 1193, "sqa28": 129, "activation7": 407, "optout28": 13 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "direct", "arm": "control", "assigned": 768, "sqa28": 79, "activation7": 207, "optout28": 7 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "direct", "arm": "treatment", "assigned": 818, "sqa28": 59, "activation7": 300, "optout28": 5 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "product_community", "arm": "control", "assigned": 2050, "sqa28": 198, "activation7": 513, "optout28": 21 }, { "dataset": "upstream_only", "dimension": "acquisition_source", "segment": "product_community", "arm": "treatment", "assigned": 1989, "sqa28": 219, "activation7": 656, "optout28": 19 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "1-10", "arm": "control", "assigned": 2040, "sqa28": 204, "activation7": 505, "optout28": 22 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "1-10", "arm": "treatment", "assigned": 1984, "sqa28": 194, "activation7": 690, "optout28": 14 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "11-50", "arm": "control", "assigned": 1374, "sqa28": 130, "activation7": 372, "optout28": 11 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "11-50", "arm": "treatment", "assigned": 1432, "sqa28": 156, "activation7": 483, "optout28": 17 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "51-200", "arm": "control", "assigned": 586, "sqa28": 60, "activation7": 142, "optout28": 6 }, { "dataset": "upstream_only", "dimension": "company_size", "segment": "51-200", "arm": "treatment", "assigned": 584, "sqa28": 57, "activation7": 190, "optout28": 6 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "import", "arm": "control", "assigned": 1402, "sqa28": 146, "activation7": 347, "optout28": 17 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "import", "arm": "treatment", "assigned": 1418, "sqa28": 146, "activation7": 495, "optout28": 15 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "sharing", "arm": "control", "assigned": 1394, "sqa28": 137, "activation7": 349, "optout28": 11 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "sharing", "arm": "treatment", "assigned": 1427, "sqa28": 143, "activation7": 473, "optout28": 10 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "unknown", "arm": "control", "assigned": 1204, "sqa28": 111, "activation7": 323, "optout28": 11 }, { "dataset": "upstream_only", "dimension": "task_need", "segment": "unknown", "arm": "treatment", "assigned": 1155, "sqa28": 118, "activation7": 395, "optout28": 12 } ], "rowsPublishedIn": null }, { "id": "costs", "sqlFile": "scripts/duck-yeah/sql/experiment_costs.sql", "sql": "-- Duck Yeah v2 · ASSUMED synthetic cost allocation per scenario × arm\n-- (protocol costs block; parameters bound from the frozen protocol, not\n-- vendor prices). Ratios with a zero denominator are NULL, never zero.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')),\nagg AS (\n SELECT dataset, arm, count(*) AS assigned, sum(contacts) AS contacts, sum(sales_minutes) AS sales_minutes,\n count(*) FILTER (WHERE activation7_status = 'event') AS activated7,\n count(*) FILTER (WHERE sqa28_status = 'event') AS sqa28\n FROM o GROUP BY ALL\n),\nc AS (\n SELECT *,\n assigned * $acquisition_per_account AS acquisition,\n assigned * $platform_per_account AS platform_allocation,\n contacts * $delivery_per_contact AS delivery,\n CASE arm WHEN 'control' THEN $content_ops_control ELSE $content_ops_treatment END AS content_ops,\n assigned * CASE arm WHEN 'control' THEN $inference_control ELSE $inference_treatment END AS inference,\n sales_minutes * $sales_hourly_rate / 60.0 AS sales_labor\n FROM agg\n)\nSELECT *, acquisition + platform_allocation + delivery + content_ops + inference + sales_labor AS total,\n (acquisition + platform_allocation + delivery + content_ops + inference + sales_labor) / assigned AS per_assigned,\n (acquisition + platform_allocation + delivery + content_ops + inference + sales_labor) / nullif(activated7, 0) AS per_activated7,\n (acquisition + platform_allocation + delivery + content_ops + inference + sales_labor) / nullif(sqa28, 0) AS per_sqa28,\n sales_minutes / assigned AS sales_minutes_per_assigned\nFROM c ORDER BY dataset, arm;\n", "sha256": "e9831b3f1aba0bd4e3e021b3f48468486c4440f42c0171c6043945eb45a48bf5", "elapsedMs": 59.83, "rowCount": 8, "columns": [ "dataset", "arm", "assigned", "contacts", "sales_minutes", "activated7", "sqa28", "acquisition", "platform_allocation", "delivery", "content_ops", "inference", "sales_labor", "total", "per_assigned", "per_activated7", "per_sqa28", "sales_minutes_per_assigned" ], "rows": [ { "dataset": "beneficial", "arm": "control", "assigned": 4000, "contacts": 7993, "sales_minutes": 2398.1349, "activated7": 1008, "sqa28": 390, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.965, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2398.1349, "total": 26598.0999, "per_assigned": 6.649524975, "per_activated7": 26.387003869, "per_sqa28": 68.2002561538, "sales_minutes_per_assigned": 0.599533725 }, { "dataset": "beneficial", "arm": "treatment", "assigned": 4000, "contacts": 7998, "sales_minutes": 3597.9164, "activated7": 1154, "sqa28": 478, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.99, "content_ops": 240.0, "inference": 8.0, "sales_labor": 3597.9164, "total": 27925.9064, "per_assigned": 6.9814766, "per_activated7": 24.1992256499, "per_sqa28": 58.4223983264, "sales_minutes_per_assigned": 0.8994791 }, { "dataset": "harm", "arm": "control", "assigned": 4000, "contacts": 7999, "sales_minutes": 2398.0449, "activated7": 996, "sqa28": 397, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.995, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2398.0449, "total": 26598.0399, "per_assigned": 6.649509975, "per_activated7": 26.7048593373, "per_sqa28": 66.9975816121, "sales_minutes_per_assigned": 0.599511225 }, { "dataset": "harm", "arm": "treatment", "assigned": 4000, "contacts": 7992, "sales_minutes": 5200.5454, "activated7": 763, "sqa28": 515, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.96, "content_ops": 240.0, "inference": 8.0, "sales_labor": 5200.5454, "total": 29528.5054, "per_assigned": 7.38212635, "per_activated7": 38.7005313237, "per_sqa28": 57.3369036893, "sales_minutes_per_assigned": 1.30013635 }, { "dataset": "insufficient_enrollment", "arm": "control", "assigned": 1500, "contacts": 2998, "sales_minutes": 900.7931, "activated7": 391, "sqa28": 166, "acquisition": 9000.0, "platform_allocation": 15.0, "delivery": 14.99, "content_ops": 120.0, "inference": 0.0, "sales_labor": 900.7931, "total": 10050.7831, "per_assigned": 6.7005220667, "per_activated7": 25.7053276215, "per_sqa28": 60.5468861446, "sales_minutes_per_assigned": 0.6005287333 }, { "dataset": "insufficient_enrollment", "arm": "treatment", "assigned": 1500, "contacts": 2999, "sales_minutes": 1349.0462, "activated7": 410, "sqa28": 209, "acquisition": 9000.0, "platform_allocation": 15.0, "delivery": 14.995, "content_ops": 240.0, "inference": 3.0, "sales_labor": 1349.0462, "total": 10622.0412, "per_assigned": 7.0813608, "per_activated7": 25.907417561, "per_sqa28": 50.8231636364, "sales_minutes_per_assigned": 0.8993641333 }, { "dataset": "upstream_only", "arm": "control", "assigned": 4000, "contacts": 7996, "sales_minutes": 2404.5432, "activated7": 1019, "sqa28": 394, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.98, "content_ops": 120.0, "inference": 0.0, "sales_labor": 2404.5432, "total": 26604.5232, "per_assigned": 6.6511308, "per_activated7": 26.1084624141, "per_sqa28": 67.5241705584, "sales_minutes_per_assigned": 0.6011358 }, { "dataset": "upstream_only", "arm": "treatment", "assigned": 4000, "contacts": 7997, "sales_minutes": 3601.2896, "activated7": 1363, "sqa28": 407, "acquisition": 24000.0, "platform_allocation": 40.0, "delivery": 39.985, "content_ops": 240.0, "inference": 8.0, "sales_labor": 3601.2896, "total": 27929.2746, "per_assigned": 6.98231865, "per_activated7": 20.4910305209, "per_sqa28": 68.6222963145, "sales_minutes_per_assigned": 0.9003224 } ], "rowsPublishedIn": null }, { "id": "trace", "sqlFile": "scripts/duck-yeah/sql/experiment_trace.sql", "sql": "-- Duck Yeah v2 · one example account trace per scenario: the first treatment\n-- account (by enrollment order) with a documented SQA, shown with every raw\n-- event. Chosen by a fixed rule, not for being representative or favourable.\nWITH pick AS (\n SELECT dataset, arg_min(account_id, enrollment_index) AS account_id\n FROM duck_yeah_experiment.account_outcomes\n WHERE arm = 'treatment' AND sqa28_status = 'event'\n GROUP BY dataset\n)\nSELECT e.dataset, e.account_id, s.arm, s.enrollment_index, s.block_index, s.token, a.assigned_at,\n a.acquisition_source, a.task_need, a.company_size, e.event_id, e.event_ts,\n round(date_diff('second', a.assigned_at, e.event_ts) / 86400.0, 4) AS elapsed_days,\n e.event_type, e.detail, e.user_id, e.evidence_event_id, e.sales_minutes\nFROM pick\nJOIN duck_yeah_experiment.events e USING (dataset, account_id)\nJOIN duck_yeah_experiment.accounts a USING (dataset, account_id)\nJOIN duck_yeah_experiment.assignments s USING (dataset, account_id)\nORDER BY e.dataset, e.event_ts, e.event_type;\n", "sha256": "c2249f4dc1238c5c811077c3f36e5eaca187bc11ebe1b44e388f42a8a43677ce", "elapsedMs": 61.94, "rowCount": 28, "columns": [ "dataset", "account_id", "arm", "enrollment_index", "block_index", "token", "assigned_at", "acquisition_source", "task_need", "company_size", "event_id", "event_ts", "elapsed_days", "event_type", "detail", "user_id", "evidence_event_id", "sales_minutes" ], "rows": [ { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:trial_started", "event_ts": "2026-05-01T00:22:40.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:contact_sent:0", "event_ts": "2026-05-02T00:22:40.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000004-u1", "evidence_event_id": "beneficial:account-000004:trial_started", "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:data_imported:0", "event_ts": "2026-05-02T02:22:40.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:own_data_query:0", "event_ts": "2026-05-02T03:22:40.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:contact_sent:1", "event_ts": "2026-05-04T00:22:40.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000004-u1", "evidence_event_id": "beneficial:account-000004:trial_started", "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:own_data_query:1", "event_ts": "2026-05-04T03:22:40.000Z", "elapsed_days": 3.125, "event_type": "own_data_query", "detail": "repeat", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:sqa_qualified:0", "event_ts": "2026-05-26T05:22:40.000Z", "elapsed_days": 25.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000004-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "beneficial", "account_id": "account-000004", "arm": "treatment", "enrollment_index": 3, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:22:40.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "11-50", "event_id": "beneficial:account-000004:sales_effort_logged:0", "event_ts": "2026-05-28T23:22:40.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9585 }, { "dataset": "harm", "account_id": "account-000008", "arm": "treatment", "enrollment_index": 7, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:52:55.000Z", "acquisition_source": "direct", "task_need": "sharing", "company_size": "1-10", "event_id": "harm:account-000008:trial_started", "event_ts": "2026-05-01T00:52:55.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000008-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "harm", "account_id": "account-000008", "arm": "treatment", "enrollment_index": 7, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:52:55.000Z", "acquisition_source": "direct", "task_need": "sharing", "company_size": "1-10", "event_id": "harm:account-000008:contact_sent:0", "event_ts": "2026-05-02T00:52:55.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000008-u1", "evidence_event_id": "harm:account-000008:trial_started", "sales_minutes": null }, { "dataset": "harm", "account_id": "account-000008", "arm": "treatment", "enrollment_index": 7, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:52:55.000Z", "acquisition_source": "direct", "task_need": "sharing", "company_size": "1-10", "event_id": "harm:account-000008:contact_sent:1", "event_ts": "2026-05-04T00:52:55.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000008-u1", "evidence_event_id": "harm:account-000008:trial_started", "sales_minutes": null }, { "dataset": "harm", "account_id": "account-000008", "arm": "treatment", "enrollment_index": 7, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:52:55.000Z", "acquisition_source": "direct", "task_need": "sharing", "company_size": "1-10", "event_id": "harm:account-000008:sqa_qualified:0", "event_ts": "2026-05-24T05:52:55.000Z", "elapsed_days": 23.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000008-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "harm", "account_id": "account-000008", "arm": "treatment", "enrollment_index": 7, "block_index": 0, "token": "T4", "assigned_at": "2026-05-01T00:52:55.000Z", "acquisition_source": "direct", "task_need": "sharing", "company_size": "1-10", "event_id": "harm:account-000008:sales_effort_logged:0", "event_ts": "2026-05-28T23:52:55.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 1.2111 }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:trial_started", "event_ts": "2026-05-01T01:40:48.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=sharing", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:contact_sent:0", "event_ts": "2026-05-02T01:40:48.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_sharing", "user_id": "account-000006-u1", "evidence_event_id": "insufficient_enrollment:account-000006:trial_started", "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:data_imported:0", "event_ts": "2026-05-02T03:40:48.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:own_data_query:0", "event_ts": "2026-05-02T04:40:48.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:own_data_query:1", "event_ts": "2026-05-03T04:40:48.000Z", "elapsed_days": 2.125, "event_type": "own_data_query", "detail": "repeat", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:contact_sent:1", "event_ts": "2026-05-04T01:40:48.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_sharing", "user_id": "account-000006-u1", "evidence_event_id": "insufficient_enrollment:account-000006:trial_started", "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:sqa_qualified:0", "event_ts": "2026-05-14T06:40:48.000Z", "elapsed_days": 13.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000006-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "insufficient_enrollment", "account_id": "account-000006", "arm": "treatment", "enrollment_index": 5, "block_index": 0, "token": "T2", "assigned_at": "2026-05-01T01:40:48.000Z", "acquisition_source": "product_community", "task_need": "sharing", "company_size": "1-10", "event_id": "insufficient_enrollment:account-000006:sales_effort_logged:0", "event_ts": "2026-05-29T00:40:48.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9213 }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:trial_started", "event_ts": "2026-05-01T00:45:21.000Z", "elapsed_days": 0.0, "event_type": "trial_started", "detail": "task_need=unknown", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:contact_sent:0", "event_ts": "2026-05-02T00:45:21.000Z", "elapsed_days": 1.0, "event_type": "contact_sent", "detail": "day1:help_general", "user_id": "account-000007-u1", "evidence_event_id": "upstream_only:account-000007:trial_started", "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:data_imported:0", "event_ts": "2026-05-02T02:45:21.000Z", "elapsed_days": 1.0833, "event_type": "data_imported", "detail": null, "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:own_data_query:0", "event_ts": "2026-05-02T03:45:21.000Z", "elapsed_days": 1.125, "event_type": "own_data_query", "detail": "first", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:contact_sent:1", "event_ts": "2026-05-04T00:45:21.000Z", "elapsed_days": 3.0, "event_type": "contact_sent", "detail": "day3:help_general", "user_id": "account-000007-u1", "evidence_event_id": "upstream_only:account-000007:trial_started", "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:sqa_qualified:0", "event_ts": "2026-05-17T05:45:21.000Z", "elapsed_days": 16.2083, "event_type": "sqa_qualified", "detail": "company_fit;team_use_case;next_step_agreed", "user_id": "account-000007-u1", "evidence_event_id": null, "sales_minutes": null }, { "dataset": "upstream_only", "account_id": "account-000007", "arm": "treatment", "enrollment_index": 6, "block_index": 0, "token": "T1", "assigned_at": "2026-05-01T00:45:21.000Z", "acquisition_source": "direct", "task_need": "unknown", "company_size": "1-10", "event_id": "upstream_only:account-000007:sales_effort_logged:0", "event_ts": "2026-05-28T23:45:21.000Z", "elapsed_days": 27.9583, "event_type": "sales_effort_logged", "detail": "account_total", "user_id": null, "evidence_event_id": null, "sales_minutes": 0.9516 } ], "rowsPublishedIn": null }, { "id": "baseline_volume", "sqlFile": "scripts/duck-yeah/sql/experiment_baseline_volume.sql", "sql": "-- Duck Yeah v2 · BASELINE view 1: identity, maturity and weekly eligible\n-- volume in the separately versioned SYNTHETIC history ($baseline_id).\n-- Descriptive planning input only — not MotherDuck data.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE dataset = $baseline_id),\na AS (SELECT * FROM duck_yeah_experiment.accounts WHERE dataset = $baseline_id)\nSELECT (date_diff('day', (SELECT min(assigned_at) FROM o), o.assigned_at) // 7) + 1 AS week,\n count(*) AS eligible_accounts,\n count(DISTINCT a.company_domain) AS distinct_companies,\n count(*) FILTER (WHERE o.immature) AS immature,\n count(*) FILTER (WHERE o.sqa28_status IN ('missing', 'conflict')\n OR o.activation7_status IN ('missing', 'conflict')\n OR o.optout28_status IN ('missing', 'conflict')) AS unresolved_ascertainment,\n count(*) FILTER (WHERE o.activation7_status = 'event') AS activated7,\n count(*) FILTER (WHERE o.sqa28_status = 'event') AS sqa28,\n round(sum(o.sales_minutes), 2) AS sales_minutes\nFROM o JOIN a USING (dataset, account_id)\nGROUP BY ALL ORDER BY week;\n", "sha256": "adacc531eab925ea145d39d0de6dba969e5b8bb88745f1e7d84181ffca6b8035", "elapsedMs": 53.53, "rowCount": 12, "columns": [ "week", "eligible_accounts", "distinct_companies", "immature", "unresolved_ascertainment", "activated7", "sqa28", "sales_minutes" ], "rows": [ { "week": 1, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 128, "sqa28": 40, "sales_minutes": 299.14 }, { "week": 2, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 118, "sqa28": 47, "sales_minutes": 300.68 }, { "week": 3, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 123, "sqa28": 48, "sales_minutes": 300.69 }, { "week": 4, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 127, "sqa28": 45, "sales_minutes": 300.92 }, { "week": 5, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 132, "sqa28": 44, "sales_minutes": 301.74 }, { "week": 6, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 136, "sqa28": 41, "sales_minutes": 300.16 }, { "week": 7, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 113, "sqa28": 57, "sales_minutes": 301.18 }, { "week": 8, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 128, "sqa28": 57, "sales_minutes": 299.13 }, { "week": 9, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 116, "sqa28": 55, "sales_minutes": 298.67 }, { "week": 10, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 99, "sqa28": 47, "sales_minutes": 298.95 }, { "week": 11, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 136, "sqa28": 46, "sales_minutes": 299.42 }, { "week": 12, "eligible_accounts": 500, "distinct_companies": 500, "immature": 0, "unresolved_ascertainment": 0, "activated7": 129, "sqa28": 48, "sales_minutes": 300.81 } ], "rowsPublishedIn": null }, { "id": "baseline_funnel", "sqlFile": "scripts/duck-yeah/sql/experiment_baseline_funnel.sql", "sql": "-- Duck Yeah v2 · BASELINE view 2: where accounts stall, overall and by the\n-- prespecified descriptive segments (acquisition source, declared task need,\n-- company size). All stages use every eligible account as denominator.\n-- Segment gaps are hypotheses, not causal effects or MotherDuck priorities.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE dataset = $baseline_id),\nlong AS (\n SELECT 'all' AS dimension, 'all' AS segment, * FROM o\n UNION ALL BY NAME SELECT 'acquisition_source' AS dimension, acquisition_source AS segment, * FROM o\n UNION ALL BY NAME SELECT 'task_need' AS dimension, task_need AS segment, * FROM o\n UNION ALL BY NAME SELECT 'company_size' AS dimension, company_size AS segment, * FROM o\n)\nSELECT dimension, segment,\n count(*) AS eligible,\n count(*) FILTER (WHERE imported_at IS NOT NULL) AS imported7,\n count(*) FILTER (WHERE first_query_at IS NOT NULL) AS first_query7,\n count(*) FILTER (WHERE activation7_status = 'event') AS activated7,\n count(*) FILTER (WHERE sharing7_status = 'event') AS shared7,\n count(*) FILTER (WHERE sqa28_status = 'event') AS sqa28,\n count(*) FILTER (WHERE sqa28_status = 'event' AND activation7_status = 'event') AS sqa28_among_activated,\n count(*) FILTER (WHERE optout28_status = 'event') AS optout28\nFROM long GROUP BY ALL ORDER BY dimension, segment;\n", "sha256": "252853f44d5a9113e8c5e74416047d09b1951a1d6173b891285f5cd61edd167e", "elapsedMs": 55.22, "rowCount": 10, "columns": [ "dimension", "segment", "eligible", "imported7", "first_query7", "activated7", "shared7", "sqa28", "sqa28_among_activated", "optout28" ], "rows": [ { "dimension": "acquisition_source", "segment": "content", "eligible": 1802, "imported7": 1076, "first_query7": 823, "activated7": 448, "shared7": 212, "sqa28": 178, "sqa28_among_activated": 36, "optout28": 26 }, { "dimension": "acquisition_source", "segment": "direct", "eligible": 1203, "imported7": 719, "first_query7": 538, "activated7": 292, "shared7": 144, "sqa28": 113, "sqa28_among_activated": 25, "optout28": 13 }, { "dimension": "acquisition_source", "segment": "product_community", "eligible": 2995, "imported7": 1757, "first_query7": 1325, "activated7": 745, "shared7": 398, "sqa28": 284, "sqa28_among_activated": 71, "optout28": 29 }, { "dimension": "all", "segment": "all", "eligible": 6000, "imported7": 3552, "first_query7": 2686, "activated7": 1485, "shared7": 754, "sqa28": 575, "sqa28_among_activated": 132, "optout28": 68 }, { "dimension": "company_size", "segment": "1-10", "eligible": 3014, "imported7": 1776, "first_query7": 1341, "activated7": 761, "shared7": 378, "sqa28": 295, "sqa28_among_activated": 71, "optout28": 35 }, { "dimension": "company_size", "segment": "11-50", "eligible": 2074, "imported7": 1236, "first_query7": 933, "activated7": 495, "shared7": 256, "sqa28": 195, "sqa28_among_activated": 45, "optout28": 25 }, { "dimension": "company_size", "segment": "51-200", "eligible": 912, "imported7": 540, "first_query7": 412, "activated7": 229, "shared7": 120, "sqa28": 85, "sqa28_among_activated": 16, "optout28": 8 }, { "dimension": "task_need", "segment": "import", "eligible": 2112, "imported7": 1267, "first_query7": 940, "activated7": 500, "shared7": 252, "sqa28": 189, "sqa28_among_activated": 41, "optout28": 25 }, { "dimension": "task_need", "segment": "sharing", "eligible": 2087, "imported7": 1222, "first_query7": 926, "activated7": 544, "shared7": 255, "sqa28": 207, "sqa28_among_activated": 51, "optout28": 26 }, { "dimension": "task_need", "segment": "unknown", "eligible": 1801, "imported7": 1063, "first_query7": 820, "activated7": 441, "shared7": 247, "sqa28": 179, "sqa28_among_activated": 40, "optout28": 17 } ], "rowsPublishedIn": null }, { "id": "baseline_time_to_value", "sqlFile": "scripts/duck-yeah/sql/experiment_baseline_time_to_value.sql", "sql": "-- Duck Yeah v2 · BASELINE view 3: time to value and Sales capacity.\n-- Cumulative activation / SQA by elapsed day over ALL eligible accounts\n-- (non-converters remain at risk; no converter-only medians), plus Sales\n-- minutes per account — the capacity side of the planning inputs.\nWITH o AS (SELECT * FROM duck_yeah_experiment.account_outcomes WHERE dataset = $baseline_id),\ndays AS (SELECT unnest([1, 2, 3, 5, 7, 14, 21, 28]) AS day)\nSELECT d.day,\n count(*) AS at_risk,\n count(*) FILTER (WHERE o.activation7_status = 'event' AND o.activated_at < o.assigned_at + to_days(CAST(d.day AS INTEGER))) AS activated_cum,\n count(*) FILTER (WHERE o.sqa28_status = 'event' AND o.sqa_at < o.assigned_at + to_days(CAST(d.day AS INTEGER))) AS sqa_cum,\n round(sum(o.sales_minutes) / count(*), 4) AS sales_minutes_per_account,\n round(sum(o.contacts) / count(*), 4) AS contacts_per_account\nFROM o CROSS JOIN days d\nGROUP BY d.day ORDER BY d.day;\n", "sha256": "05b8b02b9d0e6db5da9833ec32a598bea5af235d792f0ca3a710198895d0b049", "elapsedMs": 48.86, "rowCount": 8, "columns": [ "day", "at_risk", "activated_cum", "sqa_cum", "sales_minutes_per_account", "contacts_per_account" ], "rows": [ { "day": 1, "at_risk": 6000, "activated_cum": 0, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 2, "at_risk": 6000, "activated_cum": 0, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 3, "at_risk": 6000, "activated_cum": 318, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 5, "at_risk": 6000, "activated_cum": 906, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 7, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 0, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 14, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 153, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 21, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 379, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 }, { "day": 28, "at_risk": 6000, "activated_cum": 1485, "sqa_cum": 575, "sales_minutes_per_account": 0.6002, "contacts_per_account": 1.9992 } ], "rowsPublishedIn": null }, { "id": "samples", "sqlFile": "scripts/duck-yeah/sql/experiment_samples.sql", "sql": "-- Duck Yeah v2 · bounded account sample per scenario (12 accounts): for each\n-- arm, the first two accounts by enrollment order in each of three outcome\n-- patterns (SQA; activated without SQA; neither). A fixed stratified rule\n-- for inspection, not a representative or favourable selection.\nWITH o AS (\n SELECT *, CASE WHEN sqa28_status = 'event' THEN 'sqa28'\n WHEN activation7_status = 'event' THEN 'activated_no_sqa'\n ELSE 'neither' END AS pattern\n FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')\n),\nr AS (SELECT *, row_number() OVER (PARTITION BY dataset, arm, pattern ORDER BY enrollment_index) AS rn FROM o)\nSELECT dataset, account_id, arm, pattern, enrollment_index, assigned_at, acquisition_source, task_need, company_size,\n contacts, imported_at, first_query_at, activated_at, shared_at, sqa_at, optout_at,\n round(sales_minutes, 4) AS sales_minutes,\n sqa28_status, activation7_status, optout28_status, sharing7_status\nFROM r WHERE rn <= 2\nORDER BY dataset, arm, pattern, enrollment_index;\n", "sha256": "1d41dc398226130caac9c819b8522e233cdf8ab56886a819aa423bb6acda0898", "elapsedMs": 65.34, "rowCount": 48, "columns": [ "dataset", "account_id", "arm", "pattern", "enrollment_index", "assigned_at", "acquisition_source", "task_need", "company_size", "contacts", "imported_at", "first_query_at", "activated_at", "shared_at", "sqa_at", "optout_at", "sales_minutes", "sqa28_status", "activation7_status", "optout28_status", "sharing7_status" ], "rows": null, "rowsPublishedIn": "experiment-samples.json" }, { "id": "sample_events", "sqlFile": "scripts/duck-yeah/sql/experiment_sample_events.sql", "sql": "-- Duck Yeah v2 · raw events for the bounded sample accounts (same fixed\n-- rule as experiment_samples.sql).\nWITH o AS (\n SELECT *, CASE WHEN sqa28_status = 'event' THEN 'sqa28'\n WHEN activation7_status = 'event' THEN 'activated_no_sqa'\n ELSE 'neither' END AS pattern\n FROM duck_yeah_experiment.account_outcomes WHERE arm IN ('control', 'treatment')\n),\nr AS (SELECT *, row_number() OVER (PARTITION BY dataset, arm, pattern ORDER BY enrollment_index) AS rn FROM o)\nSELECT e.dataset, e.account_id, e.event_id, e.event_ts,\n round(date_diff('second', r.assigned_at, e.event_ts) / 86400.0, 4) AS elapsed_days,\n e.event_type, e.detail, e.user_id, e.evidence_event_id, e.sales_minutes\nFROM r JOIN duck_yeah_experiment.events e USING (dataset, account_id)\nWHERE r.rn <= 2\nORDER BY e.dataset, e.account_id, e.event_ts, e.event_type;\n", "sha256": "ea31d63579b29cec7d1e76896caf676cc4d8925b0fc17a5af65c88f7c0e6c40c", "elapsedMs": 65.17, "rowCount": 305, "columns": [ "dataset", "account_id", "event_id", "event_ts", "elapsed_days", "event_type", "detail", "user_id", "evidence_event_id", "sales_minutes" ], "rows": null, "rowsPublishedIn": "experiment-samples.json" } ], "limitations": [ "Every account, event, rate, effect, cost and timeline here is SYNTHETIC. Nothing is MotherDuck data or a real customer outcome.", "Scenario labels describe generative assumptions; the decision is computed afterwards by the frozen rule and may differ from the label.", "The three 8,000-account cases assume more eligible volume than the synthetic baseline supplies (≈500/week → ≈16 weeks for 8,000).", "Sales minutes are modeled per account from scenario means (protocol costs.salesMinutes); the synthetic metadata carries no explicit team-evaluation evidence, so no Sales invitations are generated.", "Treatment help selection uses only the task need declared at trial start (evidence available before each contact); it does not use Jev or Clay. Jev/Clay are not claimed on these cases.", "Contact send time (hour 0 of elapsed days 1 and 3) is an implementation detail the protocol leaves open; it affects only contact counts and delivery cost.", "Costs are invented allocations from the protocol, not vendor prices or measured charges; measured execution costs belong in separate receipts.", "Segment cuts are prespecified but descriptive; none can replace the registered primary population.", "Operational fixtures (real company facts, Clay, test Sales queue) are root-owned and are not part of this artifact or its denominators." ], "receipts": [ { "id": "run:f1d1083aefb0--motherduck--20260929T143732Z", "kind": "experiment_run", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:32.551Z", "finishedAt": "2026-09-29T14:37:51.051Z", "sourcePath": "scripts/duck-yeah/experiment.py", "sourceHash": "152ef3b9b3d2e350f7a1bc28998e39b5b5d1f3c28bb855adf1086fa6374ee78b", "scope": "Local synthetic generation; load and query rows in MotherDuck; local confidence intervals and decision rule", "cost": null, "limitations": [ "Measured provider charges are not captured here." ], "executedSourceSnapshot": "/duck-yeah/history/experiment-pipeline-v2.1-run.py", "metadataCorrection": { "correctedAt": "2026-09-29T14:39:05.608241Z", "originalScope": "Generate, load and analyse synthetic rows on MotherDuck", "reason": "Clarify execution hosts; generation and statistical decision run locally, while table loads and all SQL queries run in MotherDuck. Counts, estimates, timestamps and execution verification are unchanged.", "originalArtifactSha256": "0b3d09c1b99ab74a468d3cda8eb1529d9bdfefcf79ad747b0a1b0466bc2cba38" } }, { "id": "load:accounts", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:40.742Z", "finishedAt": "2026-09-29T14:37:41.123Z", "elapsedMs": 380.8, "sourcePath": "experiment_generate.py → duck_yeah_experiment.accounts", "sourceHash": "df73d4e651b913853819e9f8c1ae06d3a88a3fa9b7f4da3a1eb873593920900d", "rowCount": 33000, "scope": "Synthetic accounts rows loaded into duck_yeah_experiment.accounts", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "load:assignments", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:41.454Z", "finishedAt": "2026-09-29T14:37:41.699Z", "elapsedMs": 245.42, "sourcePath": "experiment_generate.py → duck_yeah_experiment.assignments", "sourceHash": "67dcec57e89b019971ed2b0c6d09d1c09260e99080493e0dcfe57c249f5717be", "rowCount": 27000, "scope": "Synthetic assignments rows loaded into duck_yeah_experiment.assignments", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "load:users", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:41.841Z", "finishedAt": "2026-09-29T14:37:42.104Z", "elapsedMs": 263.81, "sourcePath": "experiment_generate.py → duck_yeah_experiment.users", "sourceHash": "2b412e2e7e8efd1c2155cb1294df38c64c18758533513c2c917e480380a130f1", "rowCount": 37845, "scope": "Synthetic users rows loaded into duck_yeah_experiment.users", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "load:events", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:44.656Z", "finishedAt": "2026-09-29T14:37:45.707Z", "elapsedMs": 1050.76, "sourcePath": "experiment_generate.py → duck_yeah_experiment.events", "sourceHash": "b3c24f015e2f76215deaaf441305126ed6455f54020bcdf37f07fed88bb8096f", "rowCount": 184678, "scope": "Synthetic events rows loaded into duck_yeah_experiment.events", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "load:contact_opportunities", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:46.555Z", "finishedAt": "2026-09-29T14:37:47.034Z", "elapsedMs": 478.99, "sourcePath": "experiment_generate.py → duck_yeah_experiment.contact_opportunities", "sourceHash": "7c97d4c7d8f775ce0e637cb0b19605162c6228aa432c7da4349155cad16956ef", "rowCount": 66000, "scope": "Synthetic contact_opportunities rows loaded into duck_yeah_experiment.contact_opportunities", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "load:ascertainment", "kind": "dataset_load", "status": "completed", "startedAt": "2026-09-29T14:37:48.245Z", "finishedAt": "2026-09-29T14:37:48.848Z", "elapsedMs": 603.7, "sourcePath": "experiment_generate.py → duck_yeah_experiment.ascertainment", "sourceHash": "ca148426d337388301a02424fb0dc88c527d8ff11bc283f317e4eaef8cfed21c", "rowCount": 132000, "scope": "Synthetic ascertainment rows loaded into duck_yeah_experiment.ascertainment", "cost": null, "limitations": [ "Synthetic rows; loaded from a private local CSV whose path is not published." ], "runId": "f1d1083aefb0--motherduck--20260929T143732Z" }, { "id": "query:derive_account_outcomes", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:48.902Z", "finishedAt": "2026-09-29T14:37:49.693Z", "elapsedMs": 791.24, "sourcePath": "scripts/duck-yeah/sql/experiment_derive_account_outcomes.sql", "sourceHash": "35c8e0dfcefd89e2b74b296a6645c3545b48b9c62573f83eb48c29062da90d20", "rowCount": 1, "scope": "Builds one row per account from raw rows (all datasets).", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:integrity", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:49.694Z", "finishedAt": "2026-09-29T14:37:50.205Z", "elapsedMs": 510.52, "sourcePath": "scripts/duck-yeah/sql/experiment_integrity.sql", "sourceHash": "f2fdbe25acef778aaf12f9c82c1a4a2960243c1297d378673ac63456afd81072", "rowCount": 96, "scope": "Zero-tolerance integrity checks against the assignment ledger, incl. SQL replay of the block shuffle.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:maturity", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.206Z", "finishedAt": "2026-09-29T14:37:50.274Z", "elapsedMs": 67.95, "sourcePath": "scripts/duck-yeah/sql/experiment_maturity.sql", "sourceHash": "b7c92a926f7bb08494de4d18000bcb953793fa9be34f841015cff45baf9e1088", "rowCount": 4, "scope": "Enrollment span, arm sizes, blocks and maturity at the final-analysis cutoff.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:metrics", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.275Z", "finishedAt": "2026-09-29T14:37:50.348Z", "elapsedMs": 72.99, "sourcePath": "scripts/duck-yeah/sql/experiment_metrics.sql", "sourceHash": "7f51f37a1e27bb7fd7a90d1f9b1399bdd6645680cce3e12d6c74276fd54c0e06", "rowCount": 40, "scope": "ITT counts per scenario × arm × outcome, with confirmed zeros, missing, pending and conflicts.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:funnel", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.348Z", "finishedAt": "2026-09-29T14:37:50.396Z", "elapsedMs": 47.4, "sourcePath": "scripts/duck-yeah/sql/experiment_funnel.sql", "sourceHash": "eaa08e547c57654b2c58c7c33a7d2fbda7de05f7f753f2ab56780916bb9f4980", "rowCount": 8, "scope": "Account funnel per scenario × arm over all assigned accounts.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:cohort_curves", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.396Z", "finishedAt": "2026-09-29T14:37:50.552Z", "elapsedMs": 154.84, "sourcePath": "scripts/duck-yeah/sql/experiment_cohort_curves.sql", "sourceHash": "85b956f419f667091cef8786091e31cc534a7525605bde5721bcebe2e4a2e1e4", "rowCount": 232, "scope": "Cumulative activation/SQA by elapsed day over all assigned accounts.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:segments", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.552Z", "finishedAt": "2026-09-29T14:37:50.625Z", "elapsedMs": 72.49, "sourcePath": "scripts/duck-yeah/sql/experiment_segments.sql", "sourceHash": "6fc55a1d28c72b0ec11cc6485edafbc27c2c158a7835a0ed49e440d47acc4038", "rowCount": 72, "scope": "Prespecified descriptive segment cuts.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:costs", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.626Z", "finishedAt": "2026-09-29T14:37:50.686Z", "elapsedMs": 59.83, "sourcePath": "scripts/duck-yeah/sql/experiment_costs.sql", "sourceHash": "e9831b3f1aba0bd4e3e021b3f48468486c4440f42c0171c6043945eb45a48bf5", "rowCount": 8, "scope": "Assumed synthetic cost allocation per scenario × arm.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:trace", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.686Z", "finishedAt": "2026-09-29T14:37:50.748Z", "elapsedMs": 61.94, "sourcePath": "scripts/duck-yeah/sql/experiment_trace.sql", "sourceHash": "c2249f4dc1238c5c811077c3f36e5eaca187bc11ebe1b44e388f42a8a43677ce", "rowCount": 28, "scope": "One fixed-rule example account trace per scenario.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:baseline_volume", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.749Z", "finishedAt": "2026-09-29T14:37:50.803Z", "elapsedMs": 53.53, "sourcePath": "scripts/duck-yeah/sql/experiment_baseline_volume.sql", "sourceHash": "adacc531eab925ea145d39d0de6dba969e5b8bb88745f1e7d84181ffca6b8035", "rowCount": 12, "scope": "Baseline: identity, maturity and weekly eligible volume.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:baseline_funnel", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.803Z", "finishedAt": "2026-09-29T14:37:50.859Z", "elapsedMs": 55.22, "sourcePath": "scripts/duck-yeah/sql/experiment_baseline_funnel.sql", "sourceHash": "252853f44d5a9113e8c5e74416047d09b1951a1d6173b891285f5cd61edd167e", "rowCount": 10, "scope": "Baseline: funnel stalls overall and by descriptive segment.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:baseline_time_to_value", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.859Z", "finishedAt": "2026-09-29T14:37:50.908Z", "elapsedMs": 48.86, "sourcePath": "scripts/duck-yeah/sql/experiment_baseline_time_to_value.sql", "sourceHash": "05b8b02b9d0e6db5da9833ec32a598bea5af235d792f0ca3a710198895d0b049", "rowCount": 8, "scope": "Baseline: time to value and Sales capacity.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:samples", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.909Z", "finishedAt": "2026-09-29T14:37:50.974Z", "elapsedMs": 65.34, "sourcePath": "scripts/duck-yeah/sql/experiment_samples.sql", "sourceHash": "1d41dc398226130caac9c819b8522e233cdf8ab56886a819aa423bb6acda0898", "rowCount": 48, "scope": "Bounded 12-account sample per scenario.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] }, { "id": "query:sample_events", "kind": "sql_query", "status": "completed", "runId": "f1d1083aefb0--motherduck--20260929T143732Z", "startedAt": "2026-09-29T14:37:50.975Z", "finishedAt": "2026-09-29T14:37:51.040Z", "elapsedMs": 65.17, "sourcePath": "scripts/duck-yeah/sql/experiment_sample_events.sql", "sourceHash": "ea31d63579b29cec7d1e76896caf676cc4d8925b0fc17a5af65c88f7c0e6c40c", "rowCount": 305, "scope": "Raw events for the sample accounts.", "cost": null, "limitations": [ "Runs over synthetic rows only.", "Provider charges, if any, are not measured by this receipt." ] } ] }