[{"data":1,"prerenderedAt":1008},["ShallowReactive",2],{"navigation_docs":3,"-docs-operations-failure-and-recovery":356,"-docs-operations-failure-and-recovery-surround":1003},[4,8,34,64,90,139,157,171,193,227,241,266,312,334],{"title":5,"path":6,"stem":7},"Documentation","\u002Fdocs","docs\u002Findex",{"title":9,"path":10,"stem":11,"children":12},"Getting started","\u002Fdocs\u002Fgetting-started","docs\u002F01.getting-started\u002Findex",[13,14,18,22,26,30],{"title":9,"path":10,"stem":11},{"title":15,"path":16,"stem":17},"Introduction","\u002Fdocs\u002Fgetting-started\u002Fintroduction","docs\u002F01.getting-started\u002F01.introduction",{"title":19,"path":20,"stem":21},"Concepts","\u002Fdocs\u002Fgetting-started\u002Fconcepts","docs\u002F01.getting-started\u002F02.concepts",{"title":23,"path":24,"stem":25},"Quickstart (5 minutes)","\u002Fdocs\u002Fgetting-started\u002Fquickstart","docs\u002F01.getting-started\u002F03.quickstart",{"title":27,"path":28,"stem":29},"Local development","\u002Fdocs\u002Fgetting-started\u002Flocal-development","docs\u002F01.getting-started\u002F04.local-development",{"title":31,"path":32,"stem":33},"FAQ","\u002Fdocs\u002Fgetting-started\u002Ffaq","docs\u002F01.getting-started\u002F05.faq",{"title":35,"path":36,"stem":37,"children":38},"Architecture","\u002Fdocs\u002Farchitecture","docs\u002F02.architecture\u002Findex",[39,40,44,48,52,56,60],{"title":35,"path":36,"stem":37},{"title":41,"path":42,"stem":43},"Architecture overview","\u002Fdocs\u002Farchitecture\u002Foverview","docs\u002F02.architecture\u002F01.overview",{"title":45,"path":46,"stem":47},"Services","\u002Fdocs\u002Farchitecture\u002Fservices","docs\u002F02.architecture\u002F02.services",{"title":49,"path":50,"stem":51},"Request lifecycle","\u002Fdocs\u002Farchitecture\u002Frequest-lifecycle","docs\u002F02.architecture\u002F03.request-lifecycle",{"title":53,"path":54,"stem":55},"Data model","\u002Fdocs\u002Farchitecture\u002Fdata-model","docs\u002F02.architecture\u002F04.data-model",{"title":57,"path":58,"stem":59},"Status and evidence","\u002Fdocs\u002Farchitecture\u002Fstatus-and-evidence","docs\u002F02.architecture\u002F05.status-and-evidence",{"title":61,"path":62,"stem":63},"Target architecture and authority","\u002Fdocs\u002Farchitecture\u002Ftarget-architecture","docs\u002F02.architecture\u002F06.target-architecture",{"title":65,"path":66,"stem":67,"children":68},"Auth","\u002Fdocs\u002Fauth","docs\u002F03.auth\u002Findex",[69,70,74,78,82,86],{"title":65,"path":66,"stem":67},{"title":71,"path":72,"stem":73},"Overview","\u002Fdocs\u002Fauth\u002Foverview","docs\u002F03.auth\u002F01.overview",{"title":75,"path":76,"stem":77},"Organization and agent model","\u002Fdocs\u002Fauth\u002Forg-project-model","docs\u002F03.auth\u002F02.org-project-model",{"title":79,"path":80,"stem":81},"Issuing API keys","\u002Fdocs\u002Fauth\u002Fissuing-api-keys","docs\u002F03.auth\u002F03.issuing-api-keys",{"title":83,"path":84,"stem":85},"Multi-tenant RLS","\u002Fdocs\u002Fauth\u002Fmulti-tenant-rls","docs\u002F03.auth\u002F04.multi-tenant-rls",{"title":87,"path":88,"stem":89},"Operator sessions and step-up","\u002Fdocs\u002Fauth\u002Foperator-sessions","docs\u002F03.auth\u002F05.operator-sessions",{"title":91,"path":92,"stem":93,"children":94},"Proxy","\u002Fdocs\u002Fproxy","docs\u002F04.proxy\u002Findex",[95,96,99,103,107,111,115,119,123,127,131,135],{"title":91,"path":92,"stem":93},{"title":71,"path":97,"stem":98},"\u002Fdocs\u002Fproxy\u002Foverview","docs\u002F04.proxy\u002F01.overview",{"title":100,"path":101,"stem":102},"Authentication","\u002Fdocs\u002Fproxy\u002Fauthentication","docs\u002F04.proxy\u002F02.authentication",{"title":104,"path":105,"stem":106},"Request routing","\u002Fdocs\u002Fproxy\u002Frequest-routing","docs\u002F04.proxy\u002F03.request-routing",{"title":108,"path":109,"stem":110},"Sessions","\u002Fdocs\u002Fproxy\u002Fsessions","docs\u002F04.proxy\u002F04.sessions",{"title":112,"path":113,"stem":114},"Directives","\u002Fdocs\u002Fproxy\u002Fdirectives","docs\u002F04.proxy\u002F05.directives",{"title":116,"path":117,"stem":118},"Idempotency and budgets","\u002Fdocs\u002Fproxy\u002Fidempotency-and-budgets","docs\u002F04.proxy\u002F06.idempotency-and-budgets",{"title":120,"path":121,"stem":122},"Rate limiting","\u002Fdocs\u002Fproxy\u002Frate-limiting","docs\u002F04.proxy\u002F07.rate-limiting",{"title":124,"path":125,"stem":126},"Provider adapters","\u002Fdocs\u002Fproxy\u002Fprovider-adapters","docs\u002F04.proxy\u002F08.provider-adapters",{"title":128,"path":129,"stem":130},"Auth caching","\u002Fdocs\u002Fproxy\u002Fauth-caching","docs\u002F04.proxy\u002F09.auth-caching",{"title":132,"path":133,"stem":134},"Configuration","\u002Fdocs\u002Fproxy\u002Fconfiguration","docs\u002F04.proxy\u002F10.configuration",{"title":136,"path":137,"stem":138},"Provider routing and resilience","\u002Fdocs\u002Fproxy\u002Fprovider-routing-and-resilience","docs\u002F04.proxy\u002F11.provider-routing-and-resilience",{"title":140,"path":141,"stem":142,"children":143},"Memory","\u002Fdocs\u002Fmemory","docs\u002F05.memory\u002Findex",[144,145,149,153],{"title":140,"path":141,"stem":142},{"title":146,"path":147,"stem":148},"Memory overview","\u002Fdocs\u002Fmemory\u002Foverview","docs\u002F05.memory\u002F01.overview",{"title":150,"path":151,"stem":152},"Write path","\u002Fdocs\u002Fmemory\u002Fwrite-path","docs\u002F05.memory\u002F02.write-path",{"title":154,"path":155,"stem":156},"Search and lifecycle","\u002Fdocs\u002Fmemory\u002Fsearch-and-lifecycle","docs\u002F05.memory\u002F03.search-and-lifecycle",{"title":158,"path":159,"stem":160,"children":161},"Context","\u002Fdocs\u002Fcontext","docs\u002F06.context\u002Findex",[162,163,167],{"title":158,"path":159,"stem":160},{"title":164,"path":165,"stem":166},"Assembly pipeline","\u002Fdocs\u002Fcontext\u002Foverview","docs\u002F06.context\u002F01.overview",{"title":168,"path":169,"stem":170},"Ranking and packing","\u002Fdocs\u002Fcontext\u002Franking-and-packing","docs\u002F06.context\u002F02.ranking-and-packing",{"title":172,"path":173,"stem":174,"children":175},"Intelligence","\u002Fdocs\u002Fintelligence","docs\u002F07.intelligence\u002Findex",[176,177,181,185,189],{"title":172,"path":173,"stem":174},{"title":178,"path":179,"stem":180},"Workers","\u002Fdocs\u002Fintelligence\u002Fworkers","docs\u002F07.intelligence\u002F01.workers",{"title":182,"path":183,"stem":184},"Extraction","\u002Fdocs\u002Fintelligence\u002Fextraction","docs\u002F07.intelligence\u002F02.extraction",{"title":186,"path":187,"stem":188},"Embedder","\u002Fdocs\u002Fintelligence\u002Fembedder","docs\u002F07.intelligence\u002F03.embedder",{"title":190,"path":191,"stem":192},"Extraction evaluation","\u002Fdocs\u002Fintelligence\u002Fevaluation","docs\u002F07.intelligence\u002F04.evaluation",{"title":194,"path":195,"stem":196,"children":197},"API reference","\u002Fdocs\u002Fapi-reference","docs\u002F08.api-reference\u002Findex",[198,199,203,207,211,215,219,223],{"title":194,"path":195,"stem":196},{"title":200,"path":201,"stem":202},"Chat completions","\u002Fdocs\u002Fapi-reference\u002Fchat-completions","docs\u002F08.api-reference\u002F01.chat-completions",{"title":204,"path":205,"stem":206},"Responses","\u002Fdocs\u002Fapi-reference\u002Fresponses","docs\u002F08.api-reference\u002F02.responses",{"title":208,"path":209,"stem":210},"Errors","\u002Fdocs\u002Fapi-reference\u002Ferrors","docs\u002F08.api-reference\u002F03.errors",{"title":212,"path":213,"stem":214},"Health and metrics","\u002Fdocs\u002Fapi-reference\u002Fproxy-health","docs\u002F08.api-reference\u002F04.proxy-health",{"title":216,"path":217,"stem":218},"HTTP route inventory","\u002Fdocs\u002Fapi-reference\u002Fhttp-routes","docs\u002F08.api-reference\u002F05.http-routes",{"title":220,"path":221,"stem":222},"Management API","\u002Fdocs\u002Fapi-reference\u002Fmanagement-api","docs\u002F08.api-reference\u002F06.management-api",{"title":224,"path":225,"stem":226},"Auth gRPC (ValidateToken, ValidateAgent)","\u002Fdocs\u002Fapi-reference\u002Fauth-grpc","docs\u002F08.api-reference\u002F07.auth-grpc",{"title":228,"path":229,"stem":230,"children":231},"MCP","\u002Fdocs\u002Fmcp","docs\u002F09.mcp\u002Findex",[232,233,237],{"title":228,"path":229,"stem":230},{"title":234,"path":235,"stem":236},"Memory server","\u002Fdocs\u002Fmcp\u002Fmemory-server","docs\u002F09.mcp\u002F01.memory-server",{"title":238,"path":239,"stem":240},"Operator gateway","\u002Fdocs\u002Fmcp\u002Foperator-gateway","docs\u002F09.mcp\u002F02.operator-gateway",{"title":242,"path":243,"stem":244,"children":245},"Security","\u002Fdocs\u002Fsecurity","docs\u002F10.security\u002Findex",[246,247,251,254,258,262],{"title":242,"path":243,"stem":244},{"title":248,"path":249,"stem":250},"Security overview","\u002Fdocs\u002Fsecurity\u002Foverview","docs\u002F10.security\u002F01.overview",{"title":100,"path":252,"stem":253},"\u002Fdocs\u002Fsecurity\u002Fauthentication","docs\u002F10.security\u002F02.authentication",{"title":255,"path":256,"stem":257},"Tenant isolation","\u002Fdocs\u002Fsecurity\u002Ftenant-isolation","docs\u002F10.security\u002F03.tenant-isolation",{"title":259,"path":260,"stem":261},"Secrets and keys","\u002Fdocs\u002Fsecurity\u002Fsecrets-and-keys","docs\u002F10.security\u002F04.secrets-and-keys",{"title":263,"path":264,"stem":265},"Prompt injection","\u002Fdocs\u002Fsecurity\u002Fprompt-injection","docs\u002F10.security\u002F05.prompt-injection",{"title":267,"path":268,"stem":269,"children":270},"Operations","\u002Fdocs\u002Foperations","docs\u002F11.operations\u002Findex",[271,272,276,280,284,288,292,296,300,304,308],{"title":267,"path":268,"stem":269},{"title":273,"path":274,"stem":275},"Observability","\u002Fdocs\u002Foperations\u002Fobservability","docs\u002F11.operations\u002F01.observability",{"title":277,"path":278,"stem":279},"Traces and ClickHouse","\u002Fdocs\u002Foperations\u002Ftraces-and-clickhouse","docs\u002F11.operations\u002F02.traces-and-clickhouse",{"title":281,"path":282,"stem":283},"Health checks","\u002Fdocs\u002Foperations\u002Fhealth-checks","docs\u002F11.operations\u002F03.health-checks",{"title":285,"path":286,"stem":287},"Incident response","\u002Fdocs\u002Foperations\u002Fincident-response","docs\u002F11.operations\u002F04.incident-response",{"title":289,"path":290,"stem":291},"Troubleshooting","\u002Fdocs\u002Foperations\u002Ftroubleshooting","docs\u002F11.operations\u002F05.troubleshooting",{"title":293,"path":294,"stem":295},"Billing and budgets","\u002Fdocs\u002Foperations\u002Fbilling-and-budgets","docs\u002F11.operations\u002F06.billing-and-budgets",{"title":297,"path":298,"stem":299},"Evidence and relay","\u002Fdocs\u002Foperations\u002Fevidence-and-relay","docs\u002F11.operations\u002F07.evidence-and-relay",{"title":301,"path":302,"stem":303},"Console","\u002Fdocs\u002Foperations\u002Fconsole","docs\u002F11.operations\u002F08.console",{"title":305,"path":306,"stem":307},"Failure modes and recovery","\u002Fdocs\u002Foperations\u002Ffailure-and-recovery","docs\u002F11.operations\u002F09.failure-and-recovery",{"title":309,"path":310,"stem":311},"Data lifecycle and deletion","\u002Fdocs\u002Foperations\u002Fdata-lifecycle-and-deletion","docs\u002F11.operations\u002F10.data-lifecycle-and-deletion",{"title":313,"path":314,"stem":315,"children":316},"Deployment","\u002Fdocs\u002Fdeployment","docs\u002F12.deployment\u002Findex",[317,318,322,326,330],{"title":313,"path":314,"stem":315},{"title":319,"path":320,"stem":321},"Docker Compose (dev)","\u002Fdocs\u002Fdeployment\u002Fdocker-compose","docs\u002F12.deployment\u002F01.docker-compose",{"title":323,"path":324,"stem":325},"Environment variables","\u002Fdocs\u002Fdeployment\u002Fenvironment-variables","docs\u002F12.deployment\u002F02.environment-variables",{"title":327,"path":328,"stem":329},"Kubernetes","\u002Fdocs\u002Fdeployment\u002Fkubernetes","docs\u002F12.deployment\u002F03.kubernetes",{"title":331,"path":332,"stem":333},"Production readiness","\u002Fdocs\u002Fdeployment\u002Fproduction-readiness","docs\u002F12.deployment\u002F04.production-readiness",{"title":335,"path":336,"stem":337,"children":338},"Reference","\u002Fdocs\u002Freference","docs\u002F13.reference\u002Findex",[339,340,344,348,352],{"title":335,"path":336,"stem":337},{"title":341,"path":342,"stem":343},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","docs\u002F13.reference\u002F01.glossary",{"title":345,"path":346,"stem":347},"Roadmap and gates","\u002Fdocs\u002Freference\u002Froadmap","docs\u002F13.reference\u002F02.roadmap",{"title":349,"path":350,"stem":351},"Architecture decisions","\u002Fdocs\u002Freference\u002Fadr","docs\u002F13.reference\u002F03.adr",{"title":353,"path":354,"stem":355},"Compatibility and versioning","\u002Fdocs\u002Freference\u002Fcompatibility","docs\u002F13.reference\u002F04.compatibility",{"id":357,"title":305,"audience":358,"author":359,"authorUrl":359,"body":360,"coverIndex":359,"coverLabel":359,"coverTone":359,"date":359,"decisionStatus":359,"description":995,"excerpt":359,"extension":996,"featured":997,"historical":997,"meta":998,"navigation":904,"path":306,"readingTime":359,"scope":999,"seo":1000,"sourceOfTruth":1001,"status":367,"stem":307,"tags":359,"verifiedDate":368,"__hash__":1002},"docs\u002Fdocs\u002F11.operations\u002F09.failure-and-recovery.md","operator",null,{"type":361,"value":362,"toc":977},"minimark",[363,369,382,387,520,524,528,621,624,627,640,644,647,651,654,658,713,716,720,723,805,808,812,815,825,828,835,839,842,861,864,868,889,893,940,944],[364,365],"page-status",{"note":366,"status":367,"verified":368},"Failure behavior is implemented in bounded local paths; restore, RPO\u002FRTO, and HA drills are not accepted","PROVISIONAL","2026-10-21",[370,371,372,373,377,378,381],"p",{},"IBEX separates ",[374,375,376],"strong",{},"authority failures"," from ",[374,379,380],{},"quality failures",". Authority failures deny or return a typed dependency error. Quality failures may degrade, queue, or abstain, but cannot broaden visibility or invent a successful mutation.",[383,384,386],"h2",{"id":385},"failure-classes","Failure classes",[388,389,390,406],"table",{},[391,392,393],"thead",{},[394,395,396,400,403],"tr",{},[397,398,399],"th",{},"Dependency",[397,401,402],{},"Class",[397,404,405],{},"Required behavior",[407,408,409,421,432,443,454,465,476,487,498,509],"tbody",{},[394,410,411,415,418],{},[412,413,414],"td",{},"Auth, principal, policy",[412,416,417],{},"Authority",[412,419,420],{},"Deny or typed 503; no provider, context, or tool work",[394,422,423,426,429],{},[412,424,425],{},"Budget\u002Freservation",[412,427,428],{},"Authority\u002Fintegrity",[412,430,431],{},"Deny or 503; never claim a hard budget that was not held",[394,433,434,437,440],{},[412,435,436],{},"Redis on a protected path",[412,438,439],{},"Integrity\u002Fcoordination",[412,441,442],{},"503 under the protected profile; no permissive fallback",[394,444,445,448,451],{},[412,446,447],{},"Context retrieval",[412,449,450],{},"Quality",[412,452,453],{},"Directive-only or verified hot-cache context; never wider scope",[394,455,456,459,462],{},[412,457,458],{},"Embedder\u002Freranker\u002Fmodel",[412,460,461],{},"Quality\u002Fasync",[412,463,464],{},"Queue, fallback, quarantine, or abstain",[394,466,467,470,473],{},[412,468,469],{},"Provider",[412,471,472],{},"External",[412,474,475],{},"Typed error, bounded retry, breaker, optional policy-compatible fallback",[394,477,478,481,484],{},[412,479,480],{},"Evidence outbox",[412,482,483],{},"Integrity",[412,485,486],{},"Durable buffer or explicit uncertified outcome; never silent loss",[394,488,489,492,495],{},[412,490,491],{},"ClickHouse\u002FOTel",[412,493,494],{},"Projection",[412,496,497],{},"Buffer or degraded analytics; canonical evidence remains PostgreSQL",[394,499,500,503,506],{},[412,501,502],{},"Object storage",[412,504,505],{},"Artifact",[412,507,508],{},"Block artifact-dependent action or use a verified retained reference",[394,510,511,514,517],{},[412,512,513],{},"Queue\u002Fworker",[412,515,516],{},"Async",[412,518,519],{},"Retry, dead-letter, replay, and visible operation status",[383,521,523],{"id":522},"service-level-behavior","Service-level behavior",[525,526,91],"h3",{"id":527},"proxy",[388,529,530,540],{},[391,531,532],{},[394,533,534,537],{},[397,535,536],{},"Failure",[397,538,539],{},"Behavior",[407,541,542,557,570,580,590,598,606],{},[394,543,544,551],{},[412,545,546,550],{},[547,548,549],"code",{},"ValidateToken"," timeout or outage",[412,552,553,556],{},[547,554,555],{},"503 SERVICE_DEGRADED","; provider is not called",[394,558,559,565],{},[412,560,561,564],{},[547,562,563],{},"ValidateAgent"," outage",[412,566,567,556],{},[547,568,569],{},"503 AUTH_UNAVAILABLE",[394,571,572,575],{},[412,573,574],{},"Rate-limit Redis failure outside development",[412,576,577,579],{},[547,578,555],{}," before provider work",[394,581,582,585],{},[412,583,584],{},"Org model-policy load failure",[412,586,587,589],{},[547,588,555],{},"; no ungoverned route",[394,591,592,595],{},[412,593,594],{},"Context assembly timeout",[412,596,597],{},"Proceed with directive-only or history-only context and record fallback",[394,599,600,603],{},[412,601,602],{},"ClickHouse unavailable",[412,604,605],{},"Drop or buffer traces; chat continues",[394,607,608,611],{},[412,609,610],{},"Provider breaker open",[412,612,613,616,617,620],{},[547,614,615],{},"503"," with ",[547,618,619],{},"Retry-After","; optional eligible fallback",[370,622,623],{},"The proxy must never call a provider after a security, policy, capability, reservation, or evidence-integrity denial.",[525,625,65],{"id":626},"auth",[370,628,629,632,633,636,637,639],{},[547,630,631],{},"\u002Fready"," checks PostgreSQL and the gRPC listener. A missing policy-signing key makes ",[547,634,635],{},"IssuePolicySnapshot"," fail closed outside development. ",[547,638,549],{}," has no caller bearer and must remain on a trusted private network. If Redis is unavailable, revocation propagation and the auth cache are not silently treated as healthy.",[525,641,643],{"id":642},"memory-and-context","Memory and context",[370,645,646],{},"Memory requires PostgreSQL\u002Fpgvector for durable reads and keyed writes; the embedder is required for vector writes. Redis is optional for hot-cache behavior. Context is a quality dependency: its failure must degrade to directive-only, not block inference.",[525,648,650],{"id":649},"workers-and-evidence","Workers and evidence",[370,652,653],{},"Extraction, embeddings, deletion, billing reconciliation, and audit tasks are asynchronous. Queue delivery is not exactly-once. Every mutation needs an operation ID and idempotent effect. The relay is at-least-once by design; a projection cannot be treated as the authority for deletion or acknowledgement.",[383,655,657],{"id":656},"readiness-versus-liveness","Readiness versus liveness",[388,659,660,673],{},[391,661,662],{},[394,663,664,667,670],{},[397,665,666],{},"Endpoint",[397,668,669],{},"Meaning",[397,671,672],{},"Must not mean",[407,674,675,688,700],{},[394,676,677,682,685],{},[412,678,679],{},[547,680,681],{},"\u002Fhealth",[412,683,684],{},"Process is alive",[412,686,687],{},"Dependencies are usable",[394,689,690,694,697],{},[412,691,692],{},[547,693,631],{},[412,695,696],{},"Selected profile's required dependencies are available",[412,698,699],{},"Every optional quality service is healthy",[394,701,702,707,710],{},[412,703,704],{},[547,705,706],{},"\u002Fmetrics",[412,708,709],{},"Telemetry is exposed",[412,711,712],{},"A measured SLO exists",[370,714,715],{},"A process may be live and not ready. Readiness must not report healthy when security or evidence integrity is unavailable. Optional quality dependencies may be degraded and visibly reported.",[383,717,719],{"id":718},"recovery-classes","Recovery classes",[370,721,722],{},"Recovery is planned per data class, not as one database restore:",[388,724,725,737],{},[391,726,727],{},[394,728,729,731,734],{},[397,730,402],{},[397,732,733],{},"Examples",[397,735,736],{},"Recovery rule",[407,738,739,750,761,772,783,794],{},[394,740,741,744,747],{},[412,742,743],{},"Canonical control",[412,745,746],{},"Organizations, users, agents, tokens, policies, reservations, operations, lifecycle",[412,748,749],{},"Restore first; authority must be consistent before serving",[394,751,752,755,758],{},[412,753,754],{},"Evidence\u002Foutbox",[412,756,757],{},"Runs, events, spans, outbox rows",[412,759,760],{},"Restore with canonical state; replay must not resurrect tombstoned resources",[394,762,763,766,769],{},[412,764,765],{},"Object artifacts",[412,767,768],{},"Redacted payloads, archives, model artifacts",[412,770,771],{},"Restore with metadata and retention\u002Flegal-hold state",[394,773,774,777,780],{},[412,775,776],{},"Redis projections",[412,778,779],{},"Cache, hot memories, rate counters, leases",[412,781,782],{},"Rebuild; never treat as durable truth",[394,784,785,788,791],{},[412,786,787],{},"Vector\u002Fsearch projections",[412,789,790],{},"Embeddings, HNSW, full-text",[412,792,793],{},"Rebuild from canonical rows and pinned embedding profile",[394,795,796,799,802],{},[412,797,798],{},"Queues",[412,800,801],{},"Extraction and audit work",[412,803,804],{},"Replay only with idempotent effects; otherwise reconcile operation state",[370,806,807],{},"A cache or index may be discarded and rebuilt. A deletion tombstone may not be discarded to “repair” a projection.",[383,809,811],{"id":810},"migrations","Migrations",[370,813,814],{},"The target sequence is:",[816,817,823],"pre",{"className":818,"code":820,"language":821,"meta":822},[819],"language-text","expand → dual-read\u002Fwrite → backfill → verify → contract\n","text","",[547,824,820],{"__ignoreMap":822},[370,826,827],{},"Every migration needs lock and statement timeouts, throttling\u002Fbackpressure, restart\u002Fresume behavior, abort criteria, and a roll-forward or rollback decision. Mixed-version compatibility must be tested. Embedding or tokenizer profile changes are versioned migrations, never silent replacement of an index.",[370,829,830,831,834],{},"The repository currently has PostgreSQL migrations through ",[547,832,833],{},"000047_relay_fence_checker",". Migration presence is not restore or HA evidence.",[383,836,838],{"id":837},"shutdown-and-drain","Shutdown and drain",[370,840,841],{},"On shutdown, services should:",[843,844,845,849,852,855,858],"ol",{},[846,847,848],"li",{},"Stop accepting new admissions.",[846,850,851],{},"Drain in-flight streams within the configured deadline.",[846,853,854],{},"Cancel bounded work that cannot finish.",[846,856,857],{},"Persist worker state and release or expire leases.",[846,859,860],{},"Close dependencies after in-flight work completes.",[370,862,863],{},"Provider SSE and operator-event SSE have separate deadlines. Do not replace the ASGI server's SIGTERM handling with an incompatible custom handler.",[383,865,867],{"id":866},"what-is-not-proven","What is not proven",[869,870,871,874,877,880,883,886],"ul",{},[846,872,873],{},"No tested RPO\u002FRTO per data class.",[846,875,876],{},"No hosted backup\u002Frestore drill is recorded.",[846,878,879],{},"No HA failover, multi-zone, or capacity evidence is accepted.",[846,881,882],{},"Helm is render\u002Flint scaffolding only; it is not a deployable production topology.",[846,884,885],{},"Object-store purge and full deletion certification remain open.",[846,887,888],{},"Evidence replay has local PostgreSQL tests, not production ClickHouse or hosted recovery evidence.",[383,890,892],{"id":891},"operator-checklist","Operator checklist",[869,894,897,910,916,922,928,934],{"className":895},[896],"contains-task-list",[846,898,901,906,907,909],{"className":899},[900],"task-list-item",[902,903],"input",{"disabled":904,"type":905},true,"checkbox"," Confirm ",[547,908,631],{}," reflects the dependencies required by the selected profile.",[846,911,913,915],{"className":912},[900],[902,914],{"disabled":904,"type":905}," Confirm the failure is authority or quality before choosing deny versus degrade.",[846,917,919,921],{"className":918},[900],[902,920],{"disabled":904,"type":905}," Check whether a provider call happened before a deny.",[846,923,925,927],{"className":924},[900],[902,926],{"disabled":904,"type":905}," Preserve the request ID, trace ID, and operation ID; do not log tokens or content.",[846,929,931,933],{"className":930},[900],[902,932],{"disabled":904,"type":905}," For deletion or replay incidents, check tombstones before rebuilding projections.",[846,935,937,939],{"className":936},[900],[902,938],{"disabled":904,"type":905}," Record the profile, artifact digests, and limitations with any recovery result.",[383,941,943],{"id":942},"related","Related",[869,945,946,952,957,962,967,972],{},[846,947,948,951],{},[949,950,281],"a",{"href":282}," — per-service probes",[846,953,954,956],{},[949,955,285],{"href":286}," — severity and containment",[846,958,959,961],{},[949,960,297],{"href":298}," — outbox and fencing",[846,963,964,966],{},[949,965,293],{"href":294}," — reservation failures",[846,968,969,971],{},[949,970,327],{"href":328}," — why the chart is not a deployment guarantee",[846,973,974,976],{},[949,975,57],{"href":58}," — gate vocabulary",{"title":822,"searchDepth":978,"depth":978,"links":979},2,[980,981,988,989,990,991,992,993,994],{"id":385,"depth":978,"text":386},{"id":522,"depth":978,"text":523,"children":982},[983,985,986,987],{"id":527,"depth":984,"text":91},3,{"id":626,"depth":984,"text":65},{"id":642,"depth":984,"text":643},{"id":649,"depth":984,"text":650},{"id":656,"depth":978,"text":657},{"id":718,"depth":978,"text":719},{"id":810,"depth":978,"text":811},{"id":837,"depth":978,"text":838},{"id":866,"depth":978,"text":867},{"id":891,"depth":978,"text":892},{"id":942,"depth":978,"text":943},"Which dependencies may degrade, which must fail closed, and what recovery evidence is still missing.","md",false,{},"provisional",{"title":305,"description":995},"engineering\u002Farchitecture-v2\u002F14-failure-modes-and-readiness.md; engineering\u002Farchitecture-v2\u002F15-runtime-operations-and-recovery.md; services\u002F*\u002FREADME.md","OKXjNIc8QtCmbQPTHG1GSSEuxGPCEr4a2bAfwDwLdWM",[1004,1006],{"title":301,"path":302,"stem":303,"description":1005,"children":-1},"The canonical operator application, its live data allowlist, and the surfaces that are still preview-only.",{"title":309,"path":310,"stem":311,"description":1007,"children":-1},"Tombstones, forget, organization deletion, legal holds, and the retention boundaries that remain unproven.",1792875079134]