[{"data":1,"prerenderedAt":337},["ShallowReactive",2],{"sanity-LevEgKvCUg3Ul3GKyhTsfZIfaGswLQ-qsmdjo9-lWe4":3,"sanity-Cejge34Gu7vMRZ-On6SP6pP_nyOOyhJJEoI5l_JR9dE":49,"shiki-block-v-0-0-0-0-0-1-0-1":336},{"data":4},{"_createdAt":5,"_id":6,"_rev":7,"_type":6,"_updatedAt":8,"acceptAllLink":9,"acceptNecessaryLink":18,"bannerColorSource":23,"bannerDescription":24,"bannerTitle":25,"categories":26,"declineAllLink":27,"managePreferencesLink":31,"modalColorSource":23,"modalTitle":42,"preferencesIconColorSource":23,"provider":43,"savePreferencesLink":44,"thirdPartyEmbedCode":26,"title":48},"2026-11-04T19:10:06Z","cookieSettings","0HimK5Zy5fEFWyyiuf4lm8","2026-11-10T22:53:51Z",{"_type":10,"style":11,"title":16,"type":17},"link",{"_type":12,"stylePreset":13},"link.style",{"_ref":14,"_type":15},"driftwire-link-style-primary","reference","Accept all","button",{"_type":10,"style":19,"title":22,"type":17},{"_type":12,"stylePreset":20},{"_ref":21,"_type":15},"driftwire-link-style-secondary","Necessary only","inherit","We use strictly necessary cookies to keep you signed in, and — only with your consent — analytics to see which docs pages help. No advertising cookies, ever.","Cookies, briefly",null,{"_type":10,"style":28,"title":30,"type":17},{"_type":12,"stylePreset":29},{"_ref":21,"_type":15},"Decline all",{"_type":10,"style":32,"title":41,"type":17},{"_type":12,"icon":33,"iconPosition":39,"stylePreset":40},{"_type":34,"hugeicon":35,"type":38},"icon.config",{"data":36,"name":37},"[[\"path\",{\"d\":\"M15.5 12C15.5 13.933 13.933 15.5 12 15.5C10.067 15.5 8.5 13.933 8.5 12C8.5 10.067 10.067 8.5 12 8.5C13.933 8.5 15.5 10.067 15.5 12Z\",\"stroke\":\"currentColor\",\"strokeWidth\":\"1.5\",\"key\":\"0\"}],[\"path\",{\"d\":\"M21.011 14.0965C21.5329 13.9558 21.7939 13.8854 21.8969 13.7508C22 13.6163 22 13.3998 22 12.9669V11.0332C22 10.6003 22 10.3838 21.8969 10.2493C21.7938 10.1147 21.5329 10.0443 21.011 9.90358C19.0606 9.37759 17.8399 7.33851 18.3433 5.40087C18.4817 4.86799 18.5509 4.60156 18.4848 4.44529C18.4187 4.28902 18.2291 4.18134 17.8497 3.96596L16.125 2.98673C15.7528 2.77539 15.5667 2.66972 15.3997 2.69222C15.2326 2.71472 15.0442 2.90273 14.6672 3.27873C13.208 4.73448 10.7936 4.73442 9.33434 3.27864C8.95743 2.90263 8.76898 2.71463 8.60193 2.69212C8.43489 2.66962 8.24877 2.77529 7.87653 2.98663L6.15184 3.96587C5.77253 4.18123 5.58287 4.28891 5.51678 4.44515C5.45068 4.6014 5.51987 4.86787 5.65825 5.4008C6.16137 7.3385 4.93972 9.37763 2.98902 9.9036C2.46712 10.0443 2.20617 10.1147 2.10308 10.2492C2 10.3838 2 10.6003 2 11.0332V12.9669C2 13.3998 2 13.6163 2.10308 13.7508C2.20615 13.8854 2.46711 13.9558 2.98902 14.0965C4.9394 14.6225 6.16008 16.6616 5.65672 18.5992C5.51829 19.1321 5.44907 19.3985 5.51516 19.5548C5.58126 19.7111 5.77092 19.8188 6.15025 20.0341L7.87495 21.0134C8.24721 21.2247 8.43334 21.3304 8.6004 21.3079C8.76746 21.2854 8.95588 21.0973 9.33271 20.7213C10.7927 19.2644 13.2088 19.2643 14.6689 20.7212C15.0457 21.0973 15.2341 21.2853 15.4012 21.3078C15.5682 21.3303 15.7544 21.2246 16.1266 21.0133L17.8513 20.034C18.2307 19.8187 18.4204 19.711 18.4864 19.5547C18.5525 19.3984 18.4833 19.132 18.3448 18.5991C17.8412 16.6616 19.0609 14.6226 21.011 14.0965Z\",\"stroke\":\"currentColor\",\"strokeLinecap\":\"round\",\"strokeWidth\":\"1.5\",\"key\":\"1\"}]]","Settings02Icon","hugeicon","start",{"_ref":21,"_type":15},"Preferences","Cookie preferences","custom",{"_type":10,"style":45,"title":47,"type":17},{"_type":12,"stylePreset":46},{"_ref":14,"_type":15},"Save preferences","Cookie Settings",{"data":50,"sourceMap":-1},[51],{"_createdAt":52,"_id":53,"_rev":54,"_type":55,"_updatedAt":8,"authorProfile":56,"authorRef":68,"body":70,"categories":255,"excerpt":263,"footer":26,"layout":264,"mainImage":268,"navigation":26,"publishedAt":272,"readTime":273,"relatedPosts":274,"routeOptions":325,"seo":26,"showOnWebsite":327,"slug":330,"tags":331,"title":335},"2026-11-06T17:00:52Z","driftwire-post-edge-propagation","0HimK5Zy5fEFWyyiuf4lpW","post",{"avatar":57,"bio":62,"name":63,"role":64,"sameAs":65,"slug":67},{"_type":58,"alt":59,"asset":60},"image","Initials avatar for Avery Lin",{"_ref":61,"_type":15},"image-25375e75ea85051781794390a13c63f315cc700f-256x256-png","Avery builds the edge evaluation layer at Driftwire: the part that answers \"is this flag on?\" four hundred thousand times a second without asking anyone.","Avery Lin","Staff Engineer, Edge",[66],"https:\u002F\u002Fexample.com\u002Fteam\u002Favery-lin","avery-lin",{"_ref":69,"_type":15},"driftwire-author-avery-lin",[71,91,99,108,125,130,138,147,155,163,171,179,187,196,205,213,223,231,239,247],{"_key":72,"_type":73,"children":74,"markDefs":89,"style":90},"cd321b3fedd7","block",[75,80,85],{"_key":76,"_type":77,"marks":78,"text":79},"e4c2741cfc57","span",[],"When a payment provider starts failing at 2 a.m., the only number that matters is how long it takes for the switch you just flipped to be true everywhere. For Driftwire that number is ",{"_key":81,"_type":77,"marks":82,"text":84},"87732091ab4d",[83],"strong","38 ms at p99",{"_key":86,"_type":77,"marks":87,"text":88},"4bcc0b18b91c",[],", measured from the write committing in the control plane to the last of 312 edge nodes serving the new value.",[],"normal",{"_key":92,"_type":73,"children":93,"markDefs":98,"style":90},"cc61aaeed948",[94],{"_key":95,"_type":77,"marks":96,"text":97},"ddaf34fd1944",[],"This post walks through where those milliseconds go, and the two designs we threw away on the way.",[],{"_key":100,"_type":73,"children":101,"markDefs":106,"style":107},"35b694839809",[102],{"_key":103,"_type":77,"marks":104,"text":105},"e3ca3e94e983",[],"The shape of the problem",[],"h2",{"_key":109,"_type":73,"children":110,"markDefs":124,"style":90},"d267992fe434",[111,115,120],{"_key":112,"_type":77,"marks":113,"text":114},"a42e8ef3835c",[],"Every SDK evaluates flags locally, against a ruleset it holds in memory. That is what makes evaluation fast (",{"_key":116,"_type":77,"marks":117,"text":119},"3a8b06098050",[118],"code","isEnabled()",{"_key":121,"_type":77,"marks":122,"text":123},"e55ae636e96c",[]," never makes a network call), and it is also what makes propagation hard: a change is only real once every copy of the ruleset has it.",[],{"_key":126,"_type":58,"alt":127,"asset":128,"markDefs":26},"7cd599a69b52","Diagram: the control plane fanning a change out to four regions, each feeding five edge nodes, with per-region latencies between 11 and 23 ms",{"_ref":129,"_type":15},"image-fb2065895eba1f5382d743824914b7f1a6a869f2-1600x900-png",{"_key":131,"_type":73,"children":132,"markDefs":137,"style":107},"01c01d9d71dc",[133],{"_key":134,"_type":77,"marks":135,"text":136},"2e4c55cfa155",[],"What we tried first",[],{"_key":139,"_type":73,"children":140,"markDefs":145,"style":146},"6e6815ccf9a2",[141],{"_key":142,"_type":77,"marks":143,"text":144},"5dac3b517d97",[],"Polling",[],"h3",{"_key":148,"_type":73,"children":149,"markDefs":154,"style":90},"1d6bf1c4cacc",[150],{"_key":151,"_type":77,"marks":152,"text":153},"d464d59bb7c5",[],"Version one had every edge node poll the control plane every five seconds. Simple, stateless, and wrong: the median kill switch took 2.5 s to land and the worst case was a full interval. Shortening the interval just moved the cost onto our database.",[],{"_key":156,"_type":73,"children":157,"markDefs":162,"style":146},"2885a22ae487",[158],{"_key":159,"_type":77,"marks":160,"text":161},"2ea593543cfe",[],"A global pub\u002Fsub topic",[],{"_key":164,"_type":73,"children":165,"markDefs":170,"style":90},"d2bac64ab389",[166],{"_key":167,"_type":77,"marks":168,"text":169},"1667db1010db",[],"Version two pushed changes through a single managed topic. Latency dropped to around 400 ms, but a regional outage of the broker meant a region silently stopped receiving changes — the one failure a kill switch cannot tolerate.",[],{"_key":172,"_type":73,"children":173,"markDefs":178,"style":107},"9b7bce59c1b2",[174],{"_key":175,"_type":77,"marks":176,"text":177},"ac994aec7b75",[],"What ships today",[],{"_key":180,"_type":73,"children":181,"markDefs":186,"style":90},"4cb715b571cb",[182],{"_key":183,"_type":77,"marks":184,"text":185},"7bec1f4b732f",[],"Each region runs its own relay. The control plane writes the change, then pushes a compact diff to every regional relay over a long-lived HTTP\u002F2 stream. Relays fan out to their edge nodes the same way. Every message carries a monotonic version, so a node that misses one notices the gap and fetches a full snapshot.",[],{"_key":188,"_type":189,"colorSource":118,"markDefs":26,"snippet":190},"3abc129fa696","code.block",{"_type":118,"code":191,"filename":192,"highlightedLines":193,"language":195},"\u002F\u002F What an edge node receives: a diff against the version it holds.\ninterface RulesetDiff {\n    version: number\n    previous: number\n    changed: Record\u003Cstring, FlagRules>\n    removed: string[]\n}\n\nfunction apply(diff: RulesetDiff) {\n    if (diff.previous !== store.version) return resync() \u002F\u002F missed one: fetch a snapshot\n    store.merge(diff.changed, diff.removed)\n    store.version = diff.version\n}","edge\u002Fapply.ts",[194],10,"typescript",{"_key":197,"_type":73,"children":198,"markDefs":203,"style":204},"fe9afb54afc0",[199],{"_key":200,"_type":77,"marks":201,"text":202},"08ded789f190",[],"A kill switch should fail loud. If a node cannot prove it holds the latest version, it says so — it does not quietly serve what it had.",[],"blockquote",{"_key":206,"_type":73,"children":207,"markDefs":212,"style":107},"37e48182758f",[208],{"_key":209,"_type":77,"marks":210,"text":211},"c7501e45b55c",[],"Where the 38 ms goes",[],{"_key":214,"_type":73,"children":215,"level":220,"listItem":221,"markDefs":222,"style":90},"92cc69958fe4",[216],{"_key":217,"_type":77,"marks":218,"text":219},"aabfea08bc20",[],"Control plane commit and fan-out: 6 ms",1,"bullet",[],{"_key":224,"_type":73,"children":225,"level":220,"listItem":221,"markDefs":230,"style":90},"0e4d983fbb0c",[226],{"_key":227,"_type":77,"marks":228,"text":229},"c2c54b104d3f",[],"Control plane to regional relay: 9–21 ms, depending on distance",[],{"_key":232,"_type":73,"children":233,"level":220,"listItem":221,"markDefs":238,"style":90},"57056a3268e5",[234],{"_key":235,"_type":77,"marks":236,"text":237},"0f0ec82fdea6",[],"Relay to edge node: 2–5 ms",[],{"_key":240,"_type":73,"children":241,"level":220,"listItem":221,"markDefs":246,"style":90},"f8f0823118db",[242],{"_key":243,"_type":77,"marks":244,"text":245},"b1e3fad4a12f",[],"Apply and swap the in-memory ruleset: under 1 ms",[],{"_key":248,"_type":73,"children":249,"markDefs":254,"style":90},"d7a3fe72cfd3",[250],{"_key":251,"_type":77,"marks":252,"text":253},"371405883eb0",[],"The long pole is the speed of light between continents, which is why the next step is not a faster protocol but more relays.",[],[256],{"_createdAt":52,"_id":257,"_rev":258,"_type":259,"_updatedAt":260,"slug":261,"title":262},"driftwire-category-engineering","W5BzlLjnYUuDrmlopXouBu","category","2026-11-10T22:53:47Z","engineering","Engineering","A kill switch that takes a minute to arrive is a status page with extra steps. How our control plane fans a single write out to 312 edge nodes, and where the 38 ms actually goes.",{"_type":265,"layout":266,"pageDesign":267},"post.layout.options","default","design1",{"_type":58,"alt":269,"asset":270},"Engineering cover: How we push a kill switch to 312 edge nodes in 38 ms",{"_ref":271,"_type":15},"image-75833b45f7bd6ef1e0d8f5900a4006e107fdb8bd-1600x900-jpg","2026-10-06T09:00:00.000Z",2,[275,290,310],{"_id":276,"authorProfile":277,"categories":280,"excerpt":282,"mainImage":283,"publishedAt":287,"readTime":220,"slug":288,"title":289},"driftwire-post-percentage-rollouts",{"avatar":278,"name":63,"role":64,"slug":67},{"_type":58,"alt":59,"asset":279},{"_ref":61,"_type":15},[281],{"slug":261,"title":262},"A 25% rollout is not \"one request in four\". It is a stable answer to \"is this user in?\", identical on every server, that never flips a user back out. That is a hashing problem.",{"_type":58,"alt":284,"asset":285},"Engineering cover: Percentage rollouts are a hashing problem",{"_ref":286,"_type":15},"image-4b4179c3d9320c0f9bf734ebd4c1860ecdabdd2a-1600x900-jpg","2026-09-08T09:00:00.000Z","percentage-rollouts","Percentage rollouts are a hashing problem",{"_id":291,"authorProfile":292,"categories":300,"excerpt":302,"mainImage":303,"publishedAt":307,"readTime":220,"slug":308,"title":309},"driftwire-post-testing-combinations",{"avatar":293,"name":297,"role":298,"slug":299},{"_type":58,"alt":294,"asset":295},"Initials avatar for Jordan Hale",{"_ref":296,"_type":15},"image-097a0f6135da5d3dedfc9d8fe79bba96e1550621-256x256-png","Jordan Hale","Co-founder & CTO","jordan-hale",[301],{"slug":261,"title":262},"Ten boolean flags make 1,024 combinations, and you will not test them all. You do not need to. Test the defaults, the targets, and the pairs that touch the same code.",{"_type":58,"alt":304,"asset":305},"Engineering cover: You cannot test every flag combination. Test these instead",{"_ref":306,"_type":15},"image-a6198bda12704ab24d51cd2c562ac7f6ab1ae15c-1600x900-jpg","2026-07-14T09:00:00.000Z","testing-combinations","You cannot test every flag combination. Test these instead",{"_id":311,"authorProfile":312,"categories":315,"excerpt":317,"mainImage":318,"publishedAt":322,"readTime":220,"slug":323,"title":324},"driftwire-post-edge-vs-server",{"avatar":313,"name":63,"role":64,"slug":67},{"_type":58,"alt":59,"asset":314},{"_ref":61,"_type":15},[316],{"slug":261,"title":262},"Evaluating flags at the edge is fast. Evaluating them on your server is simple. Which one you want depends on numbers you can measure in an afternoon.",{"_type":58,"alt":319,"asset":320},"Engineering cover: Edge or server evaluation? Measure before you choose",{"_ref":321,"_type":15},"image-335586df211ad6b1e711562eccf1cf16e8090a11-1600x900-jpg","2026-06-02T09:00:00.000Z","edge-vs-server","Edge or server evaluation? Measure before you choose",{"_type":326,"includeInSitemap":327,"slug":328},"page.routing.options",true,{"_type":329,"current":330},"slug","edge-propagation",[332,333,334],"Edge","Kill switches","Latency","How we push a kill switch to 312 edge nodes in 38 ms","\u003Cpre class=\"shiki shiki-themes github-light github-dark nuxion-code__pre\" style=\"--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e\" tabindex=\"0\">\u003Ccode>\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#6A737D;--shiki-dark:#6A737D\">\u002F\u002F What an edge node receives: a diff against the version it holds.\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">interface\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\"> RulesetDiff\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\"> {\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#E36209;--shiki-dark:#FFAB70\">    version\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">:\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#005CC5;--shiki-dark:#79B8FF\"> number\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#E36209;--shiki-dark:#FFAB70\">    previous\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">:\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#005CC5;--shiki-dark:#79B8FF\"> number\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#E36209;--shiki-dark:#FFAB70\">    changed\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">:\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\"> Record\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">&#x3C;\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#005CC5;--shiki-dark:#79B8FF\">string\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">, \u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\">FlagRules\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">>\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#E36209;--shiki-dark:#FFAB70\">    removed\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">:\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#005CC5;--shiki-dark:#79B8FF\"> string\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">[]\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">}\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">function\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\"> apply\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">(\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#E36209;--shiki-dark:#FFAB70\">diff\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">:\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\"> RulesetDiff\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">) {\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line nuxion-code__line--marked\">\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">    if\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\"> (diff.previous \u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">!==\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\"> store.version) \u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">return\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\"> resync\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">() \u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6A737D;--shiki-dark:#6A737D\">\u002F\u002F missed one: fetch a snapshot\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">    store.\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#6F42C1;--shiki-dark:#B392F0\">merge\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">(diff.changed, diff.removed)\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">    store.version \u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#D73A49;--shiki-dark:#F97583\">=\u003C\u002Fspan>\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\"> diff.version\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"--shiki-light:#24292E;--shiki-dark:#E1E4E8\">}\u003C\u002Fspan>\u003C\u002Fspan>\u003C\u002Fcode>\u003C\u002Fpre>",1794954844102]