{ "type": "excalidraw", "version": 2, "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", "elements": [ { "type": "text", "id": "title", "x": 60, "y": 36, "width": 1200, "height": 35, "text": "Zoom 2: the data plane - two tiers, chosen per edge", "originalText": "Zoom 2: the data plane - two tiers, chosen per edge", "fontSize": 28, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400001, "version": 2, "versionNonce": 1541333284, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "a0", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "subtitle", "x": 60, "y": 80, "width": 1000, "height": 20, "text": "how bytes actually move between a source and DuckDB (sinks mirror it, bottom)", "originalText": "how bytes actually move between a source and DuckDB (sinks mirror it, bottom)", "fontSize": 15, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400002, "version": 2, "versionNonce": 2127967004, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "a1", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "planner_lead", "x": 700, "y": 96, "width": 400, "height": 16, "text": "per edge, per run, the planner asks:", "originalText": "per edge, per run, the planner asks:", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400003, "version": 2, "versionNonce": 657398948, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "a2", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "diamond", "id": "decision", "x": 720, "y": 120, "width": 360, "height": 110, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400004, "version": 2, "versionNonce": 2086236060, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "decision_t", "type": "text" } ], "link": null, "locked": false, "index": "a3", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "text", "id": "decision_t", "x": 790, "y": 158, "width": 220, "height": 34, "text": "TryGetNativeScan()\nreturns a scan?", "originalText": "TryGetNativeScan()\nreturns a scan?", "fontSize": 13, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400005, "version": 2, "versionNonce": 602662948, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "decision", "lineHeight": 1.25, "index": "a4", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "yes_arrow", "x": 718, "y": 177, "width": 250, "height": 121, "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400006, "version": 2, "versionNonce": 1329667100, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ -250, 70 ], [ -250, 121 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "a5", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "text", "id": "yes_lbl", "x": 490, "y": 258, "width": 150, "height": 16, "text": "yes - fast path", "originalText": "yes - fast path", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400007, "version": 2, "versionNonce": 369312676, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "a6", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "no_arrow", "x": 1082, "y": 177, "width": 140, "height": 121, "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400008, "version": 2, "versionNonce": 830421148, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 140, 70 ], [ 140, 121 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "a7", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "text", "id": "no_lbl", "x": 1160, "y": 200, "width": 200, "height": 16, "text": "no - universal path", "originalText": "no - universal path", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 400009, "version": 2, "versionNonce": 329461540, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "a8", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t1_box", "x": 60, "y": 300, "width": 700, "height": 520, "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410001, "version": 2, "versionNonce": 1321583900, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "a9", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t1_label", "x": 80, "y": 316, "width": 600, "height": 20, "text": "TIER 1 - NATIVE SCAN / COPY (fast path)", "originalText": "TIER 1 - NATIVE SCAN / COPY (fast path)", "fontSize": 15, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410002, "version": 2, "versionNonce": 1027189412, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aA", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t1_src", "x": 80, "y": 360, "width": 290, "height": 50, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410003, "version": 2, "versionNonce": 210157980, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t1_src_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aB", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t1_src_t", "x": 90, "y": 377, "width": 270, "height": 15, "text": "az://analytics/clicks/*.parquet", "originalText": "az://analytics/clicks/*.parquet", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410004, "version": 2, "versionNonce": 1970583076, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t1_src", "lineHeight": 1.25, "index": "aC", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "t1_arrow", "x": 374, "y": 385, "width": 82, "height": 0, "strokeColor": "#c2410c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410005, "version": 2, "versionNonce": 1374957084, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 82, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "aD", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t1_duck", "x": 460, "y": 352, "width": 270, "height": 66, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410006, "version": 2, "versionNonce": 423776676, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t1_duck_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aE", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t1_duck_t", "x": 470, "y": 368, "width": 250, "height": 34, "text": "DuckDB parallel readers\n+ predicate pushdown", "originalText": "DuckDB parallel readers\n+ predicate pushdown", "fontSize": 13, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#ffffff", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410007, "version": 2, "versionNonce": 1342179996, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t1_duck", "lineHeight": 1.25, "index": "aF", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t1_cap", "x": 80, "y": 426, "width": 340, "height": 32, "text": "bytes go straight from storage into DuckDB -\n.NET never sees them", "originalText": "bytes go straight from storage into DuckDB -\n.NET never sees them", "fontSize": 12, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410008, "version": 2, "versionNonce": 936410404, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aG", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t1_sql_label", "x": 80, "y": 466, "width": 500, "height": 15, "text": "what the connector hands the engine (NativeScan record)", "originalText": "what the connector hands the engine (NativeScan record)", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410009, "version": 2, "versionNonce": 1230788380, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aH", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t1_sql_box", "x": 80, "y": 484, "width": 650, "height": 100, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410010, "version": 2, "versionNonce": 1250769060, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aI", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t1_sql_code", "x": 94, "y": 496, "width": 625, "height": 69, "text": "SetupStatements: CREATE SECRET src_az (TYPE azure, CONNECTION_STRING '...');\nSqlFragment: -- never logged\n CREATE TABLE staging.src_events__clicks AS\n SELECT * FROM read_parquet('az://analytics/clicks/*.parquet');\nMechanism: \"read_parquet\" -- shows up in pz plan reasons", "originalText": "SetupStatements: CREATE SECRET src_az (TYPE azure, CONNECTION_STRING '...');\nSqlFragment: -- never logged\n CREATE TABLE staging.src_events__clicks AS\n SELECT * FROM read_parquet('az://analytics/clicks/*.parquet');\nMechanism: \"read_parquet\" -- shows up in pz plan reasons", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#93c5fd", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410011, "version": 2, "versionNonce": 1518015388, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aJ", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t1_note", "x": 80, "y": 608, "width": 650, "height": 45, "text": "the connector's whole job collapses to config translation: YAML\nconnection block in, scan SQL + secret setup out. DuckDB's own\nreaders (parquet, csv, azure, ...) do the moving.", "originalText": "the connector's whole job collapses to config translation: YAML\nconnection block in, scan SQL + secret setup out. DuckDB's own\nreaders (parquet, csv, azure, ...) do the moving.", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410012, "version": 2, "versionNonce": 1831240740, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aK", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t1_plan_label", "x": 80, "y": 676, "width": 400, "height": 15, "text": "and pz plan tells you which tier won, and why:", "originalText": "and pz plan tells you which tier won, and why:", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410013, "version": 2, "versionNonce": 1099841564, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aL", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t1_plan_box", "x": 80, "y": 694, "width": 650, "height": 90, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410014, "version": 2, "versionNonce": 361389988, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aM", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t1_plan_code", "x": 94, "y": 706, "width": 625, "height": 55, "text": "$ pz plan\n lake.clicks native scan (read_parquet) connector supports NativeScan\n mssql.orders arrow stream, 2 partitions universal tier only (SqlClient)\n d365.contacts arrow stream, 4 partitions no scanner for SaaS APIs", "originalText": "$ pz plan\n lake.clicks native scan (read_parquet) connector supports NativeScan\n mssql.orders arrow stream, 2 partitions universal tier only (SqlClient)\n d365.contacts arrow stream, 4 partitions no scanner for SaaS APIs", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#22c55e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 410015, "version": 2, "versionNonce": 646438044, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aN", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_box", "x": 800, "y": 300, "width": 840, "height": 520, "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420001, "version": 2, "versionNonce": 1273416484, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aO", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_label", "x": 820, "y": 316, "width": 700, "height": 20, "text": "TIER 2 - ARROW BATCH STREAM (universal path)", "originalText": "TIER 2 - ARROW BATCH STREAM (universal path)", "fontSize": 15, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420002, "version": 2, "versionNonce": 1662642460, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aP", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_r1", "x": 820, "y": 370, "width": 200, "height": 46, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420003, "version": 2, "versionNonce": 1655293604, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t2_r1_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aQ", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_r1_t", "x": 830, "y": 385, "width": 180, "height": 15, "text": "partition reader 1", "originalText": "partition reader 1", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420004, "version": 2, "versionNonce": 1253467548, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t2_r1", "lineHeight": 1.25, "index": "aR", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_r2", "x": 820, "y": 440, "width": 200, "height": 46, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420005, "version": 2, "versionNonce": 780227108, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t2_r2_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aS", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_r2_t", "x": 830, "y": 455, "width": 180, "height": 15, "text": "partition reader 2", "originalText": "partition reader 2", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420006, "version": 2, "versionNonce": 1921995292, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t2_r2", "lineHeight": 1.25, "index": "aT", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t2_r_cap", "x": 820, "y": 496, "width": 220, "height": 28, "text": "connector code: async I/O over\na REST / OData API,\n1..N partition readers", "originalText": "connector code: async I/O over\na REST / OData API,\n1..N partition readers", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420007, "version": 2, "versionNonce": 1046179236, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aU", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t2_ch_label", "x": 1060, "y": 352, "width": 250, "height": 14, "text": "BoundedChannel, cap 4", "originalText": "BoundedChannel, cap 4", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420008, "version": 2, "versionNonce": 2039918236, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aV", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_ch", "x": 1090, "y": 380, "width": 190, "height": 90, "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420009, "version": 2, "versionNonce": 325458212, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aW", "frameId": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_b1", "x": 1104, "y": 402, "width": 28, "height": 28, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420010, "version": 2, "versionNonce": 1350466332, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aX", "frameId": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_b2", "x": 1148, "y": 402, "width": 28, "height": 28, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420011, "version": 2, "versionNonce": 1494176932, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aY", "frameId": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_b3", "x": 1192, "y": 402, "width": 28, "height": 28, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420012, "version": 2, "versionNonce": 28882844, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aZ", "frameId": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_b4", "x": 1236, "y": 402, "width": 28, "height": 28, "strokeColor": "#1e3a5f", "backgroundColor": "#dbeafe", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420013, "version": 2, "versionNonce": 486875172, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "aa", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_ch_cap", "x": 1098, "y": 444, "width": 180, "height": 13, "text": "batches of ~32 MB each", "originalText": "batches of ~32 MB each", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420014, "version": 2, "versionNonce": 2037867548, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "ab", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "t2_a1", "x": 1024, "y": 393, "width": 62, "height": 20, "strokeColor": "#c2410c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420015, "version": 2, "versionNonce": 120945572, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 62, 20 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "ac", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "arrow", "id": "t2_a2", "x": 1024, "y": 463, "width": 62, "height": 22, "strokeColor": "#c2410c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420016, "version": 2, "versionNonce": 1303150748, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 62, -22 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "ad", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "arrow", "id": "t2_a3", "x": 1284, "y": 425, "width": 62, "height": 0, "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420017, "version": 2, "versionNonce": 298739492, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 62, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "ae", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_ing", "x": 1350, "y": 380, "width": 250, "height": 90, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420018, "version": 2, "versionNonce": 101322012, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t2_ing_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "af", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_ing_t", "x": 1360, "y": 399, "width": 230, "height": 51, "text": "arrow_scan ingest\nzero-copy via the Arrow\nC Data Interface", "originalText": "arrow_scan ingest\nzero-copy via the Arrow\nC Data Interface", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#ffffff", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420019, "version": 2, "versionNonce": 1094574756, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t2_ing", "lineHeight": 1.25, "index": "ag", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "t2_bp", "x": 1340, "y": 495, "width": 250, "height": 0, "strokeColor": "#dc2626", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420020, "version": 2, "versionNonce": 1677855132, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ -250, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "ah", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_bp_lbl", "x": 1075, "y": 505, "width": 320, "height": 13, "text": "backpressure: channel full => readers suspend", "originalText": "backpressure: channel full => readers suspend", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#dc2626", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420021, "version": 2, "versionNonce": 1546776100, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "ai", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "arrow", "id": "t2_a4", "x": 1475, "y": 472, "width": 0, "height": 66, "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420022, "version": 2, "versionNonce": 717649436, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 66 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "aj", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "t2_stg", "x": 1350, "y": 540, "width": 250, "height": 44, "strokeColor": "#1e3a5f", "backgroundColor": "#3b82f6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420023, "version": 2, "versionNonce": 597199268, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "t2_stg_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "ak", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_stg_t", "x": 1360, "y": 554, "width": 230, "height": 15, "text": "staging.src_d365__contacts", "originalText": "staging.src_d365__contacts", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#ffffff", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420024, "version": 2, "versionNonce": 1732858524, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "t2_stg", "lineHeight": 1.25, "index": "al", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t2_gc_note", "x": 820, "y": 548, "width": 470, "height": 45, "text": "Arrow buffers live off-heap in pooled native memory - a 32 MB\nbatch is invisible to the GC. DuckDB reads the connector's\nbuffers directly; no per-value marshaling anywhere.", "originalText": "Arrow buffers live off-heap in pooled native memory - a 32 MB\nbatch is invisible to the GC. DuckDB reads the connector's\nbuffers directly; no per-value marshaling anywhere.", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420025, "version": 2, "versionNonce": 178238756, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "am", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t2_abi_label", "x": 820, "y": 624, "width": 350, "height": 15, "text": "the whole universal contract (connector ABI)", "originalText": "the whole universal contract (connector ABI)", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420026, "version": 2, "versionNonce": 943496988, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "an", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_abi_box", "x": 820, "y": 642, "width": 400, "height": 86, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420027, "version": 2, "versionNonce": 1857384612, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "ao", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_abi_code", "x": 834, "y": 653, "width": 375, "height": 63, "text": "PlanReadAsync(spec, hints, ct)\n -> IDatasetPartition [1..N]\npartition.ReadAsync(options, ct)\n -> IAsyncEnumerable", "originalText": "PlanReadAsync(spec, hints, ct)\n -> IDatasetPartition [1..N]\npartition.ReadAsync(options, ct)\n -> IAsyncEnumerable", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#93c5fd", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420028, "version": 2, "versionNonce": 370301852, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "ap", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "t2_diag_label", "x": 1250, "y": 624, "width": 380, "height": 15, "text": "channel stall metrics name the bottleneck", "originalText": "channel stall metrics name the bottleneck", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420029, "version": 2, "versionNonce": 1334295588, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aq", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "t2_diag_box", "x": 1250, "y": 642, "width": 380, "height": 86, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420030, "version": 2, "versionNonce": 452757532, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "ar", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "t2_diag_code", "x": 1264, "y": 653, "width": 355, "height": 63, "text": "orders load: source-bound -\n reader busy 92%, ingest idle 71%.\n consider partitioned read\n (connector supports 8).", "originalText": "orders load: source-bound -\n reader busy 92%, ingest idle 71%.\n consider partitioned read\n (connector supports 8).", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#22c55e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 420031, "version": 2, "versionNonce": 117741476, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "as", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "sink_label", "x": 60, "y": 856, "width": 500, "height": 20, "text": "SINKS MIRROR THE SAME TWO TIERS", "originalText": "SINKS MIRROR THE SAME TWO TIERS", "fontSize": 15, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 430001, "version": 2, "versionNonce": 1257055388, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "at", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "sink_text", "x": 60, "y": 886, "width": 1300, "height": 51, "text": "native copy: the sink returns COPY (SELECT * FROM staging.order_totals) TO 'out/totals/' (FORMAT csv) - DuckDB writes directly.\nuniversal: DuckDB result stream -> bounded channel -> ISinkWriteSession.WriteBatchAsync per batch, then Commit XOR Abort.\ntemp-write + atomic swap makes write.strategy: replace all-or-nothing - a failed run never leaves a half-replaced output.", "originalText": "native copy: the sink returns COPY (SELECT * FROM staging.order_totals) TO 'out/totals/' (FORMAT csv) - DuckDB writes directly.\nuniversal: DuckDB result stream -> bounded channel -> ISinkWriteSession.WriteBatchAsync per batch, then Commit XOR Abort.\ntemp-write + atomic swap makes write.strategy: replace all-or-nothing - a failed run never leaves a half-replaced output.", "fontSize": 12, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 430002, "version": 2, "versionNonce": 22676260, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "au", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "line", "id": "vol_divider", "x": 60, "y": 964, "width": 1580, "height": 0, "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440001, "version": 2, "versionNonce": 1536569628, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 1580, 0 ] ], "lastCommittedPoint": null, "index": "av", "frameId": null, "roundness": null, "updated": 1783594575529, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null }, { "type": "text", "id": "vol_label", "x": 60, "y": 986, "width": 800, "height": 20, "text": "HOW MUCH DATA PER RUN? THREE MODES", "originalText": "HOW MUCH DATA PER RUN? THREE MODES", "fontSize": 15, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440002, "version": 2, "versionNonce": 1962278564, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "aw", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "vol_intro", "x": 60, "y": 1012, "width": 1200, "height": 30, "text": "within a run, data never arrives 'all at once': it streams as ~32 MB Arrow batches (tier 2) or a DuckDB scan (tier 1).\nwhat IS configurable is how much of the dataset each run claims:", "originalText": "within a run, data never arrives 'all at once': it streams as ~32 MB Arrow batches (tier 2) or a DuckDB scan (tier 1).\nwhat IS configurable is how much of the dataset each run claims:", "fontSize": 11, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440003, "version": 2, "versionNonce": 2087300508, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "ax", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m1_label", "x": 60, "y": 1062, "width": 300, "height": 17, "text": "FULL LOAD (default)", "originalText": "FULL LOAD (default)", "fontSize": 13, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#3b82f6", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440004, "version": 2, "versionNonce": 445024804, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "ay", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m1_yaml_box", "x": 60, "y": 1088, "width": 230, "height": 58, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440005, "version": 2, "versionNonce": 513382940, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "az", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m1_yaml", "x": 74, "y": 1099, "width": 205, "height": 38, "text": "datasets:\n orders:\n table: orders", "originalText": "datasets:\n orders:\n table: orders", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#22c55e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440006, "version": 2, "versionNonce": 1694082468, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b00", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m1_bar", "x": 60, "y": 1170, "width": 400, "height": 32, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440007, "version": 2, "versionNonce": 1024980636, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "m1_bar_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b01", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m1_bar_t", "x": 70, "y": 1179, "width": 380, "height": 14, "text": "the entire dataset, every run", "originalText": "the entire dataset, every run", "fontSize": 11, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440008, "version": 2, "versionNonce": 964514084, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "m1_bar", "lineHeight": 1.25, "index": "b02", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m1_cap", "x": 60, "y": 1216, "width": 420, "height": 28, "text": "fine for small/reference tables; sinks with\nwrite.strategy: replace swap it in atomically", "originalText": "fine for small/reference tables; sinks with\nwrite.strategy: replace swap it in atomically", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440009, "version": 2, "versionNonce": 613860124, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b03", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m2_label", "x": 580, "y": 1062, "width": 400, "height": 17, "text": "INCREMENTAL (watermark)", "originalText": "INCREMENTAL (watermark)", "fontSize": 13, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#3b82f6", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440010, "version": 2, "versionNonce": 662670500, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b04", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m2_yaml_box", "x": 580, "y": 1088, "width": 250, "height": 46, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440011, "version": 2, "versionNonce": 998768540, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b05", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m2_yaml", "x": 594, "y": 1099, "width": 225, "height": 25, "text": "sync:\n mode: incremental\n cursor: updated_at", "originalText": "sync:\n mode: incremental\n cursor: updated_at", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#22c55e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440012, "version": 2, "versionNonce": 548296740, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b06", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m2_bar_done", "x": 580, "y": 1170, "width": 210, "height": 32, "strokeColor": "#1e40af", "backgroundColor": "#dbeafe", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440013, "version": 2, "versionNonce": 1243458588, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "m2_bar_done_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b07", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m2_bar_done_t", "x": 590, "y": 1179, "width": 190, "height": 14, "text": "already loaded", "originalText": "already loaded", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440014, "version": 2, "versionNonce": 821539748, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "m2_bar_done", "lineHeight": 1.25, "index": "b08", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m2_bar_new", "x": 790, "y": 1170, "width": 150, "height": 32, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440015, "version": 2, "versionNonce": 1112131740, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "m2_bar_new_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b09", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m2_bar_new_t", "x": 800, "y": 1179, "width": 130, "height": 14, "text": "this run", "originalText": "this run", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440016, "version": 2, "versionNonce": 914988836, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "m2_bar_new", "lineHeight": 1.25, "index": "b0A", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "line", "id": "m2_wm_tick", "x": 790, "y": 1158, "width": 0, "height": 56, "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440017, "version": 2, "versionNonce": 129914140, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 56 ] ], "lastCommittedPoint": null, "index": "b0B", "frameId": null, "roundness": null, "updated": 1783594575529, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null }, { "type": "text", "id": "m2_wm_lbl", "x": 762, "y": 1142, "width": 60, "height": 13, "text": "$wm", "originalText": "$wm", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440018, "version": 2, "versionNonce": 433199780, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0C", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m2_cap", "x": 580, "y": 1226, "width": 460, "height": 28, "text": "extracts WHERE updated_at > $wm (pushed down);\nwatermark persists in .pz/state, advances only after sinks commit", "originalText": "extracts WHERE updated_at > $wm (pushed down);\nwatermark persists in .pz/state, advances only after sinks commit", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 440019, "version": 2, "versionNonce": 938299804, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0D", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m3_label", "x": 1120, "y": 1062, "width": 114.2578125, "height": 16.25, "text": "BOUNDED WINDOWS", "originalText": "BOUNDED WINDOWS", "fontSize": 13, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#3b82f6", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450001, "version": 5, "versionNonce": 287905820, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0E", "frameId": null, "roundness": null, "updated": 1783595194606, "autoResize": true }, { "type": "rectangle", "id": "m3_yaml_box", "x": 1120, "y": 1088, "width": 280, "height": 84, "strokeColor": "#1e293b", "backgroundColor": "#1e293b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450002, "version": 2, "versionNonce": 1913175580, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0F", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m3_yaml", "x": 1134, "y": 1098, "width": 255, "height": 63, "text": "sync:\n mode: incremental\n cursor: updated_at\n max_window: 1d\n initial: \"2020-01-01\"\n until: \"2026-07-01\"", "originalText": "sync:\n mode: incremental\n cursor: updated_at\n max_window: 1d\n initial: \"2020-01-01\"\n until: \"2026-07-01\"", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#22c55e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450003, "version": 2, "versionNonce": 511623588, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0G", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m3_bar_done", "x": 1120, "y": 1190, "width": 100, "height": 32, "strokeColor": "#1e40af", "backgroundColor": "#dbeafe", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450004, "version": 2, "versionNonce": 520264348, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "m3_bar_done_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0H", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m3_bar_done_t", "x": 1128, "y": 1199, "width": 84, "height": 14, "text": "done", "originalText": "done", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450005, "version": 2, "versionNonce": 1006562596, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "m3_bar_done", "lineHeight": 1.25, "index": "b0I", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m3_bar_slice", "x": 1220, "y": 1190, "width": 95, "height": 32, "strokeColor": "#c2410c", "backgroundColor": "#fed7aa", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450006, "version": 2, "versionNonce": 965535516, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "m3_bar_slice_t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0J", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m3_bar_slice_t", "x": 1228, "y": 1199, "width": 79, "height": 14, "text": "this run", "originalText": "this run", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450007, "version": 2, "versionNonce": 527280292, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "m3_bar_slice", "lineHeight": 1.25, "index": "b0K", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "m3_bar_next1", "x": 1315, "y": 1190, "width": 95, "height": 32, "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450008, "version": 2, "versionNonce": 370779036, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0L", "frameId": null, "updated": 1783594575529 }, { "type": "rectangle", "id": "m3_bar_next2", "x": 1410, "y": 1190, "width": 95, "height": 32, "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450009, "version": 2, "versionNonce": 746124324, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0M", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "m3_next_lbl", "x": 1340, "y": 1170, "width": 160, "height": 13, "text": "future runs", "originalText": "future runs", "fontSize": 9, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450010, "version": 2, "versionNonce": 718983196, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0N", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m3_ticks", "x": 1206, "y": 1228, "width": 260, "height": 13, "text": "lo hi = lo + max_window", "originalText": "lo hi = lo + max_window", "fontSize": 9, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e3a5f", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450011, "version": 2, "versionNonce": 554618788, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0O", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m3_cap", "x": 1120, "y": 1252, "width": 345.703125, "height": 25, "text": "this run: WHERE updated_at > lo AND updated_at <= hi\none giant backfill becomes a loop of small committed slices", "originalText": "this run: WHERE updated_at > lo AND updated_at <= hi\none giant backfill becomes a loop of small committed slices", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450012, "version": 5, "versionNonce": 782172798, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0P", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "arrow", "id": "m3_loop", "x": 1505, "y": 1292, "width": 375, "height": 0, "strokeColor": "#047857", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450013, "version": 2, "versionNonce": 1297955620, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ -375, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "b0Q", "frameId": null, "roundness": null, "updated": 1783594575529 }, { "type": "text", "id": "m3_loop_lbl", "x": 1120, "y": 1302, "width": 500, "height": 13, "text": "external loop: run -> watermark commits -> repeat until caught up", "originalText": "external loop: run -> watermark commits -> repeat until caught up", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#047857", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450014, "version": 2, "versionNonce": 780302620, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0R", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "m3_gap", "x": 1120, "y": 1324, "width": 520, "height": 13, "text": "an empty slice still advances the watermark - a data gap can't stall the loop", "originalText": "an empty slice still advances the watermark - a data gap can't stall the loop", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#64748b", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 450015, "version": 2, "versionNonce": 1272067748, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0S", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "text", "id": "gr_label", "x": 60, "y": 1372, "width": 198.046875, "height": 16.25, "text": "GUARDRAILS AROUND THE FLOW", "originalText": "GUARDRAILS AROUND THE FLOW", "fontSize": 13, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#1e40af", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460001, "version": 5, "versionNonce": 870716836, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0T", "frameId": null, "roundness": null, "updated": 1783595204745, "autoResize": true }, { "type": "rectangle", "id": "gr1", "x": 60, "y": 1400, "width": 350, "height": 50, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460002, "version": 2, "versionNonce": 958120484, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "gr1t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0U", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "gr1t", "x": 70, "y": 1412, "width": 330, "height": 26, "text": "retry: {max_attempts: 5, base_delay: 2s}\nper source/sink instance, per dataset", "originalText": "retry: {max_attempts: 5, base_delay: 2s}\nper source/sink instance, per dataset", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460003, "version": 2, "versionNonce": 1226039836, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "gr1", "lineHeight": 1.25, "index": "b0V", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "gr2", "x": 430, "y": 1400, "width": 290, "height": 50, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460004, "version": 2, "versionNonce": 1814551972, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "gr2t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0W", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "gr2t", "x": 440, "y": 1412, "width": 270, "height": 26, "text": "max_concurrency: 2\ncaps an instance's parallel nodes", "originalText": "max_concurrency: 2\ncaps an instance's parallel nodes", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460005, "version": 2, "versionNonce": 822689436, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "gr2", "lineHeight": 1.25, "index": "b0X", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "gr3", "x": 740, "y": 1400, "width": 360, "height": 50, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460006, "version": 2, "versionNonce": 774146340, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "gr3t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0Y", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "gr3t", "x": 750, "y": 1412, "width": 340, "height": 26, "text": "max_rows_per_second: 50000\ntoken-bucket pacing in the batch loop", "originalText": "max_rows_per_second: 50000\ntoken-bucket pacing in the batch loop", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460007, "version": 2, "versionNonce": 455019292, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "gr3", "lineHeight": 1.25, "index": "b0Z", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "rectangle", "id": "gr4", "x": 1120, "y": 1400, "width": 460, "height": 50, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460008, "version": 2, "versionNonce": 1920976036, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "gr4t", "type": "text" } ], "link": null, "locked": false, "roundness": { "type": 3 }, "index": "b0a", "frameId": null, "updated": 1783594575529 }, { "type": "text", "id": "gr4t", "x": 1130, "y": 1412, "width": 440, "height": 26, "text": "breaker: {failure_threshold: 5, cool_down: 2m}\npause a dying instance's nodes, probe, resume", "originalText": "breaker: {failure_threshold: 5, cool_down: 2m}\npause a dying instance's nodes, probe, resume", "fontSize": 10, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 460009, "version": 2, "versionNonce": 1237796764, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "gr4", "lineHeight": 1.25, "index": "b0b", "frameId": null, "roundness": null, "updated": 1783594575529, "autoResize": true }, { "type": "diamond", "id": "mrps_decision", "x": 150, "y": 130, "width": 280, "height": 90, "strokeColor": "#b45309", "backgroundColor": "#fef3c7", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470001, "version": 2, "versionNonce": 771002001, "isDeleted": false, "groupIds": [], "boundElements": [ { "id": "mrps_decision_t", "type": "text" } ], "link": null, "locked": false, "index": "b0c", "frameId": null, "roundness": null, "updated": 1783664055230 }, { "type": "text", "id": "mrps_decision_t", "x": 200, "y": 158, "width": 180, "height": 34, "text": "max_rows_per_second\ndeclared?", "originalText": "max_rows_per_second\ndeclared?", "fontSize": 12, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#374151", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470002, "version": 2, "versionNonce": 771002002, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": "mrps_decision", "lineHeight": 1.25, "index": "b0d", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "arrow", "id": "mrps_no_arrow", "x": 430, "y": 175, "width": 290, "height": 0, "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470003, "version": 2, "versionNonce": 771002003, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 290, 0 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "b0e", "frameId": null, "roundness": null, "updated": 1783664055230 }, { "type": "text", "id": "mrps_no_lbl", "x": 500, "y": 155, "width": 130, "height": 14, "text": "no - check native scan", "originalText": "no - check native scan", "fontSize": 11, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470004, "version": 2, "versionNonce": 771002004, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0f", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "arrow", "id": "mrps_yes_arrow", "x": 290, "y": 220, "width": 85, "height": 42, "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470005, "version": 2, "versionNonce": 771002005, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 42 ], [ 85, 42 ] ], "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "lastCommittedPoint": null, "index": "b0g", "frameId": null, "roundness": null, "updated": 1783664055230 }, { "type": "text", "id": "mrps_yes_lbl", "x": 75, "y": 230, "width": 210, "height": 53, "text": "yes -> universal path\n(skips TryGetNativeScan)\npacing needs the batch loop;\nnative-only connector -> refused", "originalText": "yes -> universal path\n(skips TryGetNativeScan)\npacing needs the batch loop;\nnative-only connector -> refused", "fontSize": 10, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470006, "version": 2, "versionNonce": 771002006, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0h", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "text", "id": "t1_transient_note", "x": 470, "y": 424, "width": 260, "height": 36, "text": "transient failures (HTTP 5xx/timeouts)\nclassified for retry + breaker", "originalText": "transient failures (HTTP 5xx/timeouts)\nclassified for retry + breaker", "fontSize": 9, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470007, "version": 2, "versionNonce": 771002007, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0i", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "text", "id": "t2_pacer_note", "x": 862, "y": 417, "width": 115, "height": 24, "text": "RatePacer throttles\nthe batch loop", "originalText": "RatePacer throttles\nthe batch loop", "fontSize": 9, "fontFamily": 3, "textAlign": "center", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470008, "version": 2, "versionNonce": 771002008, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0j", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true }, { "type": "text", "id": "t2_window_note", "x": 1350, "y": 590, "width": 250, "height": 24, "text": "windowed staging trimmed\nto (lo, hi] post-land", "originalText": "windowed staging trimmed\nto (lo, hi] post-land", "fontSize": 9, "fontFamily": 3, "textAlign": "left", "verticalAlign": "top", "strokeColor": "#b45309", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "seed": 470009, "version": 2, "versionNonce": 771002009, "isDeleted": false, "groupIds": [], "boundElements": [], "link": null, "locked": false, "containerId": null, "lineHeight": 1.25, "index": "b0k", "frameId": null, "roundness": null, "updated": 1783664055230, "autoResize": true } ], "appState": { "viewBackgroundColor": "#ffffff", "gridSize": 20 }, "files": {} }