[{"data":1,"prerenderedAt":989},["ShallowReactive",2],{"navigation_docs":3,"-develop-author-a-batch-transform":388,"-develop-author-a-batch-transform-surround":984},[4,82,147,173,202,259,298,355],{"title":5,"icon":6,"redirect":7,"path":8,"stem":9,"children":10,"page":73},"Start","i-lucide-compass","\u002Fintro\u002Fquickstart","\u002Fintro","1.intro",[11,14,18,22,74,78],{"title":12,"path":7,"stem":13},"Quickstart","1.intro\u002F0.quickstart",{"title":15,"path":16,"stem":17},"Pipeline Applications","\u002Fintro\u002Fpipeline-app-walkthrough","1.intro\u002F1.pipeline-app-walkthrough",{"title":19,"path":20,"stem":21},"How Redgold works","\u002Fintro\u002Foverview","1.intro\u002F2.overview",{"title":23,"icon":24,"visibility":25,"redirect":26,"path":27,"stem":28,"children":29,"page":73},"Comparison","i-lucide-scale","private","\u002Fintro\u002Fcomparison\u002Fpositioning","\u002Fintro\u002Fcomparison","1.intro\u002F3.comparison",[30,33,37,41,45,49,53,57,61,65,69],{"title":31,"path":26,"stem":32},"Positioning","1.intro\u002F3.comparison\u002F1.positioning",{"title":34,"path":35,"stem":36},"vs Lovable","\u002Fintro\u002Fcomparison\u002Fvs-lovable","1.intro\u002F3.comparison\u002F10.vs-lovable",{"title":38,"path":39,"stem":40},"vs Cloudflare OS","\u002Fintro\u002Fcomparison\u002Fvs-cloudflare-os","1.intro\u002F3.comparison\u002F11.vs-cloudflare-os",{"title":42,"path":43,"stem":44},"vs a backend-as-a-service","\u002Fintro\u002Fcomparison\u002Fvs-baas","1.intro\u002F3.comparison\u002F2.vs-baas",{"title":46,"path":47,"stem":48},"vs serverless compute","\u002Fintro\u002Fcomparison\u002Fvs-serverless-compute","1.intro\u002F3.comparison\u002F3.vs-serverless-compute",{"title":50,"path":51,"stem":52},"vs a do-it-yourself data stack","\u002Fintro\u002Fcomparison\u002Fvs-diy-data-stack","1.intro\u002F3.comparison\u002F4.vs-diy-data-stack",{"title":54,"path":55,"stem":56},"vs Rama","\u002Fintro\u002Fcomparison\u002Fvs-rama","1.intro\u002F3.comparison\u002F5.vs-rama",{"title":58,"path":59,"stem":60},"vs Apache Spark","\u002Fintro\u002Fcomparison\u002Fvs-spark","1.intro\u002F3.comparison\u002F6.vs-spark",{"title":62,"path":63,"stem":64},"vs Apache DataFusion","\u002Fintro\u002Fcomparison\u002Fvs-datafusion","1.intro\u002F3.comparison\u002F7.vs-datafusion",{"title":66,"path":67,"stem":68},"vs Heroku","\u002Fintro\u002Fcomparison\u002Fvs-heroku","1.intro\u002F3.comparison\u002F8.vs-heroku",{"title":70,"path":71,"stem":72},"vs AI model providers","\u002Fintro\u002Fcomparison\u002Fvs-ai-providers","1.intro\u002F3.comparison\u002F9.vs-ai-providers",false,{"title":75,"path":76,"stem":77},"Common tasks","\u002Fintro\u002Fstarting-points","1.intro\u002F4.starting-points",{"title":79,"path":80,"stem":81},"FAQ","\u002Fintro\u002Ffaq","1.intro\u002F5.faq",{"title":83,"icon":84,"redirect":85,"path":86,"stem":87,"children":88,"page":73},"Examples","i-lucide-chef-hat","\u002Fcookbook\u002Foverview","\u002Fcookbook","10.cookbook",[89,91,95,99,103,107,111,115,119,123,127,131,135,139,143],{"title":83,"path":85,"stem":90},"10.cookbook\u002F1.overview",{"title":92,"path":93,"stem":94},"One-shot task in CI","\u002Fcookbook\u002Fone-shot-ci","10.cookbook\u002F10.one-shot-ci",{"title":96,"path":97,"stem":98},"Rate-limit retries","\u002Fcookbook\u002Fretry-rate-limits","10.cookbook\u002F11.retry-rate-limits",{"title":100,"path":101,"stem":102},"Connect GitHub review automation","\u002Fcookbook\u002Fconnect-github-app","10.cookbook\u002F12.connect-github-app",{"title":104,"path":105,"stem":106},"Delegate a task to another agent","\u002Fcookbook\u002Fdelegate-agent-task","10.cookbook\u002F13.delegate-agent-task",{"title":108,"path":109,"stem":110},"Validate a pipeline application change","\u002Fcookbook\u002Fvalidate-pipeline-app","10.cookbook\u002F14.validate-pipeline-app",{"title":112,"path":113,"stem":114},"Verify a managed application deployment","\u002Fcookbook\u002Fverify-managed-deploy","10.cookbook\u002F15.verify-managed-deploy",{"title":116,"path":117,"stem":118},"First API call","\u002Fcookbook\u002Ffirst-api-call","10.cookbook\u002F2.first-api-call",{"title":120,"path":121,"stem":122},"Python client","\u002Fcookbook\u002Fmodel-api-python","10.cookbook\u002F3.model-api-python",{"title":124,"path":125,"stem":126},"Anthropic Messages","\u002Fcookbook\u002Fanthropic-messages","10.cookbook\u002F4.anthropic-messages",{"title":128,"path":129,"stem":130},"Check credits","\u002Fcookbook\u002Fcheck-credits","10.cookbook\u002F5.check-credits",{"title":132,"path":133,"stem":134},"Query data via MCP","\u002Fcookbook\u002Fmcp-query-data","10.cookbook\u002F6.mcp-query-data",{"title":136,"path":137,"stem":138},"Drive an agent via MCP","\u002Fcookbook\u002Fmcp-drive-agent","10.cookbook\u002F7.mcp-drive-agent",{"title":140,"path":141,"stem":142},"Install the client","\u002Fcookbook\u002Finstall-client","10.cookbook\u002F8.install-client",{"title":144,"path":145,"stem":146},"Use another API host","\u002Fcookbook\u002Fpoint-client-at-host","10.cookbook\u002F9.point-client-at-host",{"title":148,"icon":149,"redirect":150,"path":151,"stem":152,"children":153,"page":73},"Agent operations","i-lucide-network","\u002Ffleet\u002Foverview","\u002Ffleet","11.fleet",[154,157,161,165,169],{"title":155,"path":150,"stem":156},"Running agent work","11.fleet\u002F0.overview",{"title":158,"path":159,"stem":160},"Workstreams and issues","\u002Ffleet\u002Fworkstreams-and-issues","11.fleet\u002F1.workstreams-and-issues",{"title":162,"path":163,"stem":164},"Agent lifecycle","\u002Ffleet\u002Fagent-lifecycle","11.fleet\u002F2.agent-lifecycle",{"title":166,"path":167,"stem":168},"Supervision and recovery","\u002Ffleet\u002Fsupervision","11.fleet\u002F3.supervision",{"title":170,"path":171,"stem":172},"An application workstream from request to delivery","\u002Ffleet\u002Fapplication-workstream","11.fleet\u002F4.application-workstream",{"title":174,"icon":175,"redirect":176,"path":177,"stem":178,"children":179,"page":73},"Architecture","i-lucide-layers","\u002Finfrastructure\u002Foverview","\u002Finfrastructure","3.infrastructure",[180,182,186,190,194,198],{"title":174,"path":176,"stem":181},"3.infrastructure\u002F0.overview",{"title":183,"path":184,"stem":185},"A request through the platform","\u002Finfrastructure\u002Frequest-path","3.infrastructure\u002F11.request-path",{"title":187,"path":188,"stem":189},"Agent work and durable state","\u002Finfrastructure\u002Fagent-work","3.infrastructure\u002F12.agent-work",{"title":191,"path":192,"stem":193},"Data and pipelines","\u002Finfrastructure\u002Fdata","3.infrastructure\u002F3.data",{"title":195,"path":196,"stem":197},"AI and agents","\u002Finfrastructure\u002Fai","3.infrastructure\u002F4.ai",{"title":199,"path":200,"stem":201},"Managed deployment","\u002Finfrastructure\u002Fdeployment","3.infrastructure\u002F7.deployment",{"title":203,"icon":204,"redirect":205,"path":206,"stem":207,"children":208,"page":73},"API","i-lucide-book-marked","\u002Freference\u002Fauthentication","\u002Freference","4.reference",[209,212,216,220,224,228,232,236,240,244,248],{"title":210,"path":205,"stem":211},"Authentication","4.reference\u002F1.authentication",{"title":213,"path":214,"stem":215},"Security disclosure policy","\u002Freference\u002Fsecurity-disclosure","4.reference\u002F11.security-disclosure",{"title":217,"path":218,"stem":219},"Model API","\u002Freference\u002Fllm-api","4.reference\u002F2.llm-api",{"title":221,"path":222,"stem":223},"MCP server","\u002Freference\u002Fmcp","4.reference\u002F2.mcp",{"title":225,"path":226,"stem":227},"Data platform preview","\u002Freference\u002Fdata-platform-preview","4.reference\u002F3.data-platform-preview",{"title":229,"path":230,"stem":231},"Terminal client","\u002Freference\u002Fcli","4.reference\u002F6.cli",{"title":233,"path":234,"stem":235},"Pipeline manifest preview","\u002Freference\u002Fpipeline-preview","4.reference\u002F7.pipeline-preview",{"title":237,"path":238,"stem":239},"Limits and retries","\u002Freference\u002Flimits","4.reference\u002F8.limits",{"title":241,"path":242,"stem":243},"API contract details","\u002Freference\u002Fapi-contracts","4.reference\u002F9.api-contracts",{"title":245,"path":246,"stem":247},"Errors","\u002Freference\u002Ferrors","4.reference\u002F9.errors",{"title":249,"icon":250,"visibility":251,"path":252,"stem":253,"children":254,"page":73},"Public API","i-lucide-route","public","\u002Freference\u002Froutes","4.reference\u002Froutes",[255],{"title":256,"path":257,"stem":258},"V1 routes","\u002Freference\u002Froutes\u002Fv1","4.reference\u002Froutes\u002Fv1",{"title":260,"icon":261,"visibility":25,"redirect":262,"path":263,"stem":264,"children":265,"page":73},"Develop","i-lucide-code","\u002Fdevelop\u002Fpipeline-apps\u002Foverview","\u002Fdevelop","5.develop",[266,270,286,290,294],{"title":267,"path":268,"stem":269},"Author a batch transform","\u002Fdevelop\u002Fauthor-a-batch-transform","5.develop\u002F10.author-a-batch-transform",{"title":271,"visibility":251,"redirect":262,"path":272,"stem":273,"children":274,"page":73},"Pipeline Apps","\u002Fdevelop\u002Fpipeline-apps","5.develop\u002F11.pipeline-apps",[275,278,282],{"title":276,"path":262,"stem":277},"Pipeline apps","5.develop\u002F11.pipeline-apps\u002F1.overview",{"title":279,"path":280,"stem":281},"Collaborative board walkthrough","\u002Fdevelop\u002Fpipeline-apps\u002Fcollaborative-board","5.develop\u002F11.pipeline-apps\u002F2.collaborative-board",{"title":283,"path":284,"stem":285},"Release and security lifecycle","\u002Fdevelop\u002Fpipeline-apps\u002Frelease-lifecycle","5.develop\u002F11.pipeline-apps\u002F3.release-lifecycle",{"title":287,"path":288,"stem":289},"Build a vertical","\u002Fdevelop\u002Fbuild-a-vertical","5.develop\u002F5.build-a-vertical",{"title":291,"path":292,"stem":293},"Managed application artifacts","\u002Fdevelop\u002Fscaffold-a-narrow-app","5.develop\u002F7.scaffold-a-narrow-app",{"title":295,"path":296,"stem":297},"Import your data","\u002Fdevelop\u002Fimport-your-data","5.develop\u002F8.import-your-data",{"title":299,"icon":300,"redirect":301,"path":302,"stem":303,"children":304,"page":73},"Guides","i-lucide-layout-grid","\u002Fapplications\u002Fai-agents\u002Foverview","\u002Fapplications","7.applications",[305,309,329,341],{"title":306,"path":307,"stem":308},"Application maturity catalog","\u002Fapplications\u002Fcatalog","7.applications\u002F0.catalog",{"title":310,"visibility":251,"path":311,"stem":312,"children":313,"page":73},"AI Agents","\u002Fapplications\u002Fai-agents","7.applications\u002F1.ai-agents",[314,317,321,325],{"title":315,"path":301,"stem":316},"Coding agents","7.applications\u002F1.ai-agents\u002F01.overview",{"title":318,"path":319,"stem":320},"GitHub App automation","\u002Fapplications\u002Fai-agents\u002Fgithub-app-automation","7.applications\u002F1.ai-agents\u002F04.github-app-automation",{"title":322,"path":323,"stem":324},"agent-cli assistance","\u002Fapplications\u002Fai-agents\u002Fagent-cli-assistance","7.applications\u002F1.ai-agents\u002F05.agent-cli-assistance",{"title":326,"path":327,"stem":328},"Agent REST control","\u002Fapplications\u002Fai-agents\u002Frest-agent-control","7.applications\u002F1.ai-agents\u002F06.rest-agent-control",{"title":330,"visibility":251,"path":331,"stem":332,"children":333,"page":73},"Tickets","\u002Fapplications\u002Ftickets","7.applications\u002F2.tickets",[334,337],{"title":330,"path":335,"stem":336},"\u002Fapplications\u002Ftickets\u002Foverview","7.applications\u002F2.tickets\u002F01.overview",{"title":338,"path":339,"stem":340},"API walkthrough","\u002Fapplications\u002Ftickets\u002Fapi-walkthrough","7.applications\u002F2.tickets\u002F02.api-walkthrough",{"title":342,"icon":343,"visibility":251,"path":344,"stem":345,"children":346,"page":73},"Case studies","i-lucide-book-open-check","\u002Fapplications\u002Fcase-studies","7.applications\u002F6.case-studies",[347,351],{"title":348,"path":349,"stem":350},"Case study: owner-scoped tickets","\u002Fapplications\u002Fcase-studies\u002Ftickets","7.applications\u002F6.case-studies\u002F01.tickets",{"title":352,"path":353,"stem":354},"Case study: repository review and correction","\u002Fapplications\u002Fcase-studies\u002Frepository-review","7.applications\u002F6.case-studies\u002F02.repository-review",{"title":356,"icon":357,"redirect":358,"path":359,"stem":360,"children":361,"page":73},"Resources","i-lucide-library","\u002Fresources\u002Fglossary","\u002Fresources","9.resources",[362,365,369,373,377,380,384],{"title":363,"path":358,"stem":364},"Glossary","9.resources\u002F1.glossary",{"title":366,"path":367,"stem":368},"Terms of Service","\u002Fresources\u002Fterms","9.resources\u002F2.terms",{"title":370,"path":371,"stem":372},"Privacy Policy","\u002Fresources\u002Fprivacy","9.resources\u002F3.privacy",{"title":374,"path":375,"stem":376},"Acceptable Use Policy","\u002Fresources\u002Facceptable-use","9.resources\u002F4.acceptable-use",{"title":79,"path":378,"stem":379},"\u002Fresources\u002Ffaq","9.resources\u002F5.faq",{"title":381,"path":382,"stem":383},"Documentation feedback","\u002Fresources\u002Fdocs-feedback","9.resources\u002F6.docs-feedback",{"title":385,"path":386,"stem":387},"Refund and Cancellation Policy","\u002Fresources\u002Frefund-cancellation","9.resources\u002F7.refund-cancellation",{"id":389,"title":267,"access":390,"audience":391,"body":392,"description":976,"extension":977,"last_verified":978,"links":979,"maturity":980,"meta":981,"navigation":595,"path":268,"seo":982,"stem":269,"__hash__":983},"docs\u002F5.develop\u002F10.author-a-batch-transform.md","onboarding","developer",{"type":393,"value":394,"toc":971},"minimark",[395,412,415,449,457,462,490,517,531,705,708,719,723,806,815,840,850,854,857,896,911,914,921,924,967],[396,397,398,399,403,404,407,408,411],"p",{},"A batch transform is a trusted native compute unit in an engine pipeline. It\naccepts one Arrow IPC stream, emits zero or more Arrow IPC streams, and loads in\nthe dataflow executor as a content-addressed Rust ",[400,401,402],"code",{},"cdylib"," through the\n",[400,405,406],{},"NATIVE_DYLIB"," tier. Use this path for reviewed platform code. Tenant-supplied\nWASM uses the separate capability-limited Wasmtime UDF lane, while untrusted\nnative or Python code uses the Firecracker ",[400,409,410],{},"MICROVM"," lane on opted-in KVM nodes.",[396,413,414],{},"The conversation extraction transform is the maintained native example:",[416,417,418,425,431,437,443],"ul",{},[419,420,421,422],"li",{},"input schema:\n",[400,423,424],{},"redgold.ai.conversation_message_gateway.ConversationMessageGateway",[419,426,427,428],{},"transform:\n",[400,429,430],{},"crates\u002Fexec\u002Fdata-transform-batch-conversation-message-extract",[419,432,433,434],{},"output schema:\n",[400,435,436],{},"redgold.ai.conversation_message_extracted.ConversationMessageExtracted",[419,438,439,440],{},"runnable pipeline:\n",[400,441,442],{},"pipelines\u002Fjobs\u002Fmemory-r1-messages.toml",[419,444,445,446],{},"inline ABI test:\n",[400,447,448],{},"crates\u002Fexec\u002Fdata-transform-batch-conversation-message-extract\u002Fsrc\u002Flib.rs",[450,451,452,453,456],"caution",{},"Batch artifact fetching is enabled on the managed dev executor variants. The\npublic dataflow executor does not expose the batch-transform surface, so\npublishing a ",[400,454,455],{},"PipelineSpec"," does not make a native batch job publicly callable.",[458,459,461],"h2",{"id":460},"create-one","Create one",[463,464,469],"pre",{"className":465,"code":466,"language":467,"meta":468,"style":468},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","rac transform new example-rows\n","bash","",[400,470,471],{"__ignoreMap":468},[472,473,476,480,484,487],"span",{"class":474,"line":475},"line",1,[472,477,479],{"class":478},"sBMFI","rac",[472,481,483],{"class":482},"sfazB"," transform",[472,485,486],{"class":482}," new",[472,488,489],{"class":482}," example-rows\n",[396,491,492,493,496,497,500,501,504,505,508,509,512,513,516],{},"This writes\n",[400,494,495],{},"crates\u002Fexec\u002Fdata-transform-batch-example-rows\u002F{Cargo.toml,src\u002Flib.rs,pipeline.toml,job.toml}",".\nThe generated transform is an identity stage with an inline ABI test. The\nscaffold test uses placeholder bytes, so replace it with a real Arrow IPC\nfixture as part of implementing the row conversion. Replace ",[400,498,499],{},"run_transform","\nwhile retaining the exported ",[400,502,503],{},"transform_batch_abi_version"," static and\n",[400,506,507],{},"transform_batch"," function.\n",[400,510,511],{},"pipeline.toml"," is the route-less PipelineManifest registered for inspection;\n",[400,514,515],{},"job.toml"," is its directly executable standalone form.",[396,518,519,520,523,524,526,527,530],{},"Fill in ",[400,521,522],{},"[package.metadata.batch-transform]"," with the input and output dataset\nnames, stable Rust schema type identifiers, the checked-in example job, and the\npipeline manifest. Move or adapt ",[400,525,515],{}," under ",[400,528,529],{},"pipelines\u002Fjobs\u002F",". Its shape\nis:",[463,532,536],{"className":533,"code":534,"language":535,"meta":468,"style":468},"language-toml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","[dataflow.source]\nkind = \"jsonl\"\nhos_prefix = \"raw\u002Fexample\u002F\"\n\n[[dataflow.stages]]\nkind = \"function\"\nqualified_name = \"data-transform-batch-example-rows\"\npin_artifact_hash = \"PUBLISH_RESULT_GOES_HERE\"\n\n[dataflow.sink]\nkind = \"hos\"\nhos_path = \"derived\u002Fexample\u002F{run_id}\u002Fexample_rows\"\n","toml",[400,537,538,557,575,590,597,613,627,642,657,662,676,690],{"__ignoreMap":468},[472,539,540,544,547,551,554],{"class":474,"line":475},[472,541,543],{"class":542},"sMK4o","[",[472,545,546],{"class":478},"dataflow",[472,548,550],{"class":549},"sTEyZ",".",[472,552,553],{"class":478},"source",[472,555,556],{"class":542},"]\n",[472,558,560,563,566,569,572],{"class":474,"line":559},2,[472,561,562],{"class":549},"kind ",[472,564,565],{"class":542},"=",[472,567,568],{"class":542}," \"",[472,570,571],{"class":482},"jsonl",[472,573,574],{"class":542},"\"\n",[472,576,578,581,583,585,588],{"class":474,"line":577},3,[472,579,580],{"class":549},"hos_prefix ",[472,582,565],{"class":542},[472,584,568],{"class":542},[472,586,587],{"class":482},"raw\u002Fexample\u002F",[472,589,574],{"class":542},[472,591,593],{"class":474,"line":592},4,[472,594,596],{"emptyLinePlaceholder":595},true,"\n",[472,598,600,603,605,607,610],{"class":474,"line":599},5,[472,601,602],{"class":542},"[[",[472,604,546],{"class":478},[472,606,550],{"class":549},[472,608,609],{"class":478},"stages",[472,611,612],{"class":542},"]]\n",[472,614,616,618,620,622,625],{"class":474,"line":615},6,[472,617,562],{"class":549},[472,619,565],{"class":542},[472,621,568],{"class":542},[472,623,624],{"class":482},"function",[472,626,574],{"class":542},[472,628,630,633,635,637,640],{"class":474,"line":629},7,[472,631,632],{"class":549},"qualified_name ",[472,634,565],{"class":542},[472,636,568],{"class":542},[472,638,639],{"class":482},"data-transform-batch-example-rows",[472,641,574],{"class":542},[472,643,645,648,650,652,655],{"class":474,"line":644},8,[472,646,647],{"class":549},"pin_artifact_hash ",[472,649,565],{"class":542},[472,651,568],{"class":542},[472,653,654],{"class":482},"PUBLISH_RESULT_GOES_HERE",[472,656,574],{"class":542},[472,658,660],{"class":474,"line":659},9,[472,661,596],{"emptyLinePlaceholder":595},[472,663,665,667,669,671,674],{"class":474,"line":664},10,[472,666,543],{"class":542},[472,668,546],{"class":478},[472,670,550],{"class":549},[472,672,673],{"class":478},"sink",[472,675,556],{"class":542},[472,677,679,681,683,685,688],{"class":474,"line":678},11,[472,680,562],{"class":549},[472,682,565],{"class":542},[472,684,568],{"class":542},[472,686,687],{"class":482},"hos",[472,689,574],{"class":542},[472,691,693,696,698,700,703],{"class":474,"line":692},12,[472,694,695],{"class":549},"hos_path ",[472,697,565],{"class":542},[472,699,568],{"class":542},[472,701,702],{"class":482},"derived\u002Fexample\u002F{run_id}\u002Fexample_rows",[472,704,574],{"class":542},[396,706,707],{},"Stored and wire schemas use the component's checked-in Rust serde types and\nCBOR. Reuse the focused schema micro-crate in transform code, and treat Arrow,\nLance, SQL, and index layouts as projections of that source type.",[396,709,710,711,714,715,718],{},"The ",[400,712,713],{},"qualified_name"," identifies the function. The ",[400,716,717],{},"pin_artifact_hash"," selects\nthe exact published bytes. Keep both in the committed job so an executor cannot\nsilently run a newer build under the same name.",[458,720,722],{"id":721},"check-and-publish","Check and publish",[463,724,726],{"className":465,"code":725,"language":467,"meta":468,"style":468},"rac transform check crates\u002Fexec\u002Fdata-transform-batch-example-rows\nrac run \\\n  --with-secret DTS_DATABASE_URL \\\n  --with-secret HETZNER_S3_URL \\\n  --with-secret HETZNER_S3_ACCESS \\\n  --with-secret HETZNER_S3_SECRET \\\n  -- \\\n  rac transform publish crates\u002Fexec\u002Fdata-transform-batch-example-rows\n",[400,727,728,740,750,760,769,778,787,794],{"__ignoreMap":468},[472,729,730,732,734,737],{"class":474,"line":475},[472,731,479],{"class":478},[472,733,483],{"class":482},[472,735,736],{"class":482}," check",[472,738,739],{"class":482}," crates\u002Fexec\u002Fdata-transform-batch-example-rows\n",[472,741,742,744,747],{"class":474,"line":559},[472,743,479],{"class":478},[472,745,746],{"class":482}," run",[472,748,749],{"class":549}," \\\n",[472,751,752,755,758],{"class":474,"line":577},[472,753,754],{"class":482},"  --with-secret",[472,756,757],{"class":482}," DTS_DATABASE_URL",[472,759,749],{"class":549},[472,761,762,764,767],{"class":474,"line":592},[472,763,754],{"class":482},[472,765,766],{"class":482}," HETZNER_S3_URL",[472,768,749],{"class":549},[472,770,771,773,776],{"class":474,"line":599},[472,772,754],{"class":482},[472,774,775],{"class":482}," HETZNER_S3_ACCESS",[472,777,749],{"class":549},[472,779,780,782,785],{"class":474,"line":615},[472,781,754],{"class":482},[472,783,784],{"class":482}," HETZNER_S3_SECRET",[472,786,749],{"class":549},[472,788,789,792],{"class":474,"line":629},[472,790,791],{"class":482},"  --",[472,793,749],{"class":549},[472,795,796,799,801,804],{"class":474,"line":644},[472,797,798],{"class":482},"  rac",[472,800,483],{"class":482},[472,802,803],{"class":482}," publish",[472,805,739],{"class":482},[396,807,808,811,812,814],{},[400,809,810],{},"check"," validates the package prefix, ",[400,813,402],{}," target, ABI dependency and\nversion, metadata identity, exported ABI symbols, pipeline reference, and local\njob reference. It is a structural check; run the crate's inline test to verify\nthe actual Arrow conversion before publication.",[396,816,817,820,821,824,825,827,828,831,832,835,836,839],{},[400,818,819],{},"publish"," builds the release library, computes its BLAKE3 digest, uploads it to\n",[400,822,823],{},"transforms\u002Fbatch\u002F\u003Cdigest>.so"," in HOS, registers the code metadata, injects the\npin into the route-less pipeline text in memory, and publishes the resulting\nsigned ",[400,826,455],{}," to DES. The\ncommand prints one ",[400,829,830],{},"registered transform=... artifact_blake3=..."," line after\nall registration steps succeed. Publication requires the four canonical vault\nitems shown above, a configured ",[400,833,834],{},"DATA_ENGINE_URL",", and a registered local\nsigning identity created through ",[400,837,838],{},"rac auth keygen",". A failed build, upload, or\nregistration returns an error without a usable new pin.",[396,841,842,843,846,847,849],{},"Copy the printed ",[400,844,845],{},"artifact_blake3"," value into the committed job's\n",[400,848,717],{},". Executors with HOS artifact fetching enabled retrieve that\nexact object and verify its BLAKE3 digest before loading it. Re-publishing\nunchanged bytes returns the same content address.",[458,851,853],{"id":852},"run-the-canonical-example","Run the canonical example",[396,855,856],{},"The bounded conversation job carries a checked-in artifact pin:",[463,858,860],{"className":465,"code":859,"language":467,"meta":468,"style":468},"dataflow-runner --user-id system-backfill submit \\\n  --manifest pipelines\u002Fjobs\u002Fmemory-r1-messages-validate.toml \\\n  --run-id conversation-extract-check\n",[400,861,862,878,888],{"__ignoreMap":468},[472,863,864,867,870,873,876],{"class":474,"line":475},[472,865,866],{"class":478},"dataflow-runner",[472,868,869],{"class":482}," --user-id",[472,871,872],{"class":482}," system-backfill",[472,874,875],{"class":482}," submit",[472,877,749],{"class":549},[472,879,880,883,886],{"class":474,"line":559},[472,881,882],{"class":482},"  --manifest",[472,884,885],{"class":482}," pipelines\u002Fjobs\u002Fmemory-r1-messages-validate.toml",[472,887,749],{"class":549},[472,889,890,893],{"class":474,"line":577},[472,891,892],{"class":482},"  --run-id",[472,894,895],{"class":482}," conversation-extract-check\n",[396,897,898,899,902,903,906,907,910],{},"The bounded job reads 50 gateway objects, invokes the Rust transform, deduplicates\non its ",[400,900,901],{},"data_hash"," output column, and writes Lance output to HOS. The full\n",[400,904,905],{},"memory-r1-messages.toml"," job uses the same transform and pin; its companion\n",[400,908,909],{},"memory-r1-messages-compact.toml"," performs cross-partition deduplication.",[396,912,913],{},"The route-less manifest in the transform crate and the two runnable memory jobs\ncurrently pin different published revisions. Treat each checked-in hash as an\nindependent deployment choice; publication does not rewrite any of those files.",[396,915,916,917,920],{},"The source crate is the implementation reference because its inline test\nconstructs real Arrow IPC input, calls the exported ABI, decodes every emitted\nbatch, and verifies ",[400,918,919],{},"data_hash = blake3(primary_key)"," for each result.",[396,922,923],{},"Before committing a new pin, run the transform crate's focused inline test and\nvalidate the two manifest forms:",[463,925,927],{"className":465,"code":926,"language":467,"meta":468,"style":468},"timeout 300 rac cargo test \\\n  -p redgold-data-transform-batch-example-rows\nrac transform check crates\u002Fexec\u002Fdata-transform-batch-example-rows\n",[400,928,929,949,957],{"__ignoreMap":468},[472,930,931,934,938,941,944,947],{"class":474,"line":475},[472,932,933],{"class":478},"timeout",[472,935,937],{"class":936},"sbssI"," 300",[472,939,940],{"class":482}," rac",[472,942,943],{"class":482}," cargo",[472,945,946],{"class":482}," test",[472,948,749],{"class":549},[472,950,951,954],{"class":474,"line":559},[472,952,953],{"class":482},"  -p",[472,955,956],{"class":482}," redgold-data-transform-batch-example-rows\n",[472,958,959,961,963,965],{"class":474,"line":577},[472,960,479],{"class":478},[472,962,483],{"class":482},[472,964,736],{"class":482},[472,966,739],{"class":482},[968,969,970],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}",{"title":468,"searchDepth":559,"depth":559,"links":972},[973,974,975],{"id":460,"depth":559,"text":461},{"id":721,"depth":559,"text":722},{"id":852,"depth":559,"text":853},"Scaffold, check, publish, and pin a trusted Rust transform that processes Arrow IPC batches in the dataflow engine.","md","2026-08-14",null,"preview",{"toc":595,"visibility":251},{"title":267,"description":976},"u7VzfGOJO8stW2Z6iuv8eP-LnzCACWFFywNZIqah0_Y",[985,987],{"title":256,"path":257,"stem":258,"description":986,"children":-1},"Generated reference for the supported V1 API routes.",{"title":276,"path":262,"stem":277,"description":988,"children":-1},"Portable application contracts, strict TOML compilation, and a shared declarative host built on the pipeline backend.",1789873213247]