From aa5be50afda93edf0776f8a103409a3e0ca1cf62 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 21:53:41 +0000 Subject: [PATCH 1/2] feat(cli): replace xgenext2fs and cartesi-machine spawns with native bindings Build ext2 drives with @deroll/genext2fs and configure, boot, store and hash the Cartesi machine with @deroll/cm, instead of spawning xgenext2fs, cartesi-machine and cartesi-machine-stored-hash (falling back to running them inside the SDK docker image). machine.ts now translates a cartesi.toml Config into an emulator MachineConfig directly, mirroring what the cartesi-machine CLI does with its command line: the boot args it appends to, the init script (splash, flash drive mounts and chowns, environment exports, WORKDIR and USER), the flash drives with root first so it lands on pmem0, and the virtio console setup for an interactive shell. buildMachineConfig is a pure function, covered by unit tests. With no SDK image to take the kernel from, images.ts downloads the pinned cartesi/image-kernel release on first use, verifies its checksum and caches it under XDG_CACHE_HOME. CARTESI_IMAGES_PATH and machine.ram_image still win. The emulator moves from 0.20 to 0.21, so machine hashes change. The standalone binaries are gone: a bun single file executable has no node_modules, and both addons resolve their platform .node at runtime, so they cannot be embedded. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UTEd5g3mF849BATTstssR3 --- .changeset/cuddly-pugs-invite.md | 28 ++ .github/workflows/release.yaml | 11 - CLAUDE.md | 6 +- apps/cli/build.ts | 33 +- apps/cli/package.json | 2 + apps/cli/src/builder/directory.ts | 1 - apps/cli/src/builder/docker.ts | 1 - apps/cli/src/builder/empty.ts | 2 - apps/cli/src/builder/tar.ts | 1 - apps/cli/src/commands/build.ts | 24 +- apps/cli/src/commands/shell.ts | 31 +- apps/cli/src/config.ts | 4 +- .../src/exec/cartesi-machine-stored-hash.ts | 41 +- apps/cli/src/exec/cartesi-machine.ts | 149 +++++-- apps/cli/src/exec/genext2fs.ts | 133 ++++--- apps/cli/src/images.ts | 156 ++++++++ apps/cli/src/machine.ts | 372 ++++++++++++++---- .../tests/integration/builder/empty.test.ts | 24 +- .../exec/cartesi-machine-stored-hash.test.ts | 41 +- .../integration/exec/cartesi-machine.test.ts | 27 -- .../tests/integration/exec/genext2fs.test.ts | 25 -- apps/cli/tests/unit/config/fixtures/full.toml | 2 +- .../tests/unit/exec/cartesi-machine.test.ts | 24 ++ apps/cli/tests/unit/exec/genext2fs.test.ts | 86 ++++ apps/cli/tests/unit/machine.test.ts | 298 ++++++++++++++ bun.lock | 18 + 26 files changed, 1163 insertions(+), 377 deletions(-) create mode 100644 .changeset/cuddly-pugs-invite.md create mode 100644 apps/cli/src/images.ts delete mode 100644 apps/cli/tests/integration/exec/cartesi-machine.test.ts delete mode 100644 apps/cli/tests/integration/exec/genext2fs.test.ts create mode 100644 apps/cli/tests/unit/exec/cartesi-machine.test.ts create mode 100644 apps/cli/tests/unit/exec/genext2fs.test.ts create mode 100644 apps/cli/tests/unit/machine.test.ts diff --git a/.changeset/cuddly-pugs-invite.md b/.changeset/cuddly-pugs-invite.md new file mode 100644 index 00000000..3a17e1ee --- /dev/null +++ b/.changeset/cuddly-pugs-invite.md @@ -0,0 +1,28 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `xgenext2fs` and `cartesi-machine` subprocesses with native bindings + +ext2 drives are now built with [`@deroll/genext2fs`](https://deroll.dev/genext2fs), and the +Cartesi machine is configured, booted, stored and hashed with +[`@deroll/cm`](https://deroll.dev/cm). Both are N-API addons, so `build`, `shell` and `status` +no longer shell out to `xgenext2fs`, `cartesi-machine` or `cartesi-machine-stored-hash`, and no +longer fall back to running them inside the SDK Docker image. Docker is still required to build +the root drive from a Dockerfile, and for squashfs drives (`mksquashfs`). + +Notable consequences: + +- **The machine emulator moved from 0.20 to 0.21**, which is the version `@deroll/cm` links + against. Machine hashes change, and applications have to be redeployed. +- **The Linux kernel image is downloaded and cached.** With no SDK image to take it from, the + default `ram_image` now comes from the pinned `cartesi/image-kernel` v0.21.0 release, fetched + on first use into `$XDG_CACHE_HOME/cartesi/images` (`~/.cache/cartesi/images`) and verified + against its SHA-256. A `CARTESI_IMAGES_PATH` directory containing the image is used when set, + and `machine.ram_image` in `cartesi.toml` still takes precedence over both. +- **Boot args are no longer double quoted.** The old code passed `--append-bootargs=""` to + the CLI without a shell, so the quotes ended up in the kernel command line. They are gone now. +- **The standalone binaries are no longer built or released.** A Bun single file executable has + no `node_modules`, and both addons resolve their platform specific `.node` at runtime, so they + cannot be embedded — not even for the host platform. npm is the only distribution now, and the + homebrew formula has to install the package from there. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2c30ff38..8b59cbf1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,17 +69,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Release CLI binaries - if: ${{ steps.changeset.outputs.published == 'true' && contains(fromJSON(steps.changeset.outputs.publishedPackages).*.name, '@cartesi/cli') }} - run: | - for f in cartesi-*; do tar -czf "$f.tar.gz" "$f"; done - VERSION=$(jq -r '.[] | select(.name=="@cartesi/cli") | .version' <<< '${{ steps.changeset.outputs.publishedPackages }}') - TAG="@cartesi/cli@${VERSION}" - gh release upload "$TAG" cartesi-*.tar.gz - working-directory: ./apps/cli/bin - env: - GH_TOKEN: ${{ github.token }} - build_sdk: name: Build SDK needs: release diff --git a/CLAUDE.md b/CLAUDE.md index 986d1553..c208324f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). It also produces native binaries for darwin-arm64, darwin-x64, linux-arm64, linux-x64 in `apps/cli/bin/`. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@deroll/cm` and `@deroll/genext2fs` are left external — they are native addons that resolve their platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,9 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Wrappers around subprocess execution (cartesi-machine, rollups) using `execa`. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine`, `cartesi-machine-stored-hash` and `genext2fs` are native N-API bindings (`@deroll/cm`, `@deroll/genext2fs`); `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives), mirroring what the `cartesi-machine` CLI does with its command line. +- **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/image-kernel` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. - **`contracts.ts`** — Generated contract addresses and ABI bindings (via `@wagmi/cli`). - **`wallet.ts`** — Wallet utilities using `viem` for Ethereum interaction. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 3ed24ffb..d287b440 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,35 +1,22 @@ +// native addons resolve their platform binary at runtime, so they can never be +// bundled: they are left as imports, resolved from node_modules +const external = ["@deroll/cm", "@deroll/genext2fs"]; + // build for npm package await Bun.build({ banner: "#!/usr/bin/env node", entrypoints: ["./src/index.ts"], + external, minify: true, outdir: "dist", sourcemap: true, target: "node", }); -// build bun binaries for all supported platforms -const targets: Bun.Build.CompileTarget[] = [ - "bun-darwin-arm64", - "bun-darwin-x64", - "bun-linux-arm64", - "bun-linux-x64", -]; - -await Promise.all( - targets.map((target) => - Bun.build({ - bytecode: true, - compile: { - outfile: `bin/cartesi-${target.replace("bun-", "")}`, - target, - }, - entrypoints: ["./src/index.ts"], - minify: true, - sourcemap: "linked", - target: "bun", - }), - ), -); +// NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) +// are gone. A single file executable has no node_modules, and the emulator and +// ext2 bindings resolve their platform specific .node at runtime, so they +// cannot be embedded — not even for the host platform. The npm package is the +// only distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 95ad6649..2587cece 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -16,6 +16,8 @@ ], "dependencies": { "@commander-js/extra-typings": "^14.0.0", + "@deroll/cm": "^0.2.0-alpha.4", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", diff --git a/apps/cli/src/builder/directory.ts b/apps/cli/src/builder/directory.ts index 55eb5398..8204714f 100644 --- a/apps/cli/src/builder/directory.ts +++ b/apps/cli/src/builder/directory.ts @@ -27,7 +27,6 @@ export const build = async ( input: name, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index 908b3acd..2f891b6b 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -162,7 +162,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/empty.ts b/apps/cli/src/builder/empty.ts index 9ddc0475..b52cb4f0 100644 --- a/apps/cli/src/builder/empty.ts +++ b/apps/cli/src/builder/empty.ts @@ -6,7 +6,6 @@ import { genext2fs } from "../exec/index.js"; export const build = async ( name: string, drive: EmptyDriveConfig, - sdkImage: string, destination: string, ): Promise => { const filename = `${name}.${drive.format}`; @@ -16,7 +15,6 @@ export const build = async ( output: filename, size: drive.size, cwd: destination, - image: sdkImage, }); break; } diff --git a/apps/cli/src/builder/tar.ts b/apps/cli/src/builder/tar.ts index 34633ecc..fe34c8f7 100644 --- a/apps/cli/src/builder/tar.ts +++ b/apps/cli/src/builder/tar.ts @@ -24,7 +24,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index fa6cdde9..b0e3c16d 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -63,7 +63,7 @@ const buildDriveTask = ( break; } case "empty": { - await buildEmpty(name, drive, sdk, destination); + await buildEmpty(name, drive, destination); break; } case "tar": { @@ -154,20 +154,30 @@ export const createBuildCommand = () => { } // create machine snapshot - await bootMachine( + const { exitCode, rootHash } = await bootMachine( config, result.imageInfo, { + cwd: destination, finalHash: true, + reporter: (line) => console.error(line), store: "image", }, - { - cwd: destination, - stdio: "inherit", - }, ); - // make snapshot readable by all users, because cartesi-machine sets to 600 + if (exitCode !== 0) { + throw new Error( + exitCode === 2 + ? "Machine did not stop at a rollup accept, it is not a valid rolling template" + : `Machine stopped with exit code ${exitCode}`, + ); + } + + if (rootHash) { + console.error(`Machine hash: ${chalk.cyan(rootHash)}`); + } + + // make snapshot readable by all users, because the emulator sets to 600 await fs.chmod(path.join(destination, "image"), 0o755); }); }; diff --git a/apps/cli/src/commands/shell.ts b/apps/cli/src/commands/shell.ts index 9ffaa8c8..3f8da783 100755 --- a/apps/cli/src/commands/shell.ts +++ b/apps/cli/src/commands/shell.ts @@ -1,5 +1,4 @@ import { Command } from "@commander-js/extra-typings"; -import { ExecaError } from "execa"; import fs from "fs-extra"; import path from "node:path"; import { getApplicationConfig, getContextPath } from "../base.js"; @@ -39,26 +38,16 @@ export const createShellCommand = () => { // run as root if flag is set config.machine.user = runAsRoot ? "root" : undefined; - // boot machine - try { - await bootMachine( - config, - undefined, - { interactive: true }, // start with interactive mode on - { - cwd: destination, - stdio: "inherit", - tty: true, - }, - ); - } catch (error: unknown) { - if (error instanceof ExecaError) { - // just continue gracefully - if (error.exitCode === 130) { - return; - } - throw error; - } + // boot machine, in interactive mode + const { exitCode } = await bootMachine(config, undefined, { + cwd: destination, + interactive: true, + reporter: (line) => console.error(line), + }); + + // 130 is the shell being interrupted, which is not a failure + if (exitCode !== 0 && exitCode !== 130) { + throw new Error(`Machine stopped with exit code ${exitCode}`); } }); }; diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index 7096f8ff..43e1d2d1 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -162,9 +162,9 @@ export type MachineConfig = { entrypoint?: string; env: Record; // explicit environment variables injected into cartesi-machine ENV envFile?: string; // path to a .env file with environment variables injected into cartesi-machine ENV - maxMCycle?: bigint; // default given by cartesi-machine + maxMCycle?: bigint; // default is no limit ramLength: string; - ramImage?: string; // default given by cartesi-machine + ramImage?: string; // default is the pinned cartesi image-kernel release useDockerEnv: boolean; // inject docker image ENV into cartesi-machine ENV useDockerWorkdir: boolean; // inject docker image WORKDIR into cartesi-machine WORKDIR user?: string; // default given by cartesi-machine diff --git a/apps/cli/src/exec/cartesi-machine-stored-hash.ts b/apps/cli/src/exec/cartesi-machine-stored-hash.ts index 6c360f3f..6a7af2c2 100644 --- a/apps/cli/src/exec/cartesi-machine-stored-hash.ts +++ b/apps/cli/src/exec/cartesi-machine-stored-hash.ts @@ -1,42 +1,21 @@ -import { isHash, type Hash } from "viem"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; -import { execaDockerFallback, type DockerFallbackOptions } from "./util.js"; - -type ComputeHashOptions = { cwd?: string } & DockerFallbackOptions; +import { load } from "@deroll/cm"; +import type { Hash } from "viem"; /** - * - * @param machineDir - * @param options - * @returns + * Reads the root hash of a stored Cartesi machine snapshot. + * @param machineDir directory holding the machine snapshot + * @returns the machine hash, or undefined if the snapshot can't be read */ export const computeHash = async ( machineDir: string, - options?: ComputeHashOptions, ): Promise => { - const defaultImage = `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`; - const execaOptions = Object.assign( - {}, - { image: defaultImage, cwd: process.cwd() }, - options, - ); - try { - const { stdout } = await execaDockerFallback( - "cartesi-machine-stored-hash", - [machineDir], - execaOptions, - ); - - if (undefined !== stdout) { - const hash = `0x${stdout.toString().trim()}`; - - if (isHash(hash)) { - return hash; - } + const machine = load(machineDir); + try { + return `0x${machine.getRootHash().toString("hex")}`; + } finally { + machine.destroy(); } - - return undefined; } catch { return undefined; } diff --git a/apps/cli/src/exec/cartesi-machine.ts b/apps/cli/src/exec/cartesi-machine.ts index 2a27ab0f..92ca5575 100644 --- a/apps/cli/src/exec/cartesi-machine.ts +++ b/apps/cli/src/exec/cartesi-machine.ts @@ -1,33 +1,126 @@ -import { parse, Range, type SemVer } from "semver"; import { - execaDockerFallback, - type DockerFallbackOptions, - type ExecaOptionsDockerFallback, -} from "./util.js"; - -export const requiredVersion = new Range("^0.20.0"); - -export const boot = ( - args: readonly string[], - options: ExecaOptionsDockerFallback, -) => execaDockerFallback("cartesi-machine", args, options); - -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { image } = options || {}; + BreakReason, + create, + getDefaultConfig, + getVersion, + HtifYieldCommand, + HtifYieldReason, + type MachineConfig, + type MachineRuntimeConfig, + MAX_MCYCLE, + Reg, +} from "@deroll/cm"; +import { parse, Range, type SemVer } from "semver"; +import type { Hash } from "viem"; + +export const requiredVersion = new Range("^0.21.0"); + +export type RunOptions = { + /** fail unless the machine stopped at a rollup accept yield */ + assertRollingTemplate?: boolean; + /** compute the machine root hash once the run is over */ + finalHash?: boolean; + /** target mcycle to stop at, defaults to no limit */ + maxMCycle?: bigint; + /** directory to store the machine snapshot into */ + store?: string; +}; + +export type RunResult = { + breakReason: BreakReason; + /** exit code of the guest, mirroring what the cartesi-machine CLI reports */ + exitCode: number; + rootHash?: Hash; +}; + +/** Machine configuration defaults, as filled in by the emulator itself. */ +export const defaultConfig = (): MachineConfig => getDefaultConfig(); + +/** + * A machine stopped at a halt, a manual yield, or an mcycle overflow no longer + * advances on its own. + */ +const isAtFixedPoint = (breakReason: BreakReason): boolean => + breakReason === BreakReason.Halted || + breakReason === BreakReason.YieldedManually || + breakReason === BreakReason.McycleOverflow; + +/** + * Creates a machine, runs it to a fixed point (or to the requested mcycle), + * and optionally hashes and stores it. Automatic yields are acknowledged and + * discarded, and console I/O breaks just resume the run, which is what the + * cartesi-machine CLI does for a plain boot. + */ +export const run = ( + config: MachineConfig, + runtimeConfig: MachineRuntimeConfig | undefined, + options: RunOptions = {}, +): RunResult => { + const machine = create(config, runtimeConfig); try { - const { stdout } = await execaDockerFallback( - "cartesi-machine", - ["--version-json"], - { image }, - ); - if (typeof stdout === "string") { - const output = JSON.parse(stdout); - return parse(output.version); + const target = options.maxMCycle ?? MAX_MCYCLE; + let breakReason: BreakReason; + for (;;) { + breakReason = machine.run(target); + if ( + isAtFixedPoint(breakReason) || + breakReason === BreakReason.ReachedTargetMcycle + ) { + break; + } + if (breakReason === BreakReason.YieldedAutomatically) { + // acknowledge the yield so the machine can carry on + machine.receiveCmioRequest(); + } + // any other reason (a soft yield or console I/O) just keeps going } - return null; - } catch { - return null; + + let exitCode = 0; + if (breakReason === BreakReason.Halted) { + exitCode = Number(machine.readReg(Reg.HtifToHostData) >> 1n); + } else if (breakReason === BreakReason.McycleOverflow) { + exitCode = 1; + } + + const rootHash: Hash | undefined = options.finalHash + ? `0x${machine.getRootHash().toString("hex")}` + : undefined; + + if (options.store) { + machine.store(options.store); + } + + if (options.assertRollingTemplate && exitCode === 0) { + // the machine must be sitting at a rollup accept, waiting for input + try { + const { cmd, reason } = machine.receiveCmioRequest(); + if ( + cmd !== HtifYieldCommand.Manual || + reason !== HtifYieldReason.ManualRxAccepted + ) { + exitCode = 2; + } + } catch { + exitCode = 2; + } + } + + return { breakReason, exitCode, rootHash }; + } finally { + machine.destroy(); } }; + +/** + * Version of the machine emulator the bindings were linked against. It is + * fixed at build time, so this is a plain lookup and not a subprocess call + * anymore. + */ +export const version = (): SemVer | null => { + // encoded as (major * 1000000) + (minor * 1000) + patch + const encoded = getVersion(); + const major = encoded / 1000000n; + const minor = (encoded / 1000n) % 1000n; + const patch = encoded % 1000n; + return parse(`${major}.${minor}.${patch}`); +}; diff --git a/apps/cli/src/exec/genext2fs.ts b/apps/cli/src/exec/genext2fs.ts index 99ba584e..ff2caa8b 100644 --- a/apps/cli/src/exec/genext2fs.ts +++ b/apps/cli/src/exec/genext2fs.ts @@ -1,90 +1,97 @@ +import { + createImage, + type Genext2fsResult, + tarToExt2, + version as vendoredVersion, +} from "@deroll/genext2fs"; +import path from "node:path"; import { parse, Range, type SemVer } from "semver"; -import { type DockerFallbackOptions, execaDockerFallback } from "./util.js"; +import type { Reporter } from "./util.js"; const BLOCK_SIZE = 4096; // fixed at 4k export const requiredVersion: Range = new Range("^1.5.6"); -const baseArgs = (options: { extraBlocks: number }) => [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--readjustment", - `+${options.extraBlocks}`, -]; +type BaseOptions = { + /** directory the input and output filenames are relative to */ + cwd?: string; + reporter?: Reporter; +}; + +const resolve = (cwd: string | undefined, filename: string): string => + cwd ? path.resolve(cwd, filename) : path.resolve(filename); + +/** + * Forwards the diagnostics xgenext2fs produced to the reporter, one line at a + * time, the same way the spawned process' stderr used to be piped. + */ +const report = (result: Genext2fsResult, reporter?: Reporter): void => { + if (!reporter) { + return; + } + for (const line of result.stderr.split("\n")) { + if (line.trim()) { + reporter(line.trimEnd()); + } + } +}; -export const empty = ( +export const empty = async ( options: { - cwd?: string; size: number; output: string; - } & DockerFallbackOptions, -) => { - const { size, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, size, output, reporter } = options; const blocks = Math.ceil(size / BLOCK_SIZE); // size in blocks - return execaDockerFallback( - "xgenext2fs", - [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--size-in-blocks", - blocks.toString(), - output, - ], - { ...options, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + sizeInBlocks: blocks, + }); + report(result, reporter); + return result; }; -export const fromDirectory = ( +export const fromDirectory = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--root", input, output], - { cwd, image, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + layers: [{ type: "directory", path: resolve(cwd, input) }], + }); + report(result, reporter); + return result; }; -export const fromTar = ( +export const fromTar = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--tarball", input, output], - { cwd, image, reporter }, - ); + const result = await tarToExt2(resolve(cwd, input), resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + }); + report(result, reporter); + return result; }; -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { stdout } = await execaDockerFallback( - "xgenext2fs", - ["--version"], - options || {}, - ); - if (typeof stdout === "string") { - const regex = - /(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?/; - const m = stdout.match(regex); - if (m?.[0]) { - return parse(m[0]); - } - } - return null; -}; +/** + * Version of the xgenext2fs the bindings were built against. It is fixed at + * build time, so this is a plain lookup and not a subprocess call anymore. + */ +export const version = (): SemVer | null => parse(vendoredVersion); diff --git a/apps/cli/src/images.ts b/apps/cli/src/images.ts new file mode 100644 index 00000000..ef0a54e6 --- /dev/null +++ b/apps/cli/src/images.ts @@ -0,0 +1,156 @@ +import bytes from "bytes"; +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import type { Reporter } from "./exec/util.js"; + +/** + * Version of https://github.com/cartesi/image-kernel the machine boots by + * default, matched to the emulator version @deroll/cm is linked against. + */ +export const DEFAULT_IMAGE_KERNEL_VERSION = "0.21.0"; +export const DEFAULT_LINUX_KERNEL_VERSION = "6.5.13-ctsi-2"; +const DEFAULT_LINUX_KERNEL_SHA256 = + "5c900060da2db2bfa84cd39cd9cd722988c83c42225f3cac55f2d3157e48f32f"; + +const RELEASE_URL = `https://github.com/cartesi/image-kernel/releases/download/v${DEFAULT_IMAGE_KERNEL_VERSION}/linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_IMAGE_KERNEL_VERSION}.bin`; + +export class ImageDownloadError extends Error { + constructor(url: string, reason: string) { + super(`Failed to download ${url}: ${reason}`); + this.name = "ImageDownloadError"; + } +} + +export class ImageChecksumError extends Error { + constructor(filename: string, expected: string, actual: string) { + super( + `Checksum mismatch for ${filename}: expected ${expected}, got ${actual}`, + ); + this.name = "ImageChecksumError"; + } +} + +/** + * Directory the CLI caches downloaded machine images in. Honors XDG_CACHE_HOME + * when set, and falls back to the platform home directory otherwise. + */ +export const getCacheDir = (): string => { + const base = + process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache"); + return path.join(base, "cartesi", "images"); +}; + +const sha256 = async (filename: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of fs.createReadStream(filename)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +}; + +const download = async ( + url: string, + destination: string, + checksum: string, + reporter?: Reporter, +): Promise => { + const response = await fetch(url); + if (!response.ok || !response.body) { + throw new ImageDownloadError( + url, + `${response.status} ${response.statusText}`, + ); + } + + const total = Number(response.headers.get("content-length") ?? 0); + await fs.mkdirp(path.dirname(destination)); + + // download to a temporary file next to the destination, so a partial or + // corrupt download is never mistaken for a cached image + const partial = `${destination}.${process.pid}.part`; + try { + const hash = createHash("sha256"); + let downloaded = 0; + let reported = -1; + const file = fs.createWriteStream(partial); + try { + for await (const chunk of response.body) { + const buffer = chunk as Uint8Array; + hash.update(buffer); + downloaded += buffer.byteLength; + if (!file.write(buffer)) { + await new Promise((resolve) => file.once("drain", resolve)); + } + // report every 10%, chunks are far too small to be worth a + // line each + const percent = + total > 0 ? Math.floor((downloaded * 10) / total) * 10 : -1; + if (reporter && percent > reported) { + reported = percent; + reporter( + `Downloading ${path.basename(destination)}: ${bytes(downloaded)} of ${bytes(total)} (${percent}%)`, + ); + } + } + } finally { + await new Promise((resolve, reject) => + file.end((error?: Error) => + error ? reject(error) : resolve(), + ), + ); + } + + const actual = hash.digest("hex"); + if (actual !== checksum) { + throw new ImageChecksumError( + path.basename(destination), + checksum, + actual, + ); + } + + await fs.move(partial, destination, { overwrite: true }); + } finally { + await fs.remove(partial); + } +}; + +/** + * Resolves the Linux kernel image the machine boots from, downloading it into + * the cache directory on first use. + * + * The lookup order is the CARTESI_IMAGES_PATH directory (the same environment + * variable the cartesi-machine CLI honors), then the cache directory, and + * finally the pinned image-kernel release. + */ +export const getRamImage = async (reporter?: Reporter): Promise => { + const filename = `linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_IMAGE_KERNEL_VERSION}.bin`; + + const imagesPath = process.env.CARTESI_IMAGES_PATH; + if (imagesPath) { + for (const candidate of [ + path.join(imagesPath, filename), + path.join(imagesPath, "linux.bin"), + ]) { + if (await fs.pathExists(candidate)) { + return candidate; + } + } + } + + const cached = path.join(getCacheDir(), filename); + if (await fs.pathExists(cached)) { + // a cached image with the wrong contents is a corrupt cache, not a + // reason to fail: drop it and download again + if ((await sha256(cached)) === DEFAULT_LINUX_KERNEL_SHA256) { + return cached; + } + await fs.remove(cached); + } + + reporter?.(`Downloading Linux kernel image ${filename}`); + await download(RELEASE_URL, cached, DEFAULT_LINUX_KERNEL_SHA256, reporter); + return cached; +}; diff --git a/apps/cli/src/machine.ts b/apps/cli/src/machine.ts index b7f347ee..53bbc94b 100644 --- a/apps/cli/src/machine.ts +++ b/apps/cli/src/machine.ts @@ -1,57 +1,181 @@ +import type { + MachineConfig as EmulatorMachineConfig, + MachineRuntimeConfig, + MemoryRangeConfig, +} from "@deroll/cm"; import dotenv from "dotenv"; import fs from "node:fs"; +import path from "node:path"; import type { Config, DriveConfig, ImageInfo } from "./config.js"; import { cartesiMachine } from "./exec/index.js"; -import type { ExecaOptionsDockerFallback } from "./exec/util.js"; +import type { Reporter } from "./exec/util.js"; +import { getRamImage } from "./images.js"; -const flashDrive = (label: string, drive: DriveConfig): string => { - const { format, mount, shared, user } = drive; - const filename = `${label}.${format}`; - const vars = [`label:${label}`, `data_filename:${filename}`]; - if (mount !== undefined) { - vars.push(`mount:${mount}`); +export class InvalidMemorySizeError extends Error { + constructor(value: string) { + super(`Invalid memory size: ${value}`); + this.name = "InvalidMemorySizeError"; } - if (user) { - vars.push(`user:${user}`); +} + +export class InvalidEnvNameError extends Error { + constructor(name: string) { + super(`Invalid environment variable name: ${name}`); + this.name = "InvalidEnvNameError"; + } +} + +const SHIFTS: Record = { + Ki: 10n, + Mi: 20n, + Gi: 30n, + Ti: 40n, +}; + +/** + * Parses a memory size the way the cartesi-machine CLI does: a decimal or + * 0x-prefixed hexadecimal integer, optionally followed by a Ki/Mi/Gi/Ti + * suffix or a `<< n` shift. + */ +export const parseMemorySize = (value: string): bigint => { + const match = value.trim().match(/^(0[xX][0-9a-fA-F]+|[0-9]+)\s*(.*)$/); + if (!match) { + throw new InvalidMemorySizeError(value); + } + const [, literal, suffix] = match; + const size = BigInt( + literal.toLowerCase().startsWith("0x") + ? literal.toLowerCase() + : literal.replace(/^0+(?=[0-9])/, ""), + ); + + const rest = suffix.trim(); + let shift = 0n; + if (rest !== "") { + const shiftMatch = rest.match(/^<<\s*([0-9]+)$/); + if (rest in SHIFTS) { + shift = SHIFTS[rest]; + } else if (shiftMatch) { + shift = BigInt(shiftMatch[1]); + } else { + throw new InvalidMemorySizeError(value); + } } - if (shared) { - vars.push("shared"); + + if (size === 0n) { + return 0n; + } + if (shift >= 64n || size >> (64n - shift) !== 0n) { + throw new InvalidMemorySizeError(value); } - // don't specify start and length - return `--flash-drive=${vars.join(",")}`; + return size << shift; }; -export type BootMachineOptions = { - finalHash?: boolean; - interactive?: boolean; - store?: string; +/** + * Splash the cartesi-machine CLI prints from init on boot. Every backslash is + * doubled so `echo` emits the art verbatim. + */ +const SPLASH = String.raw`echo " + . + / \\ + / \\ +\\---/---\\ /----\\ + \\ X \\ + \\----/ \\---/---\\ + \\ / CARTESI + \\ / MACHINE + ' +" +`; + +/** + * Mount point of a drive, following the cartesi-machine defaults: a drive + * backed by a file is mounted at /mnt/