diff --git a/.changeset/cuddly-pugs-invite.md b/.changeset/cuddly-pugs-invite.md new file mode 100644 index 00000000..39b19890 --- /dev/null +++ b/.changeset/cuddly-pugs-invite.md @@ -0,0 +1,28 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `xgenext2fs` and `cartesi-machine` subprocesses with native bindings + +ext2 drives are now built with [`@deroll/genext2fs`](https://deroll.dev/genext2fs), and the +Cartesi machine is configured, booted, stored and hashed with +[`@deroll/cm`](https://deroll.dev/cm). Both are N-API addons, so `build`, `shell` and `status` +no longer shell out to `xgenext2fs`, `cartesi-machine` or `cartesi-machine-stored-hash`, and no +longer fall back to running them inside the SDK Docker image. Docker is still required to build +the root drive from a Dockerfile, and for squashfs drives (`mksquashfs`). + +Notable consequences: + +- **The machine emulator moved from 0.20 to 0.21**, which is the version `@deroll/cm` links + against. Machine hashes change, and applications have to be redeployed. +- **The Linux kernel image is downloaded and cached.** With no SDK image to take it from, the + default `ram_image` now comes from the pinned `cartesi/machine-linux-image` v0.21.0 release, + fetched on first use into `$XDG_CACHE_HOME/cartesi/images` (`~/.cache/cartesi/images`) and verified + against its SHA-256. A `CARTESI_IMAGES_PATH` directory containing the image is used when set, + and `machine.ram_image` in `cartesi.toml` still takes precedence over both. +- **Boot args are no longer double quoted.** The old code passed `--append-bootargs=""` to + the CLI without a shell, so the quotes ended up in the kernel command line. They are gone now. +- **The standalone binaries are no longer built or released.** A Bun single file executable has + no `node_modules`, and both addons resolve their platform specific `.node` at runtime, so they + cannot be embedded — not even for the host platform. npm is the only distribution now, and the + homebrew formula has to install the package from there. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2c30ff38..8b59cbf1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,17 +69,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Release CLI binaries - if: ${{ steps.changeset.outputs.published == 'true' && contains(fromJSON(steps.changeset.outputs.publishedPackages).*.name, '@cartesi/cli') }} - run: | - for f in cartesi-*; do tar -czf "$f.tar.gz" "$f"; done - VERSION=$(jq -r '.[] | select(.name=="@cartesi/cli") | .version' <<< '${{ steps.changeset.outputs.publishedPackages }}') - TAG="@cartesi/cli@${VERSION}" - gh release upload "$TAG" cartesi-*.tar.gz - working-directory: ./apps/cli/bin - env: - GH_TOKEN: ${{ github.token }} - build_sdk: name: Build SDK needs: release diff --git a/CLAUDE.md b/CLAUDE.md index 986d1553..df9c10d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). It also produces native binaries for darwin-arm64, darwin-x64, linux-arm64, linux-x64 in `apps/cli/bin/`. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@deroll/cm` and `@deroll/genext2fs` are left external — they are native addons that resolve their platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,9 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Wrappers around subprocess execution (cartesi-machine, rollups) using `execa`. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine`, `cartesi-machine-stored-hash` and `genext2fs` are native N-API bindings (`@deroll/cm`, `@deroll/genext2fs`); `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives), mirroring what the `cartesi-machine` CLI does with its command line. +- **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. - **`contracts.ts`** — Generated contract addresses and ABI bindings (via `@wagmi/cli`). - **`wallet.ts`** — Wallet utilities using `viem` for Ethereum interaction. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 3ed24ffb..d287b440 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,35 +1,22 @@ +// native addons resolve their platform binary at runtime, so they can never be +// bundled: they are left as imports, resolved from node_modules +const external = ["@deroll/cm", "@deroll/genext2fs"]; + // build for npm package await Bun.build({ banner: "#!/usr/bin/env node", entrypoints: ["./src/index.ts"], + external, minify: true, outdir: "dist", sourcemap: true, target: "node", }); -// build bun binaries for all supported platforms -const targets: Bun.Build.CompileTarget[] = [ - "bun-darwin-arm64", - "bun-darwin-x64", - "bun-linux-arm64", - "bun-linux-x64", -]; - -await Promise.all( - targets.map((target) => - Bun.build({ - bytecode: true, - compile: { - outfile: `bin/cartesi-${target.replace("bun-", "")}`, - target, - }, - entrypoints: ["./src/index.ts"], - minify: true, - sourcemap: "linked", - target: "bun", - }), - ), -); +// NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) +// are gone. A single file executable has no node_modules, and the emulator and +// ext2 bindings resolve their platform specific .node at runtime, so they +// cannot be embedded — not even for the host platform. The npm package is the +// only distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 95ad6649..2587cece 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -16,6 +16,8 @@ ], "dependencies": { "@commander-js/extra-typings": "^14.0.0", + "@deroll/cm": "^0.2.0-alpha.4", + "@deroll/genext2fs": "^0.2.0-alpha.0", "@inquirer/confirm": "^6.0.6", "@inquirer/core": "^11.1.3", "@inquirer/input": "^5.0.6", diff --git a/apps/cli/src/builder/directory.ts b/apps/cli/src/builder/directory.ts index 55eb5398..8204714f 100644 --- a/apps/cli/src/builder/directory.ts +++ b/apps/cli/src/builder/directory.ts @@ -27,7 +27,6 @@ export const build = async ( input: name, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index 908b3acd..2f891b6b 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -162,7 +162,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/builder/empty.ts b/apps/cli/src/builder/empty.ts index 9ddc0475..b52cb4f0 100644 --- a/apps/cli/src/builder/empty.ts +++ b/apps/cli/src/builder/empty.ts @@ -6,7 +6,6 @@ import { genext2fs } from "../exec/index.js"; export const build = async ( name: string, drive: EmptyDriveConfig, - sdkImage: string, destination: string, ): Promise => { const filename = `${name}.${drive.format}`; @@ -16,7 +15,6 @@ export const build = async ( output: filename, size: drive.size, cwd: destination, - image: sdkImage, }); break; } diff --git a/apps/cli/src/builder/tar.ts b/apps/cli/src/builder/tar.ts index 34633ecc..fe34c8f7 100644 --- a/apps/cli/src/builder/tar.ts +++ b/apps/cli/src/builder/tar.ts @@ -24,7 +24,6 @@ export const build = async ( input: tar, output: filename, cwd: destination, - image: sdkImage, reporter, }); break; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index fa6cdde9..b0e3c16d 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -63,7 +63,7 @@ const buildDriveTask = ( break; } case "empty": { - await buildEmpty(name, drive, sdk, destination); + await buildEmpty(name, drive, destination); break; } case "tar": { @@ -154,20 +154,30 @@ export const createBuildCommand = () => { } // create machine snapshot - await bootMachine( + const { exitCode, rootHash } = await bootMachine( config, result.imageInfo, { + cwd: destination, finalHash: true, + reporter: (line) => console.error(line), store: "image", }, - { - cwd: destination, - stdio: "inherit", - }, ); - // make snapshot readable by all users, because cartesi-machine sets to 600 + if (exitCode !== 0) { + throw new Error( + exitCode === 2 + ? "Machine did not stop at a rollup accept, it is not a valid rolling template" + : `Machine stopped with exit code ${exitCode}`, + ); + } + + if (rootHash) { + console.error(`Machine hash: ${chalk.cyan(rootHash)}`); + } + + // make snapshot readable by all users, because the emulator sets to 600 await fs.chmod(path.join(destination, "image"), 0o755); }); }; diff --git a/apps/cli/src/commands/shell.ts b/apps/cli/src/commands/shell.ts index 9ffaa8c8..3f8da783 100755 --- a/apps/cli/src/commands/shell.ts +++ b/apps/cli/src/commands/shell.ts @@ -1,5 +1,4 @@ import { Command } from "@commander-js/extra-typings"; -import { ExecaError } from "execa"; import fs from "fs-extra"; import path from "node:path"; import { getApplicationConfig, getContextPath } from "../base.js"; @@ -39,26 +38,16 @@ export const createShellCommand = () => { // run as root if flag is set config.machine.user = runAsRoot ? "root" : undefined; - // boot machine - try { - await bootMachine( - config, - undefined, - { interactive: true }, // start with interactive mode on - { - cwd: destination, - stdio: "inherit", - tty: true, - }, - ); - } catch (error: unknown) { - if (error instanceof ExecaError) { - // just continue gracefully - if (error.exitCode === 130) { - return; - } - throw error; - } + // boot machine, in interactive mode + const { exitCode } = await bootMachine(config, undefined, { + cwd: destination, + interactive: true, + reporter: (line) => console.error(line), + }); + + // 130 is the shell being interrupted, which is not a failure + if (exitCode !== 0 && exitCode !== 130) { + throw new Error(`Machine stopped with exit code ${exitCode}`); } }); }; diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index 7096f8ff..93fb9ab8 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -162,9 +162,9 @@ export type MachineConfig = { entrypoint?: string; env: Record; // explicit environment variables injected into cartesi-machine ENV envFile?: string; // path to a .env file with environment variables injected into cartesi-machine ENV - maxMCycle?: bigint; // default given by cartesi-machine + maxMCycle?: bigint; // default is no limit ramLength: string; - ramImage?: string; // default given by cartesi-machine + ramImage?: string; // default is the pinned cartesi machine-linux-image release useDockerEnv: boolean; // inject docker image ENV into cartesi-machine ENV useDockerWorkdir: boolean; // inject docker image WORKDIR into cartesi-machine WORKDIR user?: string; // default given by cartesi-machine diff --git a/apps/cli/src/exec/cartesi-machine-stored-hash.ts b/apps/cli/src/exec/cartesi-machine-stored-hash.ts index 6c360f3f..6a7af2c2 100644 --- a/apps/cli/src/exec/cartesi-machine-stored-hash.ts +++ b/apps/cli/src/exec/cartesi-machine-stored-hash.ts @@ -1,42 +1,21 @@ -import { isHash, type Hash } from "viem"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; -import { execaDockerFallback, type DockerFallbackOptions } from "./util.js"; - -type ComputeHashOptions = { cwd?: string } & DockerFallbackOptions; +import { load } from "@deroll/cm"; +import type { Hash } from "viem"; /** - * - * @param machineDir - * @param options - * @returns + * Reads the root hash of a stored Cartesi machine snapshot. + * @param machineDir directory holding the machine snapshot + * @returns the machine hash, or undefined if the snapshot can't be read */ export const computeHash = async ( machineDir: string, - options?: ComputeHashOptions, ): Promise => { - const defaultImage = `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`; - const execaOptions = Object.assign( - {}, - { image: defaultImage, cwd: process.cwd() }, - options, - ); - try { - const { stdout } = await execaDockerFallback( - "cartesi-machine-stored-hash", - [machineDir], - execaOptions, - ); - - if (undefined !== stdout) { - const hash = `0x${stdout.toString().trim()}`; - - if (isHash(hash)) { - return hash; - } + const machine = load(machineDir); + try { + return `0x${machine.getRootHash().toString("hex")}`; + } finally { + machine.destroy(); } - - return undefined; } catch { return undefined; } diff --git a/apps/cli/src/exec/cartesi-machine.ts b/apps/cli/src/exec/cartesi-machine.ts index 2a27ab0f..92ca5575 100644 --- a/apps/cli/src/exec/cartesi-machine.ts +++ b/apps/cli/src/exec/cartesi-machine.ts @@ -1,33 +1,126 @@ -import { parse, Range, type SemVer } from "semver"; import { - execaDockerFallback, - type DockerFallbackOptions, - type ExecaOptionsDockerFallback, -} from "./util.js"; - -export const requiredVersion = new Range("^0.20.0"); - -export const boot = ( - args: readonly string[], - options: ExecaOptionsDockerFallback, -) => execaDockerFallback("cartesi-machine", args, options); - -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { image } = options || {}; + BreakReason, + create, + getDefaultConfig, + getVersion, + HtifYieldCommand, + HtifYieldReason, + type MachineConfig, + type MachineRuntimeConfig, + MAX_MCYCLE, + Reg, +} from "@deroll/cm"; +import { parse, Range, type SemVer } from "semver"; +import type { Hash } from "viem"; + +export const requiredVersion = new Range("^0.21.0"); + +export type RunOptions = { + /** fail unless the machine stopped at a rollup accept yield */ + assertRollingTemplate?: boolean; + /** compute the machine root hash once the run is over */ + finalHash?: boolean; + /** target mcycle to stop at, defaults to no limit */ + maxMCycle?: bigint; + /** directory to store the machine snapshot into */ + store?: string; +}; + +export type RunResult = { + breakReason: BreakReason; + /** exit code of the guest, mirroring what the cartesi-machine CLI reports */ + exitCode: number; + rootHash?: Hash; +}; + +/** Machine configuration defaults, as filled in by the emulator itself. */ +export const defaultConfig = (): MachineConfig => getDefaultConfig(); + +/** + * A machine stopped at a halt, a manual yield, or an mcycle overflow no longer + * advances on its own. + */ +const isAtFixedPoint = (breakReason: BreakReason): boolean => + breakReason === BreakReason.Halted || + breakReason === BreakReason.YieldedManually || + breakReason === BreakReason.McycleOverflow; + +/** + * Creates a machine, runs it to a fixed point (or to the requested mcycle), + * and optionally hashes and stores it. Automatic yields are acknowledged and + * discarded, and console I/O breaks just resume the run, which is what the + * cartesi-machine CLI does for a plain boot. + */ +export const run = ( + config: MachineConfig, + runtimeConfig: MachineRuntimeConfig | undefined, + options: RunOptions = {}, +): RunResult => { + const machine = create(config, runtimeConfig); try { - const { stdout } = await execaDockerFallback( - "cartesi-machine", - ["--version-json"], - { image }, - ); - if (typeof stdout === "string") { - const output = JSON.parse(stdout); - return parse(output.version); + const target = options.maxMCycle ?? MAX_MCYCLE; + let breakReason: BreakReason; + for (;;) { + breakReason = machine.run(target); + if ( + isAtFixedPoint(breakReason) || + breakReason === BreakReason.ReachedTargetMcycle + ) { + break; + } + if (breakReason === BreakReason.YieldedAutomatically) { + // acknowledge the yield so the machine can carry on + machine.receiveCmioRequest(); + } + // any other reason (a soft yield or console I/O) just keeps going } - return null; - } catch { - return null; + + let exitCode = 0; + if (breakReason === BreakReason.Halted) { + exitCode = Number(machine.readReg(Reg.HtifToHostData) >> 1n); + } else if (breakReason === BreakReason.McycleOverflow) { + exitCode = 1; + } + + const rootHash: Hash | undefined = options.finalHash + ? `0x${machine.getRootHash().toString("hex")}` + : undefined; + + if (options.store) { + machine.store(options.store); + } + + if (options.assertRollingTemplate && exitCode === 0) { + // the machine must be sitting at a rollup accept, waiting for input + try { + const { cmd, reason } = machine.receiveCmioRequest(); + if ( + cmd !== HtifYieldCommand.Manual || + reason !== HtifYieldReason.ManualRxAccepted + ) { + exitCode = 2; + } + } catch { + exitCode = 2; + } + } + + return { breakReason, exitCode, rootHash }; + } finally { + machine.destroy(); } }; + +/** + * Version of the machine emulator the bindings were linked against. It is + * fixed at build time, so this is a plain lookup and not a subprocess call + * anymore. + */ +export const version = (): SemVer | null => { + // encoded as (major * 1000000) + (minor * 1000) + patch + const encoded = getVersion(); + const major = encoded / 1000000n; + const minor = (encoded / 1000n) % 1000n; + const patch = encoded % 1000n; + return parse(`${major}.${minor}.${patch}`); +}; diff --git a/apps/cli/src/exec/genext2fs.ts b/apps/cli/src/exec/genext2fs.ts index 99ba584e..ff2caa8b 100644 --- a/apps/cli/src/exec/genext2fs.ts +++ b/apps/cli/src/exec/genext2fs.ts @@ -1,90 +1,97 @@ +import { + createImage, + type Genext2fsResult, + tarToExt2, + version as vendoredVersion, +} from "@deroll/genext2fs"; +import path from "node:path"; import { parse, Range, type SemVer } from "semver"; -import { type DockerFallbackOptions, execaDockerFallback } from "./util.js"; +import type { Reporter } from "./util.js"; const BLOCK_SIZE = 4096; // fixed at 4k export const requiredVersion: Range = new Range("^1.5.6"); -const baseArgs = (options: { extraBlocks: number }) => [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--readjustment", - `+${options.extraBlocks}`, -]; +type BaseOptions = { + /** directory the input and output filenames are relative to */ + cwd?: string; + reporter?: Reporter; +}; + +const resolve = (cwd: string | undefined, filename: string): string => + cwd ? path.resolve(cwd, filename) : path.resolve(filename); + +/** + * Forwards the diagnostics xgenext2fs produced to the reporter, one line at a + * time, the same way the spawned process' stderr used to be piped. + */ +const report = (result: Genext2fsResult, reporter?: Reporter): void => { + if (!reporter) { + return; + } + for (const line of result.stderr.split("\n")) { + if (line.trim()) { + reporter(line.trimEnd()); + } + } +}; -export const empty = ( +export const empty = async ( options: { - cwd?: string; size: number; output: string; - } & DockerFallbackOptions, -) => { - const { size, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, size, output, reporter } = options; const blocks = Math.ceil(size / BLOCK_SIZE); // size in blocks - return execaDockerFallback( - "xgenext2fs", - [ - "--block-size", - BLOCK_SIZE.toString(), - "--faketime", - "--size-in-blocks", - blocks.toString(), - output, - ], - { ...options, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + sizeInBlocks: blocks, + }); + report(result, reporter); + return result; }; -export const fromDirectory = ( +export const fromDirectory = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--root", input, output], - { cwd, image, reporter }, - ); + const result = await createImage(resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + layers: [{ type: "directory", path: resolve(cwd, input) }], + }); + report(result, reporter); + return result; }; -export const fromTar = ( +export const fromTar = async ( options: { - cwd?: string; extraSize: number; input: string; output: string; - } & DockerFallbackOptions, -) => { - const { cwd, extraSize, image, input, output, reporter } = options; + } & BaseOptions, +): Promise => { + const { cwd, extraSize, input, output, reporter } = options; const extraBlocks = Math.ceil(extraSize / BLOCK_SIZE); - return execaDockerFallback( - "xgenext2fs", - [...baseArgs({ extraBlocks }), "--tarball", input, output], - { cwd, image, reporter }, - ); + const result = await tarToExt2(resolve(cwd, input), resolve(cwd, output), { + blockSize: BLOCK_SIZE, + faketime: true, + readjustment: `+${extraBlocks}`, + }); + report(result, reporter); + return result; }; -export const version = async ( - options?: DockerFallbackOptions, -): Promise => { - const { stdout } = await execaDockerFallback( - "xgenext2fs", - ["--version"], - options || {}, - ); - if (typeof stdout === "string") { - const regex = - /(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?/; - const m = stdout.match(regex); - if (m?.[0]) { - return parse(m[0]); - } - } - return null; -}; +/** + * Version of the xgenext2fs the bindings were built against. It is fixed at + * build time, so this is a plain lookup and not a subprocess call anymore. + */ +export const version = (): SemVer | null => parse(vendoredVersion); diff --git a/apps/cli/src/images.ts b/apps/cli/src/images.ts new file mode 100644 index 00000000..7a7f7cb2 --- /dev/null +++ b/apps/cli/src/images.ts @@ -0,0 +1,156 @@ +import bytes from "bytes"; +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import type { Reporter } from "./exec/util.js"; + +/** + * Version of https://github.com/cartesi/machine-linux-image the machine boots + * by default, matched to the emulator version @deroll/cm is linked against. + */ +export const DEFAULT_LINUX_IMAGE_VERSION = "0.21.0"; +export const DEFAULT_LINUX_KERNEL_VERSION = "6.5.13-ctsi-2"; +const DEFAULT_LINUX_KERNEL_SHA256 = + "5c900060da2db2bfa84cd39cd9cd722988c83c42225f3cac55f2d3157e48f32f"; + +const RELEASE_URL = `https://github.com/cartesi/machine-linux-image/releases/download/v${DEFAULT_LINUX_IMAGE_VERSION}/linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + +export class ImageDownloadError extends Error { + constructor(url: string, reason: string) { + super(`Failed to download ${url}: ${reason}`); + this.name = "ImageDownloadError"; + } +} + +export class ImageChecksumError extends Error { + constructor(filename: string, expected: string, actual: string) { + super( + `Checksum mismatch for ${filename}: expected ${expected}, got ${actual}`, + ); + this.name = "ImageChecksumError"; + } +} + +/** + * Directory the CLI caches downloaded machine images in. Honors XDG_CACHE_HOME + * when set, and falls back to the platform home directory otherwise. + */ +export const getCacheDir = (): string => { + const base = + process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache"); + return path.join(base, "cartesi", "images"); +}; + +const sha256 = async (filename: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of fs.createReadStream(filename)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +}; + +const download = async ( + url: string, + destination: string, + checksum: string, + reporter?: Reporter, +): Promise => { + const response = await fetch(url); + if (!response.ok || !response.body) { + throw new ImageDownloadError( + url, + `${response.status} ${response.statusText}`, + ); + } + + const total = Number(response.headers.get("content-length") ?? 0); + await fs.mkdirp(path.dirname(destination)); + + // download to a temporary file next to the destination, so a partial or + // corrupt download is never mistaken for a cached image + const partial = `${destination}.${process.pid}.part`; + try { + const hash = createHash("sha256"); + let downloaded = 0; + let reported = -1; + const file = fs.createWriteStream(partial); + try { + for await (const chunk of response.body) { + const buffer = chunk as Uint8Array; + hash.update(buffer); + downloaded += buffer.byteLength; + if (!file.write(buffer)) { + await new Promise((resolve) => file.once("drain", resolve)); + } + // report every 10%, chunks are far too small to be worth a + // line each + const percent = + total > 0 ? Math.floor((downloaded * 10) / total) * 10 : -1; + if (reporter && percent > reported) { + reported = percent; + reporter( + `Downloading ${path.basename(destination)}: ${bytes(downloaded)} of ${bytes(total)} (${percent}%)`, + ); + } + } + } finally { + await new Promise((resolve, reject) => + file.end((error?: Error) => + error ? reject(error) : resolve(), + ), + ); + } + + const actual = hash.digest("hex"); + if (actual !== checksum) { + throw new ImageChecksumError( + path.basename(destination), + checksum, + actual, + ); + } + + await fs.move(partial, destination, { overwrite: true }); + } finally { + await fs.remove(partial); + } +}; + +/** + * Resolves the Linux kernel image the machine boots from, downloading it into + * the cache directory on first use. + * + * The lookup order is the CARTESI_IMAGES_PATH directory (the same environment + * variable the cartesi-machine CLI honors), then the cache directory, and + * finally the pinned machine-linux-image release. + */ +export const getRamImage = async (reporter?: Reporter): Promise => { + const filename = `linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + + const imagesPath = process.env.CARTESI_IMAGES_PATH; + if (imagesPath) { + for (const candidate of [ + path.join(imagesPath, filename), + path.join(imagesPath, "linux.bin"), + ]) { + if (await fs.pathExists(candidate)) { + return candidate; + } + } + } + + const cached = path.join(getCacheDir(), filename); + if (await fs.pathExists(cached)) { + // a cached image with the wrong contents is a corrupt cache, not a + // reason to fail: drop it and download again + if ((await sha256(cached)) === DEFAULT_LINUX_KERNEL_SHA256) { + return cached; + } + await fs.remove(cached); + } + + reporter?.(`Downloading Linux kernel image ${filename}`); + await download(RELEASE_URL, cached, DEFAULT_LINUX_KERNEL_SHA256, reporter); + return cached; +}; diff --git a/apps/cli/src/machine.ts b/apps/cli/src/machine.ts index b7f347ee..53bbc94b 100644 --- a/apps/cli/src/machine.ts +++ b/apps/cli/src/machine.ts @@ -1,57 +1,181 @@ +import type { + MachineConfig as EmulatorMachineConfig, + MachineRuntimeConfig, + MemoryRangeConfig, +} from "@deroll/cm"; import dotenv from "dotenv"; import fs from "node:fs"; +import path from "node:path"; import type { Config, DriveConfig, ImageInfo } from "./config.js"; import { cartesiMachine } from "./exec/index.js"; -import type { ExecaOptionsDockerFallback } from "./exec/util.js"; +import type { Reporter } from "./exec/util.js"; +import { getRamImage } from "./images.js"; -const flashDrive = (label: string, drive: DriveConfig): string => { - const { format, mount, shared, user } = drive; - const filename = `${label}.${format}`; - const vars = [`label:${label}`, `data_filename:${filename}`]; - if (mount !== undefined) { - vars.push(`mount:${mount}`); +export class InvalidMemorySizeError extends Error { + constructor(value: string) { + super(`Invalid memory size: ${value}`); + this.name = "InvalidMemorySizeError"; } - if (user) { - vars.push(`user:${user}`); +} + +export class InvalidEnvNameError extends Error { + constructor(name: string) { + super(`Invalid environment variable name: ${name}`); + this.name = "InvalidEnvNameError"; + } +} + +const SHIFTS: Record = { + Ki: 10n, + Mi: 20n, + Gi: 30n, + Ti: 40n, +}; + +/** + * Parses a memory size the way the cartesi-machine CLI does: a decimal or + * 0x-prefixed hexadecimal integer, optionally followed by a Ki/Mi/Gi/Ti + * suffix or a `<< n` shift. + */ +export const parseMemorySize = (value: string): bigint => { + const match = value.trim().match(/^(0[xX][0-9a-fA-F]+|[0-9]+)\s*(.*)$/); + if (!match) { + throw new InvalidMemorySizeError(value); + } + const [, literal, suffix] = match; + const size = BigInt( + literal.toLowerCase().startsWith("0x") + ? literal.toLowerCase() + : literal.replace(/^0+(?=[0-9])/, ""), + ); + + const rest = suffix.trim(); + let shift = 0n; + if (rest !== "") { + const shiftMatch = rest.match(/^<<\s*([0-9]+)$/); + if (rest in SHIFTS) { + shift = SHIFTS[rest]; + } else if (shiftMatch) { + shift = BigInt(shiftMatch[1]); + } else { + throw new InvalidMemorySizeError(value); + } } - if (shared) { - vars.push("shared"); + + if (size === 0n) { + return 0n; + } + if (shift >= 64n || size >> (64n - shift) !== 0n) { + throw new InvalidMemorySizeError(value); } - // don't specify start and length - return `--flash-drive=${vars.join(",")}`; + return size << shift; }; -export type BootMachineOptions = { - finalHash?: boolean; - interactive?: boolean; - store?: string; +/** + * Splash the cartesi-machine CLI prints from init on boot. Every backslash is + * doubled so `echo` emits the art verbatim. + */ +const SPLASH = String.raw`echo " + . + / \\ + / \\ +\\---/---\\ /----\\ + \\ X \\ + \\----/ \\---/---\\ + \\ / CARTESI + \\ / MACHINE + ' +" +`; + +/** + * Mount point of a drive, following the cartesi-machine defaults: a drive + * backed by a file is mounted at /mnt/