Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 53 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ npm install @ioai/hdf5
## Quick start (Vite / bundlers — recommended)

```ts
import hdf5 from '@ioai/hdf5';
import hdf5, { initHdf5 } from '@ioai/hdf5';

await hdf5.ready;
await initHdf5();
hdf5.FS.mkdirTree('/work');
hdf5.FS.writeFile('/work/file.h5', uint8Array);
const file = new hdf5.File('/work/file.h5', 'r');
Expand All @@ -30,9 +30,29 @@ try {
}
```

`hdf5.ready` remains available for the same lazy default initialization:

```ts
await hdf5.ready;
```

`npm run build` in this package **requires** a prior WASM build under `dist/wasm/` (see **Building WASM**). The build copies glue into `src/wasm/` for TypeScript, runs `tsc`, then restores the untouched Emscripten glue into `dist/wasm/`.

## Escape hatch: explicit URLs (`initHdf5`)
## Loading modes

### Default SPA / bundler

For normal Vite/Webpack/Rollup apps, call `initHdf5()` with no options. The
packaged Emscripten glue is imported from the JS bundle and the sibling
`ioai_hdf5.wasm` URL is resolved by the bundler/runtime asset graph.

```ts
import { initHdf5 } from '@ioai/hdf5';

const runtime = await initHdf5();
```

### Explicit static asset URLs

For tests or non-Vite hosts, load glue/wasm from arbitrary URLs:

Expand All @@ -45,6 +65,36 @@ const runtime = await initHdf5({
});
```

You can also host both files from a common directory:

```ts
const runtime = await initHdf5({
wasmJsUrl: '/assets/ioai_hdf5.js',
assetBaseUrl: '/assets/',
});
```

### Inline/blob workers

Inline workers run from a `blob:` URL, so they should not resolve the `.wasm`
relative to `import.meta.url`. Fetch the wasm bytes from the parent realm (or
otherwise provide bytes) and pass them into the worker:

```ts
// main thread
import wasmUrl from '@ioai/hdf5/wasm/ioai_hdf5.wasm?url';

const wasmBinary = await fetch(wasmUrl).then((r) => r.arrayBuffer());
worker.postMessage({ wasmBinary }, [wasmBinary]);
```

```ts
// worker
import { initHdf5 } from '@ioai/hdf5';

const runtime = await initHdf5({ wasmBinary });
```

## Building WASM

**Production (default for CI/npm):** Docker + **HDF Group official HDF5 source** + **official zlib** — see [scripts/README-build.md](scripts/README-build.md) (`npm run build:wasm:full:docker`). Default build enables gzip/DEFLATE; SZIP/AEC is off unless you maintain a separate build profile.
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@ioai/hdf5",
"version": "0.1.5",
"version": "1.0.0",
"description": "Performance-oriented HDF5 for the web and Node: WASM core + JS IO/cache + batch API",
"type": "module",
"license": "MIT",
Expand Down Expand Up @@ -31,7 +31,9 @@
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"./wasm/ioai_hdf5.js": "./dist/wasm/ioai_hdf5.js",
"./wasm/ioai_hdf5.wasm": "./dist/wasm/ioai_hdf5.wasm"
},
"files": [
"dist",
Expand Down
64 changes: 41 additions & 23 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import type { InitHdf5Options, Hdf5Runtime, EmscriptenModule } from './types.js';
import createIoaiHdf5Module from './wasm/ioai_hdf5.js';
import { createIoaiNativeBridge } from './runtime/emscriptenBridge.js';
import { loadEmscriptenModule } from './runtime/loadEmscriptenModule.js';
import { Hdf5File as Hdf5FileBase } from './hdf5FileModels.js';
Expand All @@ -11,37 +10,56 @@ export { parseSnapshotJson, type SnapshotRoot, type SnapshotNode } from './api/s
export { Hdf5Dataset, Hdf5Group, Hdf5File } from './hdf5FileModels.js';

let _runtime: Hdf5Runtime | null = null;
let _readyPromise: Promise<Hdf5Runtime> | null = null;

/**
* Default runtime: static Emscripten glue + wasm URL (Vite / bundler asset graph).
* Await once per worker / realm before using `FS` or `File`.
* Initialize the HDF5 runtime.
*
* With no options, this uses the packaged Emscripten glue and resolves the
* sibling `.wasm` through the bundler/runtime asset graph. Inline/blob workers
* should pass `wasmBinary` (or explicit URLs) so no asset URL is resolved from
* the blob worker URL.
*/
/** Resolved at bundle/runtime time (Vite handles this pattern in dependencies). */
const wasmBinaryHref = new URL('./wasm/ioai_hdf5.wasm', import.meta.url).href;

export const ready = (async (): Promise<EmscriptenModule> => {
const moduleUnknown = await createIoaiHdf5Module({
locateFile: (path: string) => (path.endsWith('.wasm') ? wasmBinaryHref : path),
});
const module = moduleUnknown as EmscriptenModule;
export async function initHdf5(opts: InitHdf5Options = {}): Promise<Hdf5Runtime> {
const module = await loadEmscriptenModule(opts);
_runtime = { module, ioai: createIoaiNativeBridge(module) };
return module;
})();

function getRuntime(): Hdf5Runtime {
if (!_runtime) {
throw new Error('await `import("@ioai/hdf5").default.ready` before using `FS` or `File`');
}
return _runtime;
}

function getReadyRuntime(): Promise<Hdf5Runtime> {
if (_runtime) return Promise.resolve(_runtime);
_readyPromise ??= initHdf5();
return _readyPromise;
}

/**
* Escape hatch: load glue / wasm from explicit URLs (tests, non-Vite hosts).
* Prefer the default export’s `ready` in Vite apps.
* Lazy default runtime promise. It starts loading only when awaited/then'ed,
* preserving the historical `await hdf5.ready` experience without import-time
* wasm URL resolution.
*/
export async function initHdf5(opts: InitHdf5Options): Promise<Hdf5Runtime> {
const module = await loadEmscriptenModule(opts);
return { module, ioai: createIoaiNativeBridge(module) };
export const ready = {
then<TResult1 = EmscriptenModule, TResult2 = never>(
onfulfilled?: ((value: EmscriptenModule) => TResult1 | PromiseLike<TResult1>) | null,
onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,
): Promise<TResult1 | TResult2> {
return getReadyRuntime().then((runtime) => runtime.module).then(onfulfilled, onrejected);
},
catch<TResult = never>(
onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | null,
): Promise<EmscriptenModule | TResult> {
return getReadyRuntime().then((runtime) => runtime.module).catch(onrejected);
},
finally(onfinally?: (() => void) | null): Promise<EmscriptenModule> {
return getReadyRuntime().then((runtime) => runtime.module).finally(onfinally);
},
[Symbol.toStringTag]: 'Promise',
} as Promise<EmscriptenModule>;

function getRuntime(): Hdf5Runtime {
if (!_runtime) {
throw new Error('await `import("@ioai/hdf5").default.ready` or call `initHdf5()` before using `FS` or `File`');
}
return _runtime;
}

/** Open an HDF5 file on the Emscripten MEMFS path (after `await ready`). */
Expand Down
71 changes: 59 additions & 12 deletions src/runtime/loadEmscriptenModule.ts
Original file line number Diff line number Diff line change
@@ -1,24 +1,66 @@
import type { EmscriptenModule, InitHdf5Options } from '../types.js';
import { Hdf5WasmNotAvailableError } from '../errors.js';
import createIoaiHdf5Module from '../wasm/ioai_hdf5.js';

function toUrlString(u: string | URL): string {
return typeof u === 'string' ? u : u.href;
}

export async function loadEmscriptenModule(opts: InitHdf5Options): Promise<EmscriptenModule> {
const jsUrl = toUrlString(opts.wasmJsUrl);
function directoryUrl(url: string): string {
return url.endsWith('/') ? url : new URL('.', url).href;
}

function defaultScriptDirectory(opts: InitHdf5Options): string {
if (opts.assetBaseUrl) return directoryUrl(toUrlString(opts.assetBaseUrl));
if (opts.wasmJsUrl) return directoryUrl(toUrlString(opts.wasmJsUrl));
return directoryUrl(defaultPackagedWasmUrl());
}

function defaultPackagedWasmUrl(): string {
return new URL('../wasm/ioai_hdf5.wasm', import.meta.url).href;
}

function defaultLocateFile(path: string, opts: InitHdf5Options): string {
if (!opts.assetBaseUrl && !opts.wasmJsUrl && path.endsWith('.wasm')) {
return defaultPackagedWasmUrl();
}
const scriptDirectory = defaultScriptDirectory(opts);
if (opts.locateFile) {
return opts.locateFile(path, scriptDirectory);
}
return new URL(path, scriptDirectory).href;
}

function normalizeWasmBinary(binary: ArrayBuffer | Uint8Array): ArrayBuffer | Uint8Array {
if (
binary instanceof Uint8Array &&
(binary.byteOffset !== 0 || binary.byteLength !== binary.buffer.byteLength)
) {
return binary.slice();
}
return binary;
}

export async function loadEmscriptenModule(opts: InitHdf5Options = {}): Promise<EmscriptenModule> {
let modFactory: (config: Record<string, unknown>) => Promise<Record<string, unknown>>;
try {
modFactory = (await import(/* webpackIgnore: true */ /* @vite-ignore */ jsUrl)).default as (
config: Record<string, unknown>,
) => Promise<Record<string, unknown>>;
} catch (e) {
throw new Hdf5WasmNotAvailableError(
`Failed to import Emscripten glue from ${jsUrl}: ${e instanceof Error ? e.message : String(e)}`,
);
if (opts.wasmJsUrl) {
const jsUrl = toUrlString(opts.wasmJsUrl);
try {
modFactory = (await import(/* webpackIgnore: true */ /* @vite-ignore */ jsUrl)).default as (
config: Record<string, unknown>,
) => Promise<Record<string, unknown>>;
} catch (e) {
throw new Hdf5WasmNotAvailableError(
`Failed to import Emscripten glue from ${jsUrl}: ${e instanceof Error ? e.message : String(e)}`,
);
}
} else {
modFactory = createIoaiHdf5Module as (config: Record<string, unknown>) => Promise<Record<string, unknown>>;
}

const wasmBinary = opts.wasmBinaryUrl
const wasmBinary = opts.wasmBinary
? normalizeWasmBinary(opts.wasmBinary)
: opts.wasmBinaryUrl
? await fetch(toUrlString(opts.wasmBinaryUrl)).then((r) => {
if (!r.ok) throw new Hdf5WasmNotAvailableError(`Failed to fetch wasm binary: HTTP ${r.status}`);
return r.arrayBuffer();
Expand All @@ -27,7 +69,12 @@ export async function loadEmscriptenModule(opts: InitHdf5Options): Promise<Emscr

const ModuleUnknown = await modFactory({
wasmBinary,
locateFile: opts.locateFile,
locateFile: (path: string, scriptDirectory: string) =>
opts.locateFile
? opts.locateFile(path, scriptDirectory)
: opts.wasmBinary
? path
: defaultLocateFile(path, opts),
});

return ModuleUnknown as unknown as EmscriptenModule;
Expand Down
19 changes: 16 additions & 3 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,23 @@ export type EmscriptenModule = {
};

export type InitHdf5Options = {
/** URL to Emscripten `*.js` glue (MODULARIZE factory). */
wasmJsUrl: string | URL;
/** Optional wasm binary; if omitted Emscripten resolves via `locateFile`. */
/**
* URL to Emscripten `*.js` glue (MODULARIZE factory).
* Omit this in bundler builds to use the packaged glue module.
*/
wasmJsUrl?: string | URL;
/**
* Preloaded wasm bytes. This is the preferred option for inline/blob workers
* because the worker never needs to resolve a sibling `.wasm` URL.
*/
wasmBinary?: ArrayBuffer | Uint8Array;
/** Optional wasm binary URL; ignored when `wasmBinary` is provided. */
wasmBinaryUrl?: string | URL;
/**
* Base URL used by the default `locateFile` implementation. Useful when
* hosting `ioai_hdf5.js` and `ioai_hdf5.wasm` from a CDN/static directory.
*/
assetBaseUrl?: string | URL;
locateFile?: (path: string, scriptDirectory: string) => string;
};

Expand Down
56 changes: 56 additions & 0 deletions test/initHdf5.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import { initHdf5 } from '../src/index.js';

type CapturedConfig = {
wasmBinary?: ArrayBuffer | Uint8Array;
locateFile?: (path: string, scriptDirectory: string) => string;
};

function testGlobal(): typeof globalThis & { __ioaiHdf5TestConfig?: CapturedConfig } {
return globalThis as typeof globalThis & { __ioaiHdf5TestConfig?: CapturedConfig };
}

function fakeGlueModuleUrl(): string {
const source = `
export default async function createModule(config) {
globalThis.__ioaiHdf5TestConfig = config;
return {
FS: {},
HEAPU8: new Uint8Array(64),
ccall() {},
cwrap() { return () => 0; },
UTF8ToString() { return ""; },
_malloc() { return 0; },
_free() {},
getValue() { return 0; }
};
}
`;
return `data:text/javascript,${encodeURIComponent(source)}`;
}

describe('initHdf5', () => {
it('passes preloaded wasm bytes to the Emscripten factory', async () => {
const wasmBinary = new Uint8Array([1, 2, 3, 4]);
const runtime = await initHdf5({
wasmJsUrl: fakeGlueModuleUrl(),
wasmBinary,
});

expect(runtime.module.HEAPU8).toBeInstanceOf(Uint8Array);
expect(testGlobal().__ioaiHdf5TestConfig?.wasmBinary).toBe(wasmBinary);
expect(testGlobal().__ioaiHdf5TestConfig?.locateFile?.('ioai_hdf5.wasm', 'blob:http://example')).toBe(
'ioai_hdf5.wasm',
);
});

it('resolves locateFile from explicit assetBaseUrl', async () => {
await initHdf5({
wasmJsUrl: fakeGlueModuleUrl(),
assetBaseUrl: 'https://cdn.example.com/hdf5/',
});

const locateFile = testGlobal().__ioaiHdf5TestConfig?.locateFile;
expect(locateFile?.('ioai_hdf5.wasm', '')).toBe('https://cdn.example.com/hdf5/ioai_hdf5.wasm');
});
});
Loading