diff --git a/apps/api/ops/resolve-witness-hash-fees.mjs b/apps/api/ops/resolve-witness-hash-fees.mjs new file mode 100644 index 0000000..1f95257 --- /dev/null +++ b/apps/api/ops/resolve-witness-hash-fees.mjs @@ -0,0 +1,230 @@ +#!/usr/bin/env node +/** + * Resolve Bitcoin fees for the pre-662000 Counterparty transactions whose `tx_hash` is a witness + * hash (wtxid) rather than a txid, so no hash-indexed Bitcoin API can serve them. + * + * Counterparty recorded the wtxid as a transaction's identity for segwit transactions between the + * `segwit_support` activation (block 557,236) and `correct_segwit_txids` (block 662,000). Those + * identities are consensus history — upstream fixed the behaviour forward-only and can never rewrite + * them — so the mirror keeps the wtxid (CLAUDE.md rule 7) and only the derived fee is repaired here. + * + * The mapping is recovered exactly: fetch each block's raw bytes, compute both the txid (serialization + * without witness) and the wtxid (full serialization) for every transaction, and match ours by wtxid. + * No Bitcoin node required. Writes a {tx_index, tx_hash, txid, fee} record per resolved row. + * + * node ops/resolve-witness-hash-fees.mjs --rows=rows.json --out=fees.json + * + * Resumable: re-running skips rows already present in --out. + */ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; + +const arg = (name, fallback) => + process.argv.find((value) => value.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; + +const ROWS_PATH = arg("rows", "rows.json"); +const OUT_PATH = arg("out", "fees.json"); +const API = arg("api", "https://mempool.space/api").replace(/\/$/, ""); +const BLOCK_CONCURRENCY = Number(arg("concurrency", "2")); +const PAUSE_MS = Number(arg("pause-ms", "250")); + +const sha256 = (buffer) => createHash("sha256").update(buffer).digest(); +const hash256 = (buffer) => sha256(sha256(buffer)); +const littleEndianHex = (buffer) => Buffer.from(buffer).reverse().toString("hex"); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Sequential reader over a raw block. */ +class Reader { + constructor(buffer) { + this.buffer = buffer; + this.offset = 0; + } + u8() { + return this.buffer[this.offset++]; + } + u32() { + const value = this.buffer.readUInt32LE(this.offset); + this.offset += 4; + return value; + } + u64() { + const value = this.buffer.readBigUInt64LE(this.offset); + this.offset += 8; + return value; + } + varint() { + const first = this.u8(); + if (first < 0xfd) return first; + if (first === 0xfd) { + const value = this.buffer.readUInt16LE(this.offset); + this.offset += 2; + return value; + } + if (first === 0xfe) return this.u32(); + return Number(this.u64()); + } + skip(length) { + this.offset += length; + } +} + +/** + * Consume one transaction and return both identities. txid hashes version + inputs + outputs + + * locktime; wtxid hashes the transaction exactly as serialized, marker/flag and witnesses included. + */ +function readTransaction(reader) { + const start = reader.offset; + const versionStart = reader.offset; + reader.u32(); + const versionEnd = reader.offset; + + let segwit = false; + if (reader.buffer[reader.offset] === 0x00 && reader.buffer[reader.offset + 1] === 0x01) { + segwit = true; + reader.skip(2); + } + + const bodyStart = reader.offset; + const inputCount = reader.varint(); + for (let index = 0; index < inputCount; index++) { + reader.skip(36); // previous txid + vout + reader.skip(reader.varint()); // scriptSig + reader.u32(); // sequence + } + const outputCount = reader.varint(); + for (let index = 0; index < outputCount; index++) { + reader.u64(); // value + reader.skip(reader.varint()); // scriptPubKey + } + const bodyEnd = reader.offset; + + if (segwit) + for (let input = 0; input < inputCount; input++) { + const items = reader.varint(); + for (let item = 0; item < items; item++) reader.skip(reader.varint()); + } + + const locktimeStart = reader.offset; + reader.u32(); + const end = reader.offset; + + const full = reader.buffer.subarray(start, end); + const stripped = Buffer.concat([ + reader.buffer.subarray(versionStart, versionEnd), + reader.buffer.subarray(bodyStart, bodyEnd), + reader.buffer.subarray(locktimeStart, end), + ]); + return { txid: littleEndianHex(hash256(stripped)), wtxid: littleEndianHex(hash256(full)) }; +} + +async function get(path, kind) { + for (let attempt = 0; ; attempt++) { + const response = await fetch(`${API}${path}`, { signal: AbortSignal.timeout(60_000) }); + if (response.ok) return kind === "buffer" ? Buffer.from(await response.arrayBuffer()) : await response.text(); + await response.body?.cancel(); + const retryable = response.status === 429 || response.status >= 500; + if (!retryable || attempt >= 4) throw new Error(`${path} ${response.status}`); + await delay(Math.min(16_000, 1_000 * 2 ** attempt)); + } +} + +/** Map every wtxid wanted in this block to its canonical txid. */ +async function resolveBlock(height, wanted) { + const blockHash = (await get(`/block-height/${height}`, "text")).trim(); + const raw = await get(`/block/${blockHash}/raw`, "buffer"); + const reader = new Reader(raw); + reader.skip(80); + const count = reader.varint(); + const remaining = new Map(wanted.map((row) => [row.tx_hash, row])); + const resolved = []; + for (let index = 0; index < count && remaining.size > 0; index++) { + const { txid, wtxid } = readTransaction(reader); + const row = remaining.get(wtxid) ?? remaining.get(txid); + if (!row) continue; + remaining.delete(row.tx_hash); + resolved.push({ ...row, txid }); + } + return { resolved, unresolved: [...remaining.values()] }; +} + +async function fetchFee(txid) { + const transaction = JSON.parse(await get(`/tx/${txid}`, "text")); + const fee = transaction.fee; + if (!Number.isSafeInteger(fee) || fee < 0) throw new Error(`invalid fee for ${txid}`); + return fee; +} + +const rows = JSON.parse(readFileSync(ROWS_PATH, "utf8")); +const done = existsSync(OUT_PATH) ? JSON.parse(readFileSync(OUT_PATH, "utf8")) : []; +const doneIndexes = new Set(done.map((row) => row.tx_index)); +const todo = rows.filter((row) => !doneIndexes.has(row.tx_index)); + +const byBlock = new Map(); +for (const row of todo) { + if (!byBlock.has(row.block_index)) byBlock.set(row.block_index, []); + byBlock.get(row.block_index).push(row); +} + +console.log( + JSON.stringify({ + event: "start", + rows: rows.length, + already_done: done.length, + todo: todo.length, + blocks: byBlock.size, + }), +); + +const heights = [...byBlock.keys()]; +const failures = []; +const unresolved = []; +let cursor = 0; +let completedBlocks = 0; + +async function worker() { + for (;;) { + const slot = cursor++; + if (slot >= heights.length) return; + const height = heights[slot]; + const wanted = byBlock.get(height); + try { + const result = await resolveBlock(height, wanted); + for (const row of result.resolved) { + row.fee = await fetchFee(row.txid); + done.push(row); + } + unresolved.push(...result.unresolved); + } catch (error) { + failures.push({ height, error: error instanceof Error ? error.message : String(error) }); + } + completedBlocks++; + if (completedBlocks % 10 === 0 || completedBlocks === heights.length) { + writeFileSync(OUT_PATH, JSON.stringify(done, null, 1)); + console.log( + JSON.stringify({ + event: "progress", + blocks: `${completedBlocks}/${heights.length}`, + resolved: done.length, + unresolved: unresolved.length, + failed_blocks: failures.length, + }), + ); + } + await delay(PAUSE_MS); + } +} + +await Promise.all(Array.from({ length: Math.max(1, BLOCK_CONCURRENCY) }, () => worker())); + +writeFileSync(OUT_PATH, JSON.stringify(done, null, 1)); +console.log( + JSON.stringify({ + event: "complete", + requested: rows.length, + resolved: done.length, + unresolved: unresolved.length, + unresolved_sample: unresolved.slice(0, 5), + failed_blocks: failures.length, + failures: failures.slice(0, 5), + }), +);