From 7fad46e870d3ae38e10b4f619dc4150b7a05d0cd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 17:46:36 +0000 Subject: [PATCH 1/6] Add a benchmark image comparing two profanity2 revisions Builds both revisions from a fresh clone into one image and runs them alternately on the same GPU, which is the only way an 8-12% difference can be told apart from the spread between two rented machines. Kept separate from the shipping image in the repository root: it has its own directory, its own build context and its own entrypoint. Co-authored-by: Gleb Alekseev --- bench/Dockerfile | 113 ++++++++++++ bench/README.md | 168 ++++++++++++++++++ bench/build.sh | 98 +++++++++++ bench/publish.sh | 88 ++++++++++ bench/run-benchmark.sh | 385 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 852 insertions(+) create mode 100644 bench/Dockerfile create mode 100644 bench/README.md create mode 100755 bench/build.sh create mode 100755 bench/publish.sh create mode 100755 bench/run-benchmark.sh diff --git a/bench/Dockerfile b/bench/Dockerfile new file mode 100644 index 0000000..8b9692d --- /dev/null +++ b/bench/Dockerfile @@ -0,0 +1,113 @@ +# syntax=docker/dockerfile:1 +# +# Builds a self-contained image holding two revisions of profanity2 so their +# speed can be compared back to back on the same GPU. This image is a +# measuring tool, not the shipping image - see the Dockerfile in the +# repository root for that one. +# +# docker build --build-arg REF_A=master --build-arg REF_B=pr/57 -t bench bench/ +# +# REF_A and REF_B accept anything the clone can resolve: a branch, a tag, a +# full or short SHA, or pr/ for the head of a pull request. + +# --------------------------------------------------------------------------- +# Export both revisions +# --------------------------------------------------------------------------- +FROM ubuntu:24.04 AS src + +ARG REPO=https://github.com/1inch/profanity2 +ARG REF_A +ARG REF_B +# Changed by bench/build.sh whenever a revision is mutable (a branch), so that +# a moving branch is not silently served from the layer cache. +ARG CACHEBUST= + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Pull request heads live outside refs/heads and have to be fetched +# explicitly, otherwise a commit that only exists in a pull request cannot be +# resolved. The last step records whether the two revisions time a round the +# same way: if they do not, the speeds they print come from different clocks +# and the runner has to warn about it. +RUN set -eu; \ + echo "cachebust: ${CACHEBUST}"; \ + test -n "${REF_A}" || { echo "error: REF_A build argument is empty" >&2; exit 1; }; \ + test -n "${REF_B}" || { echo "error: REF_B build argument is empty" >&2; exit 1; }; \ + git clone --quiet --no-checkout "${REPO}" /repo; \ + git -C /repo fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' || \ + echo "warning: could not fetch pull request refs from ${REPO}" >&2; \ + for slot_ref in "a:${REF_A}" "b:${REF_B}"; do \ + slot="${slot_ref%%:*}"; \ + ref="${slot_ref#*:}"; \ + sha="$(git -C /repo rev-parse --verify --quiet "${ref}^{commit}" \ + || git -C /repo rev-parse --verify --quiet "origin/${ref}^{commit}" \ + || true)"; \ + if [ -z "${sha}" ]; then \ + echo "error: cannot resolve revision '${ref}' in ${REPO}" >&2; \ + exit 1; \ + fi; \ + mkdir -p "/src/${slot}"; \ + git -C /repo archive --format=tar "${sha}" | tar -x -C "/src/${slot}"; \ + printf '%s' "${ref}" > "/src/${slot}.ref"; \ + git -C /repo rev-parse --short=7 "${sha}" > "/src/${slot}.sha"; \ + echo "${slot}: ${ref} -> ${sha}"; \ + done; \ + if cmp -s /src/a/SpeedSample.cpp /src/b/SpeedSample.cpp; then \ + echo same > /src/timer.state; \ + else \ + echo differs > /src/timer.state; \ + fi + +# --------------------------------------------------------------------------- +# Compile both revisions +# --------------------------------------------------------------------------- +FROM ubuntu:24.04 AS build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + opencl-headers \ + ocl-icd-opencl-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=src /src /src +RUN make -C /src/a -j"$(nproc)" && make -C /src/b -j"$(nproc)" + +# --------------------------------------------------------------------------- +# Runtime +# --------------------------------------------------------------------------- +FROM ubuntu:24.04 + +LABEL org.opencontainers.image.title="profanity2-bench" \ + org.opencontainers.image.description="Compares the speed of two profanity2 revisions on one GPU" \ + org.opencontainers.image.source="https://github.com/1inch/profanity2" \ + org.opencontainers.image.licenses="MIT" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ocl-icd-libopencl1 \ + clinfo \ + && rm -rf /var/lib/apt/lists/* + +# The NVIDIA container runtime mounts libnvidia-opencl.so.1 but does not +# register it with the ICD loader: +# https://github.com/NVIDIA/nvidia-container-toolkit/issues/682 +RUN mkdir -p /etc/OpenCL/vendors \ + && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +ENV NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility + +# Each revision keeps its own directory: profanity2 loads keccak.cl and +# profanity.cl from the working directory and caches the compiled kernel +# there, so the two builds must never share one. +COPY --from=build /src/a/profanity2.x64 /src/a/keccak.cl /src/a/profanity.cl /opt/bench/a/ +COPY --from=build /src/b/profanity2.x64 /src/b/keccak.cl /src/b/profanity.cl /opt/bench/b/ +COPY --from=src /src/a.ref /src/a.sha /src/b.ref /src/b.sha /src/timer.state /opt/bench/ + +COPY run-benchmark.sh /usr/local/bin/bench +RUN chmod +x /usr/local/bin/bench + +WORKDIR /opt/bench +ENTRYPOINT ["/usr/local/bin/bench"] diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..6b75a05 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,168 @@ +# Comparing the speed of two profanity2 revisions + +This directory builds a **separate image** from the one in the repository root. +The image in the root ships profanity2; this one measures it. It contains two +revisions of profanity2 side by side and runs them alternately on the same GPU, +so a change can be judged without trusting that two rented machines are +equally fast. + +```bash +bench/publish.sh master pr/57 +``` + +builds both revisions, pushes the image to [ttl.sh](https://ttl.sh) and prints +its name: + +``` +IMAGE: ttl.sh/profanity2-bench-1f0c9a3e-...:24h + + vastai create instance --image ttl.sh/profanity2-bench-...:24h --disk 12 --args --mode leading +``` + +Rent any NVIDIA GPU with that image and the instance log ends with: + +``` +=================== BENCHMARK RESULT =================== +A master (069d100) median 1390.4 MH/s spread 0.3% + runs: 1392.6 MH/s 1388.1 MH/s +B pr/57 (aad8e13) median 1212.7 MH/s spread 0.4% + runs: 1210.3 MH/s 1215.0 MH/s + +B vs A: -12.8% +======================================================== +BENCH_RESULT mode=leading a_hs=1390400000 b_hs=1212700000 delta_pct=-12.8 +``` + +## What can be passed as a revision + +Anything a clone of the repository can resolve: + +| Revision | Meaning | +|---|---| +| `master`, `v1.2` | branch or tag | +| `9011bcd`, full SHA | any commit, including one that only exists in a pull request | +| `pr/57` | the head of pull request 57 | + +Pull request heads are fetched explicitly during the build, which is why a +commit like `9011bcd` resolves even though it never landed on a branch. + +Both revisions are compiled inside the image from a fresh clone, so your +working tree, your local branches and your uncommitted changes play no part. +Point the build at a fork with `--repo`: + +```bash +bench/build.sh master pr/57 --repo https://github.com/YOUR_USER/profanity2 +``` + +## Running it + +```bash +# build only, prints the local tag on the last line +bench/build.sh master pr/57 + +# build, push to ttl.sh for 24 hours, print the image name +bench/publish.sh master pr/57 + +# push an image that is already built +bench/publish.sh --image profanity2-bench:master__pr-57 --ttl 1h + +# on your own machine, if it has an NVIDIA GPU +docker run --rm --gpus all profanity2-bench:master__pr-57 +``` + +The image is built for `linux/amd64` by default because the Linux branch of the +Makefile passes `-mmmx` and `-mcmodel=large`, which do not exist on arm64, and +because GPU rental platforms are x86_64 anyway. On Apple Silicon the build +therefore runs under emulation and takes a few minutes. + +## Options + +Everything has a flag and an environment variable; flags are easier on +platforms that pass arguments to the entrypoint, variables are the only option +on platforms that replace it. + +| Flag | Variable | Default | Meaning | +|---|---|---|---| +| `--mode` | `BENCH_MODE` | `leading` | `leading` reads the program's speed counter, `exact` counts matching addresses | +| `--seconds` | `BENCH_SECONDS` | `120` | length of the measured window per run | +| `--warmup` | `BENCH_WARMUP` | `30` | seconds dropped after the run starts reporting speed | +| `--repeats` | `BENCH_REPEATS` | `2` | how many times each revision runs | +| `--extra-args` | `BENCH_EXTRA_ARGS` | `-i 255 -I 16384 -w 64` | options given to both revisions | +| `--mask` | `BENCH_EXACT_MASK` | `deadbee` | mask for `--mode exact`, 4 to 10 fixed hex characters | +| `--public-key` | `PUBLIC_KEY` | secp256k1 generator | seed public key | +| | `BENCH_SKIP_GPU_CHECK` | unset | start even when no OpenCL platform is detected | + +With the defaults a full comparison takes about ten minutes plus kernel +compilation, which is around six cents on an RTX 4090. + +An argument that does not start with a dash is executed instead of the +benchmark, which is the quickest way to inspect a rented machine: + +```bash +docker run --rm --gpus all profanity2-bench:master__pr-57 clinfo +``` + +The default seed public key is the generator point of secp256k1. It is a valid +public key whose private key is the publicly known value 1, which is fine for a +benchmark and useless for anything else - never treat a key found during a +benchmark run as yours. + +## Getting a number that means something + +**The two revisions take turns.** The order is A, B, A, B, never A, A, B, B: a +GPU that heats up or gets throttled halfway through would otherwise hand the +whole penalty to whichever revision ran last. The reported `spread` is how far +apart the repeats of one revision landed. If it is as large as the difference +between the revisions, the machine is too noisy and the result means nothing - +the runner says so explicitly. + +**Each revision has its own directory and its own kernel cache.** profanity2 +compiles its OpenCL kernel on first use and caches it next to the binary; two +revisions sharing a directory would run each other's compiled kernel. + +**The first seconds of every run are discarded.** The window only opens once +the program starts reporting speed, so kernel compilation and device +initialization never land inside it, no matter how slow the device is. + +**`--benchmark` is deliberately not supported.** In a revision where the +scoring kernel is fused into the iterate kernel, a scoring function that does +nothing lets the compiler delete the keccak as well, and the resulting number +is meaningless. That is why pull request 57 carries a `benchmark` scoring +function whose comment reads *"Prevent the compiler from deleting the keccak +behind profanity_iterate"*. Real scoring modes do not have this problem. + +**Watch out for revisions that changed the speed counter itself.** Until +commit `9011bcd` the duration of a round was truncated to whole milliseconds: + +```c++ +auto delta = std::chrono::duration_cast(newTime - m_lastTime).count(); +m_lSpeeds.push_back((1000 * V) / delta); +``` + +A round with default settings is `255 * 16384 = 4177920` keys, which on a +4090 takes about 3.8 ms and gets counted as 3 ms - the printed speed is then +27% above the truth, and on a faster card a sub-millisecond round divides by +zero. Comparing such a build against one that measures in microseconds +measures the fix, not the kernels. + +The build detects this: if `SpeedSample.cpp` differs between the two +revisions, the result block ends with a warning. Then either compare against +a baseline carrying the same timer, + +```bash +bench/publish.sh 9011bcd pr/57 +``` + +or measure without the program's clock at all: + +```bash +docker run --rm --gpus all IMAGE --mode exact +``` + +`--mode exact` uses `--exact `, which prints every address matching the +mask, and derives the throughput from how many appear per second: +`rate = matches / seconds * 16^fixed`. Nothing in that number comes from the +program's own timer. It is a counting measurement, so its precision is +`1/sqrt(matches)`: the default 7-character mask on a 1 GH/s card yields about +four matches per second, so a 120 second window gives roughly 450 matches and +5% precision. Widen `--seconds` to resolve smaller differences. diff --git a/bench/build.sh b/bench/build.sh new file mode 100755 index 0000000..c31de85 --- /dev/null +++ b/bench/build.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# Builds a benchmark image containing two revisions of profanity2. +# +# usage: bench/build.sh REF_A REF_B [OPTIONS] +# +# REF_A, REF_B anything the repository can resolve: a branch, a +# tag, a SHA, or pr/ for a pull request head +# --repo repository to build from +# [default = https://github.com/1inch/profanity2] +# --platform

target platform [default = linux/amd64] +# --tag local image tag [default = derived from the SHAs] +# -h, --help this text +# +# The image name is printed on the last line so the script can be used from +# other scripts: +# +# IMAGE=$(bench/build.sh master pr/57 | tail -1) + +set -euo pipefail + +REPO="${BENCH_REPO:-https://github.com/1inch/profanity2}" +# vast.ai and every other GPU rental platform worth using is x86_64, and the +# Linux branch of the Makefile passes -mmmx and -mcmodel=large, which do not +# exist on arm64. Building for amd64 on an Apple Silicon machine goes through +# emulation and takes a few minutes. +PLATFORM="${BENCH_PLATFORM:-linux/amd64}" +TAG="" +REF_A="" +REF_B="" + +usage() { + sed -n '/^# usage:/,/^$/p' "${BASH_SOURCE[0]}" | sed 's/^#\{0,1\} \{0,1\}//' +} + +die() { + printf 'build.sh: error: %s\n' "$*" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) REPO="${2:?--repo needs a value}"; shift 2 ;; + --platform) PLATFORM="${2:?--platform needs a value}"; shift 2 ;; + --tag) TAG="${2:?--tag needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + -*) die "unknown option '$1', try --help" ;; + *) + if [ -z "$REF_A" ]; then + REF_A="$1" + elif [ -z "$REF_B" ]; then + REF_B="$1" + else + die "expected exactly two revisions, got a third: '$1'" + fi + shift + ;; + esac +done + +[ -n "$REF_A" ] && [ -n "$REF_B" ] || die "two revisions are required, try --help" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +sanitize() { + printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-' +} + +if [ -z "$TAG" ]; then + TAG="profanity2-bench:$(sanitize "$REF_A")__$(sanitize "$REF_B")" +fi + +# A branch moves, a SHA does not. Without this the layer cache would happily +# rebuild yesterday's master and report it as today's. +cachebust="" +for ref in "$REF_A" "$REF_B"; do + if ! printf '%s' "$ref" | grep -qE '^[0-9a-f]{40}$'; then + cachebust="$(date +%s)" + break + fi +done + +printf 'build.sh: building %s\n' "$TAG" >&2 +printf 'build.sh: A = %s\n' "$REF_A" >&2 +printf 'build.sh: B = %s\n' "$REF_B" >&2 +printf 'build.sh: repository = %s, platform = %s\n' "$REPO" "$PLATFORM" >&2 + +docker build \ + --platform "$PLATFORM" \ + --build-arg "REPO=$REPO" \ + --build-arg "REF_A=$REF_A" \ + --build-arg "REF_B=$REF_B" \ + --build-arg "CACHEBUST=$cachebust" \ + --tag "$TAG" \ + --file "$script_dir/Dockerfile" \ + "$script_dir" >&2 + +printf '%s\n' "$TAG" diff --git a/bench/publish.sh b/bench/publish.sh new file mode 100755 index 0000000..483b5b4 --- /dev/null +++ b/bench/publish.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Publishes a benchmark image to ttl.sh, an anonymous registry that deletes +# what you push after the time given in the tag, and prints its name. +# +# usage: bench/publish.sh REF_A REF_B [OPTIONS] +# bench/publish.sh --image LOCAL_TAG [OPTIONS] +# +# --ttl <5m|1h|24h> how long the image stays on ttl.sh [default = 24h] +# --registry registry to push to [default = ttl.sh] +# --image publish an already built local image +# --repo passed to build.sh +# --platform

passed to build.sh +# -h, --help this text +# +# On ttl.sh the tag is the lifetime, so the image name has to be unique - the +# script generates a random one. Anyone who learns that name can pull the +# image, which is harmless here: it holds nothing but public source code. + +set -euo pipefail + +TTL="24h" +REGISTRY="${BENCH_REGISTRY:-ttl.sh}" +LOCAL_TAG="" +REFS=() +BUILD_ARGS=() + +usage() { + sed -n '/^# usage:/,/^$/p' "${BASH_SOURCE[0]}" | sed 's/^#\{0,1\} \{0,1\}//' +} + +die() { + printf 'publish.sh: error: %s\n' "$*" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --ttl) TTL="${2:?--ttl needs a value}"; shift 2 ;; + --registry) REGISTRY="${2:?--registry needs a value}"; shift 2 ;; + --image) LOCAL_TAG="${2:?--image needs a value}"; shift 2 ;; + --repo) BUILD_ARGS+=(--repo "${2:?--repo needs a value}"); shift 2 ;; + --platform) BUILD_ARGS+=(--platform "${2:?--platform needs a value}"); shift 2 ;; + -h|--help) usage; exit 0 ;; + -*) die "unknown option '$1', try --help" ;; + *) REFS+=("$1"); shift ;; + esac +done + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ -z "$LOCAL_TAG" ]; then + [ "${#REFS[@]}" -eq 2 ] || die "two revisions are required, try --help" + LOCAL_TAG="$("$script_dir/build.sh" "${REFS[@]}" ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} | tail -1)" + [ -n "$LOCAL_TAG" ] || die "the build produced no image" +fi + +random_name() { + if command -v uuidgen >/dev/null 2>&1; then + uuidgen | tr 'A-Z' 'a-z' + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import uuid; print(uuid.uuid4())' + else + od -An -tx1 -N16 /dev/urandom | tr -d ' \n' + fi +} + +remote="$REGISTRY/profanity2-bench-$(random_name):$TTL" + +printf 'publish.sh: pushing %s as %s\n' "$LOCAL_TAG" "$remote" >&2 +docker tag "$LOCAL_TAG" "$remote" +docker push "$remote" >&2 + +cat < --image $remote --disk 12 --args --mode leading + +Locally, on a machine with an NVIDIA GPU: + + docker run --rm --gpus all $remote + +EOF + +printf '%s\n' "$remote" diff --git a/bench/run-benchmark.sh b/bench/run-benchmark.sh new file mode 100755 index 0000000..3f90384 --- /dev/null +++ b/bench/run-benchmark.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bash +# +# Compares the speed of the two profanity2 revisions baked into this image. +# +# The two builds never run at the same time and never share a kernel cache. +# They take turns - A, B, A, B - so that clock drift or thermal throttling on +# the rented machine shows up as disagreement between repeats of the same +# build instead of masquerading as a difference between the two revisions. +# +# usage: bench [OPTIONS] +# +# --mode leading|exact what to measure [default = leading] +# --seconds measured window per run, in seconds [default = 120] +# --warmup seconds dropped before the window opens [default = 30] +# --repeats how many times each revision runs [default = 2] +# --extra-args "<...>" options handed to both revisions +# [default = "-i 255 -I 16384 -w 64"] +# --mask mask used by --mode exact [default = deadbee] +# --public-key seed public key [default = the secp256k1 generator] +# -h, --help this text +# +# Every option can also be given as an environment variable (BENCH_MODE, +# BENCH_SECONDS, BENCH_WARMUP, BENCH_REPEATS, BENCH_EXTRA_ARGS, +# BENCH_EXACT_MASK, PUBLIC_KEY), which is the only way to configure the run on +# platforms that replace the image entrypoint. +# +# An argument that does not start with a dash is executed instead of the +# benchmark, e.g. `clinfo` or `bash`. + +set -euo pipefail + +readonly ROOT=/opt/bench + +# Generator point of secp256k1, x and y concatenated without the 04 prefix. It +# is a valid public key, so profanity2 accepts it, and its private key is the +# publicly known value 1 - fine for a benchmark, useless for anything else. +readonly GENERATOR_PUBLIC_KEY=79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 + +# How long a revision may take to compile its kernel and initialize the device +# before the run is considered failed. +readonly STARTUP_TIMEOUT=900 + +MODE="${BENCH_MODE:-leading}" +WINDOW="${BENCH_SECONDS:-120}" +WARMUP="${BENCH_WARMUP:-30}" +REPEATS="${BENCH_REPEATS:-2}" +EXTRA_ARGS="${BENCH_EXTRA_ARGS:--i 255 -I 16384 -w 64}" +EXACT_MASK="${BENCH_EXACT_MASK:-deadbee}" +PUBLIC_KEY="${PUBLIC_KEY:-$GENERATOR_PUBLIC_KEY}" + +log() { + printf 'bench: %s\n' "$*" >&2 +} + +die() { + log "error: $*" + exit 1 +} + +usage() { + cat <<'EOF' +usage: bench [OPTIONS] + +Runs the two profanity2 revisions baked into this image one after the other, +alternating A B A B, and reports how their speeds compare. + + --mode leading|exact what to measure [default = leading] + --seconds measured window per run, in seconds [default = 120] + --warmup seconds dropped before the window opens [default = 30] + --repeats how many times each revision runs [default = 2] + --extra-args "<...>" options handed to both revisions + [default = "-i 255 -I 16384 -w 64"] + --mask mask used by --mode exact [default = deadbee] + --public-key seed public key [default = the secp256k1 generator] + -h, --help this text + +The same settings can be given as environment variables (BENCH_MODE, +BENCH_SECONDS, BENCH_WARMUP, BENCH_REPEATS, BENCH_EXTRA_ARGS, +BENCH_EXACT_MASK, PUBLIC_KEY), which is the only way to configure the run on +platforms that replace the image entrypoint. + +An argument that does not start with a dash is executed instead of the +benchmark, e.g. `clinfo` or `bash`. +EOF +} + +cleanup() { + local pids + pids="$(jobs -p)" + if [ -n "$pids" ]; then + # shellcheck disable=SC2086 + kill $pids 2>/dev/null || true + fi +} + +label() { + local slot="$1" + printf '%s (%s)' "$(cat "$ROOT/$slot.ref")" "$(cat "$ROOT/$slot.sha")" +} + +count_opencl_platforms() { + local count= + + if command -v clinfo >/dev/null 2>&1; then + count="$(clinfo -l 2>/dev/null | grep -c '^Platform #' || true)" + + case "$count" in + ''|0|*[!0-9]*) + count="$(clinfo 2>/dev/null | awk '/^Number of platforms/ { print $NF; exit }' || true)" + ;; + esac + fi + + case "$count" in + ''|*[!0-9]*) count=0 ;; + esac + + printf '%s\n' "$count" +} + +# Reads speed samples on stdin, prints their median in hashes per second. +median() { + sort -n | awk ' + { v[NR] = $1 } + END { + if (NR == 0) { print 0; exit } + if (NR % 2) { print v[(NR + 1) / 2] } + else { print (v[NR / 2] + v[NR / 2 + 1]) / 2 } + }' +} + +# The speed counter is printed to stderr, overwriting itself with a carriage +# return roughly once per round, and formatted by Dispatcher::formatSpeed as a +# number followed by a unit out of " KMGT". Slice off everything written +# before the window opened, then normalize to hashes per second. +speeds_in_window() { + local file="$1" offset="$2" + + tail -c "+$((offset + 1))" "$file" \ + | tr '\r' '\n' \ + | grep -oE 'Total: *[0-9.]+ *[KMGT]?H/s' \ + | awk ' + { + match($0, /[0-9.]+/) + v = substr($0, RSTART, RLENGTH) + 0 + if (index($0, "KH/s")) { v *= 1e3 } + else if (index($0, "MH/s")) { v *= 1e6 } + else if (index($0, "GH/s")) { v *= 1e9 } + else if (index($0, "TH/s")) { v *= 1e12 } + if (v > 0) { print v } + }' +} + +# Every hash matching the mask is printed, so the number of matches per second +# is a throughput measurement that does not go through the program's own +# timer: rate = matches / seconds * 16^(fixed nibbles). +matches_in_window() { + local file="$1" offset="$2" nibbles="$3" seconds="$4" + local count + + count="$(tail -c "+$((offset + 1))" "$file" | grep -c 'Private: 0x' || true)" + awk -v c="$count" -v k="$nibbles" -v t="$seconds" \ + 'BEGIN { printf "%.0f\n", c * (16 ^ k) / t }' +} + +format_speed() { + awk -v v="$1" 'BEGIN { printf "%.1f MH/s\n", v / 1e6 }' +} + +wait_for_first_sample() { + local file="$1" pid="$2" waited=0 + + while [ "$waited" -lt "$STARTUP_TIMEOUT" ]; do + if grep -qs 'Total:' "$file"; then + return 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done + + return 1 +} + +# Runs one revision once and prints the measured speed in hashes per second. +run_once() { + local slot="$1" + local dir="$ROOT/$slot" + local out="/tmp/bench.$slot.out" + local err="/tmp/bench.$slot.err" + local args=() + + if [ "$MODE" = exact ]; then + args=(--exact "$EXACT_MASK") + else + args=(--leading 0) + fi + + : >"$out" + : >"$err" + + # profanity2 reads keccak.cl and profanity.cl from the working directory + # and caches the compiled kernel there, hence the cd into the revision's + # own directory. + # shellcheck disable=SC2086 + ( cd "$dir" && exec ./profanity2.x64 "${args[@]}" -z "$PUBLIC_KEY" $EXTRA_ARGS ) \ + >"$out" 2>"$err" & + local pid=$! + + if ! wait_for_first_sample "$err" "$pid"; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + log "the run produced no speed samples, last output was:" + tail -n 10 "$out" "$err" >&2 || true + return 1 + fi + + sleep "$WARMUP" + local out_offset err_offset + out_offset="$(wc -c <"$out")" + err_offset="$(wc -c <"$err")" + + sleep "$WINDOW" + + if ! kill -0 "$pid" 2>/dev/null; then + log "the run stopped before the window closed, last output was:" + tail -n 10 "$out" "$err" >&2 || true + return 1 + fi + + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + + if [ "$MODE" = exact ]; then + matches_in_window "$out" "$out_offset" "$FIXED_NIBBLES" "$WINDOW" + else + speeds_in_window "$err" "$err_offset" | median + fi +} + +main() { + trap cleanup EXIT + + case "$MODE" in + leading|exact) ;; + *) die "unknown mode '$MODE', expected 'leading' or 'exact'" ;; + esac + + for value in "$WINDOW" "$WARMUP" "$REPEATS"; do + case "$value" in + ''|*[!0-9]*) die "--seconds, --warmup and --repeats must be whole numbers" ;; + esac + done + [ "$REPEATS" -ge 1 ] || die "--repeats must be at least 1" + + FIXED_NIBBLES="$(printf '%s' "$EXACT_MASK" | grep -o '[0-9a-fA-F]' | wc -l | tr -d ' ')" + if [ "$MODE" = exact ] && { [ "$FIXED_NIBBLES" -lt 4 ] || [ "$FIXED_NIBBLES" -gt 10 ]; }; then + die "--mask needs between 4 and 10 fixed hex characters, got $FIXED_NIBBLES" + fi + + local platforms + platforms="$(count_opencl_platforms)" + if [ -n "${BENCH_SKIP_GPU_CHECK:-}" ]; then + platforms=1 + fi + if [ "$platforms" -eq 0 ]; then + log "no OpenCL platform found inside the container" + log " installed ICDs: $(echo /etc/OpenCL/vendors/*.icd)" + log " start the container with GPU access (docker run --gpus all ...)" + log " or run this image with the argument \"clinfo\" to see what it finds" + exit 1 + fi + + local describe="--leading 0" + [ "$MODE" = exact ] && describe="--exact $EXACT_MASK" + + echo "profanity2 benchmark" + echo " A: $(label a)" + echo " B: $(label b)" + echo " workload: $describe $EXTRA_ARGS" + echo " window: ${REPEATS} x ${WINDOW}s per revision, first ${WARMUP}s of each run dropped" + echo " order: $(for _ in $(seq "$REPEATS"); do printf 'A B '; done)" + if [ "$PUBLIC_KEY" = "$GENERATOR_PUBLIC_KEY" ]; then + echo " key: secp256k1 generator - PUBLIC, never use a result from this run" + fi + echo + + local -A speeds=([a]="" [b]="") + local repeat slot speed + + for repeat in $(seq "$REPEATS"); do + for slot in a b; do + log "run $repeat/$REPEATS, revision ${slot^^} ($(label "$slot"))" + if ! speed="$(run_once "$slot")"; then + die "revision ${slot^^} failed to produce a measurement" + fi + log "run $repeat/$REPEATS, revision ${slot^^}: $(format_speed "$speed")" + speeds[$slot]="${speeds[$slot]} $speed" + done + done + + report "${speeds[a]}" "${speeds[b]}" +} + +spread_pct() { + # shellcheck disable=SC2086 + printf '%s\n' $1 | sort -n | awk ' + { v[NR] = $1 } + END { + if (NR < 2 || v[1] == 0) { print "0.0"; exit } + printf "%.1f", (v[NR] - v[1]) / v[1] * 100 + }' +} + +report() { + local a_runs="$1" b_runs="$2" + local a_median b_median delta a_spread b_spread noise + + # shellcheck disable=SC2086 + a_median="$(printf '%s\n' $a_runs | median)" + # shellcheck disable=SC2086 + b_median="$(printf '%s\n' $b_runs | median)" + a_spread="$(spread_pct "$a_runs")" + b_spread="$(spread_pct "$b_runs")" + delta="$(awk -v a="$a_median" -v b="$b_median" \ + 'BEGIN { if (a == 0) { print "n/a" } else { printf "%+.1f", (b / a - 1) * 100 } }')" + noise="$(awk -v x="$a_spread" -v y="$b_spread" 'BEGIN { print (x > y) ? x : y }')" + + echo + echo "=================== BENCHMARK RESULT ===================" + printf 'A %-24s median %-12s spread %s%%\n' "$(label a)" "$(format_speed "$a_median")" "$a_spread" + printf ' runs:%s\n' "$(fmt_runs "$a_runs")" + printf 'B %-24s median %-12s spread %s%%\n' "$(label b)" "$(format_speed "$b_median")" "$b_spread" + printf ' runs:%s\n' "$(fmt_runs "$b_runs")" + echo + echo "B vs A: ${delta}%" + + if awk -v n="$noise" -v d="$delta" 'BEGIN { exit !(d != "n/a" && n >= (d < 0 ? -d : d)) }' 2>/dev/null; then + echo + echo "warning: repeats of the same revision disagree by ${noise}%, which is at" + echo " least as large as the difference between the revisions - this" + echo " machine is too noisy, use a longer --seconds or another host" + fi + + if [ "$MODE" = leading ] && [ "$(cat "$ROOT/timer.state")" = differs ]; then + echo + echo "warning: SpeedSample.cpp differs between the two revisions, so the two" + echo " numbers above were produced by different timers and are not" + echo " directly comparable. Rerun with --mode exact, which counts" + echo " matching addresses instead of trusting the program's clock." + fi + + echo "========================================================" + echo "BENCH_RESULT mode=$MODE a_hs=$a_median b_hs=$b_median delta_pct=$delta" +} + +fmt_runs() { + local run + # shellcheck disable=SC2086 + for run in $1; do + printf ' %s' "$(format_speed "$run")" + done +} + +if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then + exec "$@" +fi + +while [ "$#" -gt 0 ]; do + case "$1" in + --mode) MODE="${2:?--mode needs a value}"; shift 2 ;; + --seconds) WINDOW="${2:?--seconds needs a value}"; shift 2 ;; + --warmup) WARMUP="${2:?--warmup needs a value}"; shift 2 ;; + --repeats) REPEATS="${2:?--repeats needs a value}"; shift 2 ;; + --extra-args) EXTRA_ARGS="${2:?--extra-args needs a value}"; shift 2 ;; + --mask) EXACT_MASK="${2:?--mask needs a value}"; shift 2 ;; + --public-key) PUBLIC_KEY="${2:?--public-key needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option '$1', try --help" ;; + esac +done + +main From 9f6816895cc493f33969373881683b4684a05ca5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 18:04:42 +0000 Subject: [PATCH 2/6] Sharpen the noise warning and add BENCH_SKIP_GPU_CHECK The warning fired when both revisions measured identically with no spread, calling a quiet machine noisy. It now only fires when the repeats actually disagree, and says what that means for the comparison. Co-authored-by: Gleb Alekseev --- bench/build.sh | 4 +++- bench/publish.sh | 2 +- bench/run-benchmark.sh | 9 +++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bench/build.sh b/bench/build.sh index c31de85..7d961e0 100755 --- a/bench/build.sh +++ b/bench/build.sh @@ -58,7 +58,9 @@ while [ "$#" -gt 0 ]; do esac done -[ -n "$REF_A" ] && [ -n "$REF_B" ] || die "two revisions are required, try --help" +if [ -z "$REF_A" ] || [ -z "$REF_B" ]; then + die "two revisions are required, try --help" +fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/bench/publish.sh b/bench/publish.sh index 483b5b4..cf3deab 100755 --- a/bench/publish.sh +++ b/bench/publish.sh @@ -57,7 +57,7 @@ fi random_name() { if command -v uuidgen >/dev/null 2>&1; then - uuidgen | tr 'A-Z' 'a-z' + uuidgen | tr '[:upper:]' '[:lower:]' elif command -v python3 >/dev/null 2>&1; then python3 -c 'import uuid; print(uuid.uuid4())' else diff --git a/bench/run-benchmark.sh b/bench/run-benchmark.sh index 3f90384..96b4fce 100755 --- a/bench/run-benchmark.sh +++ b/bench/run-benchmark.sh @@ -337,11 +337,12 @@ report() { echo echo "B vs A: ${delta}%" - if awk -v n="$noise" -v d="$delta" 'BEGIN { exit !(d != "n/a" && n >= (d < 0 ? -d : d)) }' 2>/dev/null; then + if awk -v n="$noise" -v d="$delta" \ + 'BEGIN { exit !(d != "n/a" && n > 0 && n >= (d < 0 ? -d : d)) }'; then echo - echo "warning: repeats of the same revision disagree by ${noise}%, which is at" - echo " least as large as the difference between the revisions - this" - echo " machine is too noisy, use a longer --seconds or another host" + echo "warning: the difference is no larger than the spread between repeats of" + echo " the same revision (${noise}%), so it cannot be told apart from" + echo " noise - use a longer --seconds, more --repeats or another host" fi if [ "$MODE" = leading ] && [ "$(cat "$ROOT/timer.state")" = differs ]; then From ecce9c0daf73d34d5e01c597a09beb359e89d3ad Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 22:48:51 +0000 Subject: [PATCH 3/6] Do not take the seed key from an untrusted PUBLIC_KEY The benchmark picked up whatever PUBLIC_KEY held in the environment, so on a platform that sets that name for its own SSH key every run died with "public key must be 128 hexademical characters long" and the container was restarted in a loop. BENCH_PUBLIC_KEY is now the name to use, a plain PUBLIC_KEY is only honoured when it holds 128 hexadecimal characters, and the header always reports which key the run used. Co-authored-by: Gleb Alekseev --- bench/README.md | 12 +++++++++++- bench/run-benchmark.sh | 44 +++++++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/bench/README.md b/bench/README.md index 6b75a05..6073b8a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -89,9 +89,19 @@ on platforms that replace it. | `--repeats` | `BENCH_REPEATS` | `2` | how many times each revision runs | | `--extra-args` | `BENCH_EXTRA_ARGS` | `-i 255 -I 16384 -w 64` | options given to both revisions | | `--mask` | `BENCH_EXACT_MASK` | `deadbee` | mask for `--mode exact`, 4 to 10 fixed hex characters | -| `--public-key` | `PUBLIC_KEY` | secp256k1 generator | seed public key | +| `--public-key` | `BENCH_PUBLIC_KEY` | secp256k1 generator | seed public key | | | `BENCH_SKIP_GPU_CHECK` | unset | start even when no OpenCL platform is detected | +A plain `PUBLIC_KEY` is honoured as well, but only when it holds 128 +hexadecimal characters. Rental platforms hand out that generic name for their +own SSH key, and it may already sit in your account-wide environment +variables, so a value that is not a seed public key is ignored and the run +falls back to the default. The header of every run says which key it used: + +``` + key: the secp256k1 generator (PUBLIC_KEY ignored, not 128 hex characters) +``` + With the defaults a full comparison takes about ten minutes plus kernel compilation, which is around six cents on an RTX 4090. diff --git a/bench/run-benchmark.sh b/bench/run-benchmark.sh index 96b4fce..1168ff1 100755 --- a/bench/run-benchmark.sh +++ b/bench/run-benchmark.sh @@ -21,8 +21,8 @@ # # Every option can also be given as an environment variable (BENCH_MODE, # BENCH_SECONDS, BENCH_WARMUP, BENCH_REPEATS, BENCH_EXTRA_ARGS, -# BENCH_EXACT_MASK, PUBLIC_KEY), which is the only way to configure the run on -# platforms that replace the image entrypoint. +# BENCH_EXACT_MASK, BENCH_PUBLIC_KEY), which is the only way to configure the +# run on platforms that replace the image entrypoint. # # An argument that does not start with a dash is executed instead of the # benchmark, e.g. `clinfo` or `bash`. @@ -46,7 +46,26 @@ WARMUP="${BENCH_WARMUP:-30}" REPEATS="${BENCH_REPEATS:-2}" EXTRA_ARGS="${BENCH_EXTRA_ARGS:--i 255 -I 16384 -w 64}" EXACT_MASK="${BENCH_EXACT_MASK:-deadbee}" -PUBLIC_KEY="${PUBLIC_KEY:-$GENERATOR_PUBLIC_KEY}" + +# PUBLIC_KEY is accepted for convenience but cannot be trusted: rental +# platforms hand out that name for their own SSH key and it may sit in an +# account-wide variable, so it is only taken when it looks like a seed public +# key. Which key won is printed in the header - a benchmark that picks up +# somebody else's value fails in a way that is hard to read otherwise. +SEED_KEY="$GENERATOR_PUBLIC_KEY" +KEY_SOURCE="the secp256k1 generator" + +if [ -n "${BENCH_PUBLIC_KEY:-}" ]; then + SEED_KEY="$BENCH_PUBLIC_KEY" + KEY_SOURCE="BENCH_PUBLIC_KEY" +elif [ -n "${PUBLIC_KEY:-}" ]; then + if printf '%s' "$PUBLIC_KEY" | grep -qE '^[0-9a-fA-F]{128}$'; then + SEED_KEY="$PUBLIC_KEY" + KEY_SOURCE="PUBLIC_KEY" + else + KEY_SOURCE="the secp256k1 generator (PUBLIC_KEY ignored, not 128 hex characters)" + fi +fi log() { printf 'bench: %s\n' "$*" >&2 @@ -76,8 +95,10 @@ alternating A B A B, and reports how their speeds compare. The same settings can be given as environment variables (BENCH_MODE, BENCH_SECONDS, BENCH_WARMUP, BENCH_REPEATS, BENCH_EXTRA_ARGS, -BENCH_EXACT_MASK, PUBLIC_KEY), which is the only way to configure the run on -platforms that replace the image entrypoint. +BENCH_EXACT_MASK, BENCH_PUBLIC_KEY), which is the only way to configure the +run on platforms that replace the image entrypoint. A plain PUBLIC_KEY is used +too, but only when it holds 128 hexadecimal characters, because rental +platforms hand out that name for their own SSH key. An argument that does not start with a dash is executed instead of the benchmark, e.g. `clinfo` or `bash`. @@ -205,7 +226,7 @@ run_once() { # and caches the compiled kernel there, hence the cd into the revision's # own directory. # shellcheck disable=SC2086 - ( cd "$dir" && exec ./profanity2.x64 "${args[@]}" -z "$PUBLIC_KEY" $EXTRA_ARGS ) \ + ( cd "$dir" && exec ./profanity2.x64 "${args[@]}" -z "$SEED_KEY" $EXTRA_ARGS ) \ >"$out" 2>"$err" & local pid=$! @@ -282,8 +303,9 @@ main() { echo " workload: $describe $EXTRA_ARGS" echo " window: ${REPEATS} x ${WINDOW}s per revision, first ${WARMUP}s of each run dropped" echo " order: $(for _ in $(seq "$REPEATS"); do printf 'A B '; done)" - if [ "$PUBLIC_KEY" = "$GENERATOR_PUBLIC_KEY" ]; then - echo " key: secp256k1 generator - PUBLIC, never use a result from this run" + echo " key: $KEY_SOURCE" + if [ "$SEED_KEY" = "$GENERATOR_PUBLIC_KEY" ]; then + echo " a key everybody knows - never use a result from this run" fi echo @@ -377,7 +399,11 @@ while [ "$#" -gt 0 ]; do --repeats) REPEATS="${2:?--repeats needs a value}"; shift 2 ;; --extra-args) EXTRA_ARGS="${2:?--extra-args needs a value}"; shift 2 ;; --mask) EXACT_MASK="${2:?--mask needs a value}"; shift 2 ;; - --public-key) PUBLIC_KEY="${2:?--public-key needs a value}"; shift 2 ;; + --public-key) + SEED_KEY="${2:?--public-key needs a value}" + KEY_SOURCE="--public-key" + shift 2 + ;; -h|--help) usage; exit 0 ;; *) die "unknown option '$1', try --help" ;; esac From f3d50f8c08c276bbb0919733275ab0fbfeb4bc53 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 23:20:33 +0000 Subject: [PATCH 4/6] Make GHCR the main way to publish the benchmark image ttl.sh was the only documented target, which forced a random image name and a lifetime of at most a day on anyone running more than one comparison. The name is now chosen to suit the registry: the local tag is reused everywhere except ttl.sh, where the tag has to carry the lifetime. Co-authored-by: Gleb Alekseev --- bench/README.md | 153 +++++++++++++++++++++-------------------------- bench/publish.sh | 74 ++++++++++++++++------- 2 files changed, 120 insertions(+), 107 deletions(-) diff --git a/bench/README.md b/bench/README.md index 6073b8a..bd56845 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,25 +1,8 @@ # Comparing the speed of two profanity2 revisions -This directory builds a **separate image** from the one in the repository root. -The image in the root ships profanity2; this one measures it. It contains two -revisions of profanity2 side by side and runs them alternately on the same GPU, -so a change can be judged without trusting that two rented machines are -equally fast. +This directory builds a **separate image** from the one in the repository root. The image in the root ships profanity2; this one measures it. It contains two revisions of profanity2 side by side and runs them alternately on the same GPU, so a change can be judged without trusting that two rented machines are equally fast. -```bash -bench/publish.sh master pr/57 -``` - -builds both revisions, pushes the image to [ttl.sh](https://ttl.sh) and prints -its name: - -``` -IMAGE: ttl.sh/profanity2-bench-1f0c9a3e-...:24h - - vastai create instance --image ttl.sh/profanity2-bench-...:24h --disk 12 --args --mode leading -``` - -Rent any NVIDIA GPU with that image and the instance log ends with: +The result of a run looks like this: ``` =================== BENCHMARK RESULT =================== @@ -33,6 +16,55 @@ B vs A: -12.8% BENCH_RESULT mode=leading a_hs=1390400000 b_hs=1212700000 delta_pct=-12.8 ``` +## Publishing to GHCR + +The GitHub Container Registry is the place to put the image if you are going to run more than one comparison: the image stays until you delete it, the name is readable, and pushing costs nothing. + +Create a personal access token with the `write:packages` scope, log in once, then build and push: + +```bash +echo YOUR_TOKEN | docker login ghcr.io -u YOUR_USER --password-stdin +bench/publish.sh master pr/57 --registry ghcr.io/YOUR_USER +``` + +The script prints the full image name, which is derived from the two revisions: + +``` +IMAGE: ghcr.io/YOUR_USER/profanity2-bench:master__pr-57 +``` + +A package pushed to GHCR is **private by default**, so a rented machine cannot pull it yet. Either make it public once, under Packages on your GitHub profile, or hand the credentials to vast.ai: + +```bash +# public package +vastai create instance --image ghcr.io/YOUR_USER/profanity2-bench:master__pr-57 \ + --disk 12 --args --mode leading + +# private package +vastai create instance --image ghcr.io/YOUR_USER/profanity2-bench:master__pr-57 \ + --login '-u YOUR_USER -p YOUR_TOKEN ghcr.io' --disk 12 --args --mode leading +``` + +Pushing the same pair of revisions again overwrites the same tag, so the registry does not fill up with junk. Use `--name` if you want to keep several builds of one pair side by side: + +```bash +bench/publish.sh master pr/57 --registry ghcr.io/YOUR_USER --name profanity2-bench:run-2 +``` + +## Publishing to ttl.sh + +[ttl.sh](https://ttl.sh) is an anonymous registry that deletes what you push after the time given in the tag. No account, no login, no cleanup - useful for a one-off comparison or when you do not want to bother with tokens: + +```bash +bench/publish.sh master pr/57 --registry ttl.sh --ttl 24h +``` + +``` +IMAGE: ttl.sh/profanity2-bench-1f0c9a3e-...:24h +``` + +On ttl.sh the tag is the lifetime and the maximum is 24 hours, so the image needs a unique name instead - the script generates a random one. Anyone who learns that name can pull the image, which is harmless here: it holds nothing but public source code. Do keep the lifetime longer than your experiment, because a machine that has to pull the image again after it expired will fail to start. + ## What can be passed as a revision Anything a clone of the repository can resolve: @@ -43,12 +75,9 @@ Anything a clone of the repository can resolve: | `9011bcd`, full SHA | any commit, including one that only exists in a pull request | | `pr/57` | the head of pull request 57 | -Pull request heads are fetched explicitly during the build, which is why a -commit like `9011bcd` resolves even though it never landed on a branch. +Pull request heads are fetched explicitly during the build, which is why a commit like `9011bcd` resolves even though it never landed on a branch. -Both revisions are compiled inside the image from a fresh clone, so your -working tree, your local branches and your uncommitted changes play no part. -Point the build at a fork with `--repo`: +Both revisions are compiled inside the image from a fresh clone, so your working tree, your local branches and your uncommitted changes play no part. Point the build at a fork with `--repo`: ```bash bench/build.sh master pr/57 --repo https://github.com/YOUR_USER/profanity2 @@ -60,26 +89,18 @@ bench/build.sh master pr/57 --repo https://github.com/YOUR_USER/profanity2 # build only, prints the local tag on the last line bench/build.sh master pr/57 -# build, push to ttl.sh for 24 hours, print the image name -bench/publish.sh master pr/57 - # push an image that is already built -bench/publish.sh --image profanity2-bench:master__pr-57 --ttl 1h +bench/publish.sh --image profanity2-bench:master__pr-57 --registry ghcr.io/YOUR_USER # on your own machine, if it has an NVIDIA GPU docker run --rm --gpus all profanity2-bench:master__pr-57 ``` -The image is built for `linux/amd64` by default because the Linux branch of the -Makefile passes `-mmmx` and `-mcmodel=large`, which do not exist on arm64, and -because GPU rental platforms are x86_64 anyway. On Apple Silicon the build -therefore runs under emulation and takes a few minutes. +The image is built for `linux/amd64` by default because the Linux branch of the Makefile passes `-mmmx` and `-mcmodel=large`, which do not exist on arm64, and because GPU rental platforms are x86_64 anyway. On Apple Silicon the build therefore runs under emulation and takes a few minutes. ## Options -Everything has a flag and an environment variable; flags are easier on -platforms that pass arguments to the entrypoint, variables are the only option -on platforms that replace it. +Everything has a flag and an environment variable; flags are easier on platforms that pass arguments to the entrypoint, variables are the only option on platforms that replace it. | Flag | Variable | Default | Meaning | |---|---|---|---| @@ -92,75 +113,45 @@ on platforms that replace it. | `--public-key` | `BENCH_PUBLIC_KEY` | secp256k1 generator | seed public key | | | `BENCH_SKIP_GPU_CHECK` | unset | start even when no OpenCL platform is detected | -A plain `PUBLIC_KEY` is honoured as well, but only when it holds 128 -hexadecimal characters. Rental platforms hand out that generic name for their -own SSH key, and it may already sit in your account-wide environment -variables, so a value that is not a seed public key is ignored and the run -falls back to the default. The header of every run says which key it used: +A plain `PUBLIC_KEY` is honoured as well, but only when it holds 128 hexadecimal characters. Rental platforms hand out that generic name for their own SSH key, and it may already sit in your account-wide environment variables, so a value that is not a seed public key is ignored and the run falls back to the default. The header of every run says which key it used: ``` key: the secp256k1 generator (PUBLIC_KEY ignored, not 128 hex characters) ``` -With the defaults a full comparison takes about ten minutes plus kernel -compilation, which is around six cents on an RTX 4090. +With the defaults a full comparison takes about ten minutes plus kernel compilation, which is around six cents on an RTX 4090. -An argument that does not start with a dash is executed instead of the -benchmark, which is the quickest way to inspect a rented machine: +An argument that does not start with a dash is executed instead of the benchmark, which is the quickest way to inspect a rented machine: ```bash docker run --rm --gpus all profanity2-bench:master__pr-57 clinfo ``` -The default seed public key is the generator point of secp256k1. It is a valid -public key whose private key is the publicly known value 1, which is fine for a -benchmark and useless for anything else - never treat a key found during a -benchmark run as yours. +The default seed public key is the generator point of secp256k1. It is a valid public key whose private key is the publicly known value 1, which is fine for a benchmark and useless for anything else - never treat a key found during a benchmark run as yours. ## Getting a number that means something -**The two revisions take turns.** The order is A, B, A, B, never A, A, B, B: a -GPU that heats up or gets throttled halfway through would otherwise hand the -whole penalty to whichever revision ran last. The reported `spread` is how far -apart the repeats of one revision landed. If it is as large as the difference -between the revisions, the machine is too noisy and the result means nothing - -the runner says so explicitly. +**The two revisions take turns.** The order is A, B, A, B, never A, A, B, B: a GPU that heats up or gets throttled halfway through would otherwise hand the whole penalty to whichever revision ran last. The reported `spread` is how far apart the repeats of one revision landed. If it is as large as the difference between the revisions, the machine is too noisy and the result means nothing - the runner says so explicitly. -**Each revision has its own directory and its own kernel cache.** profanity2 -compiles its OpenCL kernel on first use and caches it next to the binary; two -revisions sharing a directory would run each other's compiled kernel. +**Each revision has its own directory and its own kernel cache.** profanity2 compiles its OpenCL kernel on first use and caches it next to the binary; two revisions sharing a directory would run each other's compiled kernel. -**The first seconds of every run are discarded.** The window only opens once -the program starts reporting speed, so kernel compilation and device -initialization never land inside it, no matter how slow the device is. +**The first seconds of every run are discarded.** The window only opens once the program starts reporting speed, so kernel compilation and device initialization never land inside it, no matter how slow the device is. -**`--benchmark` is deliberately not supported.** In a revision where the -scoring kernel is fused into the iterate kernel, a scoring function that does -nothing lets the compiler delete the keccak as well, and the resulting number -is meaningless. That is why pull request 57 carries a `benchmark` scoring -function whose comment reads *"Prevent the compiler from deleting the keccak -behind profanity_iterate"*. Real scoring modes do not have this problem. +**`--benchmark` is deliberately not supported.** In a revision where the scoring kernel is fused into the iterate kernel, a scoring function that does nothing lets the compiler delete the keccak as well, and the resulting number is meaningless. That is why pull request 57 carries a `benchmark` scoring function whose comment reads *"Prevent the compiler from deleting the keccak behind profanity_iterate"*. Real scoring modes do not have this problem. -**Watch out for revisions that changed the speed counter itself.** Until -commit `9011bcd` the duration of a round was truncated to whole milliseconds: +**Watch out for revisions that changed the speed counter itself.** Until commit `9011bcd` the duration of a round was truncated to whole milliseconds: ```c++ auto delta = std::chrono::duration_cast(newTime - m_lastTime).count(); m_lSpeeds.push_back((1000 * V) / delta); ``` -A round with default settings is `255 * 16384 = 4177920` keys, which on a -4090 takes about 3.8 ms and gets counted as 3 ms - the printed speed is then -27% above the truth, and on a faster card a sub-millisecond round divides by -zero. Comparing such a build against one that measures in microseconds -measures the fix, not the kernels. +A round with default settings is `255 * 16384 = 4177920` keys, which on a 4090 takes about 3.8 ms and gets counted as 3 ms - the printed speed is then 27% above the truth, and on a faster card a sub-millisecond round divides by zero. Comparing such a build against one that measures in microseconds measures the fix, not the kernels. -The build detects this: if `SpeedSample.cpp` differs between the two -revisions, the result block ends with a warning. Then either compare against -a baseline carrying the same timer, +The build detects this: if `SpeedSample.cpp` differs between the two revisions, the result block ends with a warning. Then either compare against a baseline carrying the same timer, ```bash -bench/publish.sh 9011bcd pr/57 +bench/publish.sh 9011bcd pr/57 --registry ghcr.io/YOUR_USER ``` or measure without the program's clock at all: @@ -169,10 +160,4 @@ or measure without the program's clock at all: docker run --rm --gpus all IMAGE --mode exact ``` -`--mode exact` uses `--exact `, which prints every address matching the -mask, and derives the throughput from how many appear per second: -`rate = matches / seconds * 16^fixed`. Nothing in that number comes from the -program's own timer. It is a counting measurement, so its precision is -`1/sqrt(matches)`: the default 7-character mask on a 1 GH/s card yields about -four matches per second, so a 120 second window gives roughly 450 matches and -5% precision. Widen `--seconds` to resolve smaller differences. +`--mode exact` uses `--exact `, which prints every address matching the mask, and derives the throughput from how many appear per second: `rate = matches / seconds * 16^fixed`. Nothing in that number comes from the program's own timer. It is a counting measurement, so its precision is `1/sqrt(matches)`: the default 7-character mask on a 1 GH/s card yields about four matches per second, so a 120 second window gives roughly 450 matches and 5% precision. Widen `--seconds` to resolve smaller differences. diff --git a/bench/publish.sh b/bench/publish.sh index cf3deab..5ec78ac 100755 --- a/bench/publish.sh +++ b/bench/publish.sh @@ -1,26 +1,31 @@ #!/usr/bin/env bash # -# Publishes a benchmark image to ttl.sh, an anonymous registry that deletes -# what you push after the time given in the tag, and prints its name. +# Pushes a benchmark image to a registry a rented machine can pull from, and +# prints its name. # # usage: bench/publish.sh REF_A REF_B [OPTIONS] # bench/publish.sh --image LOCAL_TAG [OPTIONS] # -# --ttl <5m|1h|24h> how long the image stays on ttl.sh [default = 24h] -# --registry registry to push to [default = ttl.sh] -# --image publish an already built local image -# --repo passed to build.sh -# --platform

passed to build.sh -# -h, --help this text +# --registry where to push, e.g. ghcr.io/YOUR_USER or ttl.sh +# (also read from BENCH_REGISTRY) +# --name name to push under, overrides the default +# --ttl <5m|1h|24h> lifetime on ttl.sh, ignored elsewhere +# [default = 24h] +# --image publish an already built local image +# --repo passed to build.sh +# --platform

passed to build.sh +# -h, --help this text # -# On ttl.sh the tag is the lifetime, so the image name has to be unique - the -# script generates a random one. Anyone who learns that name can pull the -# image, which is harmless here: it holds nothing but public source code. +# The name is chosen to suit the registry. On ttl.sh the tag is the lifetime, +# so the image needs a unique random name and expires by itself. Everywhere +# else the local tag is reused, which keeps one repository with one tag per +# pair of revisions. set -euo pipefail TTL="24h" -REGISTRY="${BENCH_REGISTRY:-ttl.sh}" +REGISTRY="${BENCH_REGISTRY:-}" +NAME="" LOCAL_TAG="" REFS=() BUILD_ARGS=() @@ -34,10 +39,21 @@ die() { exit 1 } +random_name() { + if command -v uuidgen >/dev/null 2>&1; then + uuidgen | tr '[:upper:]' '[:lower:]' + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import uuid; print(uuid.uuid4())' + else + od -An -tx1 -N16 /dev/urandom | tr -d ' \n' + fi +} + while [ "$#" -gt 0 ]; do case "$1" in - --ttl) TTL="${2:?--ttl needs a value}"; shift 2 ;; --registry) REGISTRY="${2:?--registry needs a value}"; shift 2 ;; + --name) NAME="${2:?--name needs a value}"; shift 2 ;; + --ttl) TTL="${2:?--ttl needs a value}"; shift 2 ;; --image) LOCAL_TAG="${2:?--image needs a value}"; shift 2 ;; --repo) BUILD_ARGS+=(--repo "${2:?--repo needs a value}"); shift 2 ;; --platform) BUILD_ARGS+=(--platform "${2:?--platform needs a value}"); shift 2 ;; @@ -47,6 +63,21 @@ while [ "$#" -gt 0 ]; do esac done +REGISTRY="${REGISTRY%/}" + +if [ -z "$REGISTRY" ]; then + printf 'publish.sh: error: no registry given, pick one:\n' >&2 + printf ' --registry ghcr.io/YOUR_USER keeps the image, needs docker login ghcr.io\n' >&2 + printf ' --registry ttl.sh anonymous, expires after --ttl\n' >&2 + exit 1 +fi + +case "$REGISTRY" in + ghcr.io|docker.io) + die "$REGISTRY needs your account in the path, e.g. --registry $REGISTRY/YOUR_USER" + ;; +esac + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -z "$LOCAL_TAG" ]; then @@ -55,17 +86,14 @@ if [ -z "$LOCAL_TAG" ]; then [ -n "$LOCAL_TAG" ] || die "the build produced no image" fi -random_name() { - if command -v uuidgen >/dev/null 2>&1; then - uuidgen | tr '[:upper:]' '[:lower:]' - elif command -v python3 >/dev/null 2>&1; then - python3 -c 'import uuid; print(uuid.uuid4())' - else - od -An -tx1 -N16 /dev/urandom | tr -d ' \n' - fi -} +if [ -z "$NAME" ]; then + case "$REGISTRY" in + ttl.sh|*.ttl.sh) NAME="profanity2-bench-$(random_name):$TTL" ;; + *) NAME="${LOCAL_TAG##*/}" ;; + esac +fi -remote="$REGISTRY/profanity2-bench-$(random_name):$TTL" +remote="$REGISTRY/$NAME" printf 'publish.sh: pushing %s as %s\n' "$LOCAL_TAG" "$remote" >&2 docker tag "$LOCAL_TAG" "$remote" From 699197318246b8b4ffa8005393aae0cc10aade98 Mon Sep 17 00:00:00 2001 From: Gleb Alekseev Date: Mon, 3 Aug 2026 10:47:08 +0300 Subject: [PATCH 5/6] benchmarks --- .gitignore | 1 + bench-logs/1xRTX3060.vastai.log | 24 ++++++++++++++++++++++++ bench-logs/1xRTX5090.vastai.log | 23 +++++++++++++++++++++++ bench-logs/2xRTX3060.vastai.log | 23 +++++++++++++++++++++++ bench-logs/8xRTX5090.vastai.log | 23 +++++++++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 bench-logs/1xRTX3060.vastai.log create mode 100644 bench-logs/1xRTX5090.vastai.log create mode 100644 bench-logs/2xRTX3060.vastai.log create mode 100644 bench-logs/8xRTX5090.vastai.log diff --git a/.gitignore b/.gitignore index dc76ad4..f361fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ cache-opencl.* bin __pycache__/ +.venv \ No newline at end of file diff --git a/bench-logs/1xRTX3060.vastai.log b/bench-logs/1xRTX3060.vastai.log new file mode 100644 index 0000000..5ae71c8 --- /dev/null +++ b/bench-logs/1xRTX3060.vastai.log @@ -0,0 +1,24 @@ +profanity2 benchmark + A: 9011bcd (9011bcd) + B: pr/57 (aad8e13) + workload: --leading 0 -i 255 -I 16384 -w 64 + window: 2 x 120s per revision, first 30s of each run dropped + order: A B A B + key: secp256k1 generator - PUBLIC, never use a result from this run +bench: run 1/2, revision A (9011bcd (9011bcd)) +bench: run 1/2, revision A: 303.6 MH/s +bench: run 1/2, revision B (pr/57 (aad8e13)) +bench: run 1/2, revision B: 329.6 MH/s +bench: run 2/2, revision A (9011bcd (9011bcd)) +bench: run 2/2, revision A: 302.2 MH/s +bench: run 2/2, revision B (pr/57 (aad8e13)) +bench: run 2/2, revision B: 329.8 MH/s +=================== BENCHMARK RESULT =================== +A 9011bcd (9011bcd) median 302.9 MH/s spread 0.5% + runs: 303.6 MH/s 302.2 MH/s +B pr/57 (aad8e13) median 329.7 MH/s spread 0.0% + runs: 329.6 MH/s 329.8 MH/s +B vs A: +8.9% +======================================================== +BENCH_RESULT mode=leading a_hs=302913250 b_hs=329727250 delta_pct=+8.9 + diff --git a/bench-logs/1xRTX5090.vastai.log b/bench-logs/1xRTX5090.vastai.log new file mode 100644 index 0000000..f98bc13 --- /dev/null +++ b/bench-logs/1xRTX5090.vastai.log @@ -0,0 +1,23 @@ +profanity2 benchmark + A: 9011bcd (9011bcd) + B: pr/57 (aad8e13) + workload: --leading 0 -i 255 -I 16384 -w 64 + window: 2 x 120s per revision, first 30s of each run dropped + order: A B A B + key: secp256k1 generator - PUBLIC, never use a result from this run +bench: run 1/2, revision A (9011bcd (9011bcd)) +bench: run 1/2, revision A: 1695.0 MH/s +bench: run 1/2, revision B (pr/57 (aad8e13)) +bench: run 1/2, revision B: 1801.0 MH/s +bench: run 2/2, revision A (9011bcd (9011bcd)) +bench: run 2/2, revision A: 1694.0 MH/s +bench: run 2/2, revision B (pr/57 (aad8e13)) +bench: run 2/2, revision B: 1801.0 MH/s +=================== BENCHMARK RESULT =================== +A 9011bcd (9011bcd) median 1694.5 MH/s spread 0.1% + runs: 1695.0 MH/s 1694.0 MH/s +B pr/57 (aad8e13) median 1801.0 MH/s spread 0.0% + runs: 1801.0 MH/s 1801.0 MH/s +B vs A: +6.3% +======================================================== +BENCH_RESULT mode=leading a_hs=1694500000 b_hs=1801000000 delta_pct=+6.3 diff --git a/bench-logs/2xRTX3060.vastai.log b/bench-logs/2xRTX3060.vastai.log new file mode 100644 index 0000000..d4be650 --- /dev/null +++ b/bench-logs/2xRTX3060.vastai.log @@ -0,0 +1,23 @@ +profanity2 benchmark + A: 9011bcd (9011bcd) + B: pr/57 (aad8e13) + workload: --leading 0 -i 255 -I 16384 -w 64 + window: 2 x 120s per revision, first 30s of each run dropped + order: A B A B + key: secp256k1 generator - PUBLIC, never use a result from this run +bench: run 1/2, revision A (9011bcd (9011bcd)) +bench: run 1/2, revision A: 507.2 MH/s +bench: run 1/2, revision B (pr/57 (aad8e13)) +bench: run 1/2, revision B: 567.5 MH/s +bench: run 2/2, revision A (9011bcd (9011bcd)) +bench: run 2/2, revision A: 507.5 MH/s +bench: run 2/2, revision B (pr/57 (aad8e13)) +bench: run 2/2, revision B: 566.4 MH/s +=================== BENCHMARK RESULT =================== +A 9011bcd (9011bcd) median 507.4 MH/s spread 0.1% + runs: 507.2 MH/s 507.5 MH/s +B pr/57 (aad8e13) median 567.0 MH/s spread 0.2% + runs: 567.5 MH/s 566.4 MH/s +B vs A: +11.7% +======================================================== +BENCH_RESULT mode=leading a_hs=507378000 b_hs=566965000 delta_pct=+11.7 \ No newline at end of file diff --git a/bench-logs/8xRTX5090.vastai.log b/bench-logs/8xRTX5090.vastai.log new file mode 100644 index 0000000..339e6af --- /dev/null +++ b/bench-logs/8xRTX5090.vastai.log @@ -0,0 +1,23 @@ +profanity2 benchmark + A: 9011bcd (9011bcd) + B: pr/57 (aad8e13) + workload: --leading 0 -i 255 -I 16384 -w 64 + window: 2 x 120s per revision, first 30s of each run dropped + order: A B A B + key: secp256k1 generator - PUBLIC, never use a result from this run +bench: run 1/2, revision A (9011bcd (9011bcd)) +bench: run 1/2, revision A: 12904.0 MH/s +bench: run 1/2, revision B (pr/57 (aad8e13)) +bench: run 1/2, revision B: 13811.0 MH/s +bench: run 2/2, revision A (9011bcd (9011bcd)) +bench: run 2/2, revision A: 12852.0 MH/s +bench: run 2/2, revision B (pr/57 (aad8e13)) +bench: run 2/2, revision B: 13767.0 MH/s +=================== BENCHMARK RESULT =================== +A 9011bcd (9011bcd) median 12878.0 MH/s spread 0.4% + runs: 12904.0 MH/s 12852.0 MH/s +B pr/57 (aad8e13) median 13789.0 MH/s spread 0.3% + runs: 13811.0 MH/s 13767.0 MH/s +B vs A: +7.1% +======================================================== +BENCH_RESULT mode=leading a_hs=12878000000 b_hs=13789000000 delta_pct=+7.1 From a8fdfec2aaa36b869d0063a3a8c80610cc321f39 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 3 Aug 2026 08:25:55 +0000 Subject: [PATCH 6/6] Terminate .gitignore with a newline Co-authored-by: Gleb Alekseev --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f361fbd..8f7ee92 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ cache-opencl.* bin __pycache__/ -.venv \ No newline at end of file +.venv