From 5ed7b89fbe9f9ab871506dbb8283ae0317357ab4 Mon Sep 17 00:00:00 2001 From: katzman Date: Wed, 10 Jun 2026 13:02:08 -0700 Subject: [PATCH 1/2] first pass --- .github/workflows/base-std-fork-tests.yml | 8 +- FORK_TESTING.md | 52 +++-- Makefile | 14 +- script/fork/README.md | 63 ++++++ script/fork/__init__.py | 1 + script/fork/__main__.py | 264 ++++++++++++++++++++++ script/run-fork-tests.sh | 214 ------------------ 7 files changed, 378 insertions(+), 238 deletions(-) create mode 100644 script/fork/README.md create mode 100644 script/fork/__init__.py create mode 100644 script/fork/__main__.py delete mode 100755 script/run-fork-tests.sh diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index 1e9fb4c..0b7a7cb 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -122,6 +122,12 @@ jobs: "$BASE_ANVIL_DIR/target/release/anvil" --version "$BASE_ANVIL_DIR/target/release/forge" --version + - name: Set up fork-test runner venv + shell: bash + # The runner is `python -m fork` (web3 only); reuse the smoke venv target with the + # runner's preinstalled python3 instead of the python3.13 default. + run: make smoke-setup PYTHON=python3 + - name: Run base-std fork tests id: fork_tests shell: bash @@ -131,7 +137,7 @@ jobs: ANVIL_BIN="$BASE_ANVIL_DIR/target/release/anvil" \ FORGE_BIN="$BASE_ANVIL_DIR/target/release/forge" \ ANVIL_LOG="$RUNNER_TEMP/base-std-anvil.log" \ - ./script/run-fork-tests.sh 2>&1 | tee "$RUNNER_TEMP/fork-test-output.txt" + make fork-tests 2>&1 | tee "$RUNNER_TEMP/fork-test-output.txt" echo "fork_tests_exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" continue-on-error: true diff --git a/FORK_TESTING.md b/FORK_TESTING.md index bd022c4..172bcf7 100644 --- a/FORK_TESTING.md +++ b/FORK_TESTING.md @@ -57,14 +57,21 @@ Clone two repos as siblings (base/base is fetched automatically by cargo): ``` If your layout differs, set `ANVIL_BIN` and `FORGE_BIN` env vars to -override the script's defaults. +override the runner's defaults. Install Rust + the fast linker, plus stock foundry (for `cast`): ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal brew install lld # macOS; Linux uses mold per base-anvil's .cargo/config.toml -curl -L https://foundry.paradigm.xyz | bash && foundryup # stock foundry, for `cast` (used by run-fork-tests.sh) +curl -L https://foundry.paradigm.xyz | bash && foundryup # stock foundry, for `cast` (the manual probe below) +``` + +The runner itself is Python ([`script/fork/`](script/fork/)), driven by `web3`. +Create its venv once (shared with the smoke suite): + +```bash +make smoke-setup ``` Build the patched forge + anvil (~30 min first build, incremental after). @@ -79,10 +86,10 @@ cargo build --release -p anvil -p forge ```bash cd ~/code/base-std -./script/run-fork-tests.sh +make fork-tests ``` -The script: +The runner ([`script/fork/`](script/fork/)): 1. Launches `anvil --base --base-activation-admin 0x9965507D...` on port 8546. 2. Funds + impersonates the activation admin, sends `activate(bytes32)` for @@ -91,10 +98,10 @@ The script: http://localhost:8546`. 4. Tears down anvil. -Forward any `forge test` flag through the script: +Forward any `forge test` flag through `ARGS`: ```bash -./script/run-fork-tests.sh -vvvv --match-test test_transfer_success_debitsSender +make fork-tests ARGS="-vvvv --match-test test_transfer_success_debitsSender" ``` ## Exercising the inactive-feature dispatch path @@ -106,8 +113,8 @@ feature is active), set `SKIP_ACTIVATE` to a comma-separated list of feature names (or raw `0x` ids) to leave un-activated: ```bash -SKIP_ACTIVATE=POLICY_REGISTRY ./script/run-fork-tests.sh \ - --match-contract PolicyRegistryDispatchInactive +SKIP_ACTIVATE=POLICY_REGISTRY make fork-tests \ + ARGS="--match-contract PolicyRegistryDispatchInactive" ``` The inactive-dispatch tests (`test/unit/PolicyRegistry/dispatch_inactive.t.sol`) @@ -118,8 +125,8 @@ activation gate rather than masked by `FeatureNotActivated`) encodes a fix that isn't in the Rust impl yet, so it's gated behind `POLICY_DISPATCH_FIX`: ```bash -SKIP_ACTIVATE=POLICY_REGISTRY POLICY_DISPATCH_FIX=true ./script/run-fork-tests.sh \ - --match-contract PolicyRegistryDispatchInactive +SKIP_ACTIVATE=POLICY_REGISTRY POLICY_DISPATCH_FIX=true make fork-tests \ + ARGS="--match-contract PolicyRegistryDispatchInactive" ``` Run it that way against a build that carries the dispatch-ordering fix (e.g. via @@ -149,10 +156,12 @@ Skim the new commits for new precompile addresses, feature IDs, or ABI changes (`crates/common/precompiles/src/activation/storage.rs` for `FEATURE_*` constants). -**Step 2: if new feature IDs were added**, append them to the -`FEATURE_IDS` array in `script/run-fork-tests.sh`. The script must activate -every gated feature before tests run; otherwise the feature's calls revert -`FeatureNotActivated`. +**Step 2: if new feature IDs were added**, add them to the derived feature set +the runner activates: the canonical `FEATURE_*` keccak constants in +[`script/smoke/config.py`](script/smoke/config.py) and the `FEATURES` table in +[`script/fork/__main__.py`](script/fork/__main__.py) (which reuses those +constants). The runner must activate every gated feature before tests run; +otherwise the feature's calls revert `FeatureNotActivated`. **Step 3: if new precompiles were added** (a new `*Precompile::install` call appeared in `base/crates/common/precompiles/src/provider.rs`): @@ -170,7 +179,7 @@ Then rerun `./script/bump-base.sh ` to rebuild against the new pin. ```bash cd ~/code/base-std -./script/run-fork-tests.sh +make fork-tests ``` **Step 5: triage the deltas.** Compare against the last run's failure @@ -223,10 +232,10 @@ block when you're done iterating. ## Common failure modes & fixes **`anvil binary not found`** — run `cargo build --release -p anvil` in -`base-anvil/`. Or `ANVIL_BIN=/abs/path ./script/run-fork-tests.sh`. +`base-anvil/`. Or `ANVIL_BIN=/abs/path make fork-tests`. **`port 8546 is already in use`** — `pkill -f "base-anvil/target/.*/anvil"` -or `PORT=8547 ./script/run-fork-tests.sh`. +or `PORT=8547 make fork-tests`. **`anvil exited during startup`** — check `/tmp/anvil.log`. Usual cause: rust build is stale after a base/base change; rebuild. @@ -234,12 +243,13 @@ rust build is stale after a base/base change; rebuild. **Hundreds of `EvmError: Revert` with `gas: 0` in `setUp`** — either the `LIVE_PRECOMPILES` env var wasn't set (BaseTest etched the mocks over the precompile addresses) or `[profile.fork] base = true` is missing from -`foundry.toml` (forge isn't installing the precompiles). The script sets +`foundry.toml` (forge isn't installing the precompiles). The runner sets both, but if you're invoking forge directly, set both. **`FeatureNotActivated(bytes32)` revert payload** — a new gated feature -landed in base/base. Add its ID to `FEATURE_IDS` in -`script/run-fork-tests.sh`. The payload's 32-byte tail IS the feature ID +landed in base/base. Add its ID to the derived feature set (see Step 2 above: +`FEATURE_*` in `script/smoke/config.py` + the `FEATURES` table in +`script/fork/__main__.py`). The payload's 32-byte tail IS the feature ID (grep `base/crates/common/precompiles/src/activation/storage.rs` for the matching `FEATURE_*` constant). @@ -275,7 +285,7 @@ If this returns garbage / fails, the fork's build is broken or out of date. | Thing | Path | Notes | |---|---|---| -| The test runner script | `script/run-fork-tests.sh` | bash; takes forge args through `$@` | +| The test runner | `script/fork/` (`make fork-tests`) | Python + web3; forwards forge args through `ARGS` | | Forge profile config | `foundry.toml`, `[profile.fork]` | `base = true` enables Rust precompile dispatch | | Skip-etch logic | `test/lib/BaseTest.sol` | guarded by `LIVE_PRECOMPILES` env var | | Slot assertions | `test/unit/**/*.t.sol`, `test/lib/mocks/Mock*Storage.sol` | `vm.load`-based slot-layout assertions paired with surface tests | diff --git a/Makefile b/Makefile index 77d513a..f95acaf 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,12 @@ LOAD_ENV = pre=$$(export -p); set -a; [ -f .env ] && . ./.env; set +a; eval "$$p PYTHON ?= python3.13 VENV = script/smoke/.venv -# `smoke` is the package at script/smoke/, so its parent (script) is on the path. +# `smoke` and `fork` are packages under script/, so script/ is on the path. Both share the one venv +# (web3 is their only dependency); `make smoke-setup` provisions it. SMOKE_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m smoke +FORK_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m fork -.PHONY: build coverage smoke smoke-all smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants smoke-setup +.PHONY: build coverage smoke smoke-all smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants smoke-setup fork-tests # Generate an lcov coverage report and open it in the browser. # Scoped to src/ and test/lib/mocks/ (excludes test runner files and the smoke probe helper). @@ -22,6 +24,14 @@ smoke-setup: $(VENV)/bin/python -m pip install --upgrade pip $(VENV)/bin/python -m pip install -r script/smoke/requirements.txt +# Run the unit suite against a local anvil with Base's Rust precompiles, cross-validating the Solidity +# reference against the live Rust impl. Needs the patched anvil+forge from the base-anvil fork (see +# script/fork/__main__.py for env vars). Forward forge args via ARGS, e.g. +# make fork-tests ARGS="-vvvv --match-test test_transfer_success_debitsSender" +# make fork-tests ARGS="--match-contract PolicyRegistryDispatchInactive" SKIP_ACTIVATE=POLICY_REGISTRY +fork-tests: + @$(FORK_RUN) $(ARGS) + # Compile the contracts. build: forge build diff --git a/script/fork/README.md b/script/fork/README.md new file mode 100644 index 0000000..3e5b556 --- /dev/null +++ b/script/fork/README.md @@ -0,0 +1,63 @@ +# fork-test runner + +`python -m fork [forge test args...]` runs the base-std unit suite against a +**local anvil that dispatches Base's Rust precompiles**, cross-validating the +Solidity reference against the live Rust impl from `base/base`. It is the Python +port of the former `script/run-fork-tests.sh`. + +Both binaries (anvil + forge) must come from the **base-anvil fork** of +foundry-rs, which adds a `--base` flag that installs the B-20 precompile suite +into the EVM. Stock foundry binaries will not work. + +## What it does + +1. Discovers the patched `anvil` + `forge` (env override or the base-anvil + `target/release` → `target/debug` default layout). +2. Launches anvil on `$PORT` with `--base`, waits for the RPC to come up, and + tears it down on exit (pass, fail, or crash). +3. Funds + impersonates the activation admin and calls + `ActivationRegistry.activate(bytes32)` for each gated feature. +4. Runs `forge test --fork-url` under `FOUNDRY_PROFILE=fork` + + `LIVE_PRECOMPILES=true`, forwarding any extra args, and propagates forge's + exit code. + +The gated feature ids are **derived** (`keccak` of the canonical feature names) +by importing them from the sibling [`smoke` package's `config.py`](../smoke/config.py), +so there is a single source of truth shared with the smoketest — no hand-kept +hex table to drift from the Solidity `ActivationRegistryFeatureList` / Rust +`storage.rs`. + +## Running + +```bash +make smoke-setup # one-time: create the shared venv + install web3 (shared with `make smoke`) + +make fork-tests # whole suite +make fork-tests ARGS="-vvvv --match-test test_transfer_success" # scope + verbosity +make fork-tests ARGS="--match-contract PolicyRegistryDispatchInactive" SKIP_ACTIVATE=POLICY_REGISTRY +``` + +Or directly (the Makefile just sources `.env` and sets `PYTHONPATH=script`): + +```bash +PYTHONPATH=script script/smoke/.venv/bin/python -m fork --match-test test_transfer_success +``` + +## Environment + +| Var | Default | Meaning | +| --- | --- | --- | +| `ANVIL_BIN` | `../base-anvil/target/release/anvil` (→ `debug`) | patched anvil binary | +| `FORGE_BIN` | `forge` next to `ANVIL_BIN` | patched forge binary | +| `PORT` | `8546` | local RPC port for anvil | +| `ACTIVATION_ADMIN` | `0x9965…A4dc` | address authorized to activate features | +| `ANVIL_LOG` | `/tmp/anvil.log` | anvil stdout/stderr log path | +| `SKIP_ACTIVATE` | _(none)_ | comma-separated feature names or `0x` ids to leave un-activated (exercises the inactive-feature dispatch path); matched case-insensitively | + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | all targeted tests pass | +| `1` | at least one targeted test fails — the output **is** the cross-validation signal | +| `2` | environment problem (missing binary, port in use, anvil failed to start, activation tx failed) | diff --git a/script/fork/__init__.py b/script/fork/__init__.py new file mode 100644 index 0000000..c79bd62 --- /dev/null +++ b/script/fork/__init__.py @@ -0,0 +1 @@ +"""Fork-test runner: drive the base-std unit suite against a local anvil that dispatches Base's Rust precompiles.""" diff --git a/script/fork/__main__.py b/script/fork/__main__.py new file mode 100644 index 0000000..41be127 --- /dev/null +++ b/script/fork/__main__.py @@ -0,0 +1,264 @@ +"""python -m fork [forge test args...] — run the base-std unit suite against a +local anvil that dispatches Base's Rust precompiles, validating the Solidity +reference against the live Rust impl from base/base. + +Both binaries (anvil + forge) come from the base-anvil fork of foundry-rs, which +adds a single `--base` flag to NetworkConfigs that installs the B-20 precompile +suite into the EVM. Stock foundry binaries will NOT work. + +Workflow: + 1. Launch anvil on $PORT with --base (registers Base precompiles). + 2. Fund + impersonate the activation admin, activate the gated features. + 3. Run forge --fork-url against anvil + the `fork` profile (which sets + base = true so forge's own EVM also dispatches to Base precompiles). + 4. Tear down anvil regardless of success / failure. + +Any extra arguments are forwarded to `forge test` (e.g. --match-contract, +--match-test, -vvvv). + +Env vars (with defaults): + ANVIL_BIN path to the patched anvil binary + (default: ../base-anvil/target/release/anvil, falling back to + debug if release is missing) + FORGE_BIN path to the patched forge binary (default: `forge` next to ANVIL_BIN) + PORT local RPC port for anvil (default: 8546) + ACTIVATION_ADMIN address authorized to activate features + (default: 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc, the + canonical local-dev admin) + ANVIL_LOG anvil stdout/stderr log path (default: /tmp/anvil.log) + SKIP_ACTIVATE comma-separated feature names or 0x ids to leave un-activated + (default: none, so every feature is activated). Use to exercise + the inactive-feature dispatch path, e.g. + SKIP_ACTIVATE=POLICY_REGISTRY to run the policy-registry + inactive-dispatch regression tests. Names and ids are matched + case-insensitively. + +Exit codes: + 0 forge test exit 0 (all targeted tests pass) + 1 forge test exit non-zero (at least one targeted test fails — the output is + the cross-validation signal) + 2 environment problem (missing binary, port in use, anvil failed to start, + activation tx failed) +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +from web3 import Web3 + +# Feature ids are derived (not hardcoded hex) from the smoke package's config, which keccaks the canonical +# feature names — the same values the Solidity ActivationRegistryFeatureList and the Rust storage.rs use. +# Reusing them deletes the hand-maintained hex table the bash runner carried, so there is one source of truth. +from smoke.config import ( + ACTIVATION_REGISTRY, + FEATURE_B20_ASSET, + FEATURE_B20_STABLECOIN, + FEATURE_POLICY_REGISTRY, +) + +# __main__.py -> fork -> script -> project root, where forge writes `out/` and base-anvil sits alongside. +REPO_ROOT = Path(__file__).resolve().parents[2] + +ENV_ERROR = 2 # environment-problem exit code (matches the bash contract). + +# Canonical (name, id) order, mirroring the FEATURE_IDS array the bash runner activated in turn. Names are +# the SKIP_ACTIVATE / log labels; ids gate each precompile via ActivationRegistry.activate(bytes32). +FEATURES: list[tuple[str, bytes]] = [ + ("B20_ASSET", bytes(FEATURE_B20_ASSET)), + ("POLICY_REGISTRY", bytes(FEATURE_POLICY_REGISTRY)), + ("B20_STABLECOIN", bytes(FEATURE_B20_STABLECOIN)), +] + +ACTIVATE_SELECTOR = bytes(Web3.keccak(text="activate(bytes32)")[:4]) + + +def log(msg: str) -> None: + print(f"[run-fork-tests] {msg}", file=sys.stderr) + + +def die(msg: str) -> "NoReturn": # noqa: F821 - NoReturn quoted to avoid a typing import + print(f"[run-fork-tests] ERROR: {msg}", file=sys.stderr) + raise SystemExit(ENV_ERROR) + + +# ── Binary discovery ──────────────────────────────────────────────────────────── + + +def _executable(path: Path) -> bool: + return path.is_file() and os.access(path, os.X_OK) + + +def discover_binaries() -> tuple[Path, Path]: + """Resolve (anvil, forge) from $ANVIL_BIN/$FORGE_BIN or the base-anvil default layout.""" + anvil_env = os.environ.get("ANVIL_BIN") + if anvil_env: + anvil = Path(anvil_env) + else: + release = REPO_ROOT / ".." / "base-anvil" / "target" / "release" / "anvil" + debug = REPO_ROOT / ".." / "base-anvil" / "target" / "debug" / "anvil" + if _executable(release): + anvil = release + elif _executable(debug): + anvil = debug + else: + die( + "anvil binary not found. Expected at:\n" + f" {release}\n {debug}\n" + "Build with: cd ../base-anvil && cargo build --release -p anvil -p forge\n" + "Or set ANVIL_BIN=/path/to/anvil." + ) + + forge = Path(os.environ.get("FORGE_BIN") or anvil.parent / "forge") + if not _executable(forge): + die( + f"patched forge binary not found at {forge}.\n" + f"Build with: cd {anvil.parent.parent.parent} && cargo build --release -p forge\n" + "Or set FORGE_BIN=/path/to/forge.\n" + "(System forge will NOT work — it lacks the --base injection. forge must come from " + "the base-anvil fork of foundry-rs.)" + ) + return anvil, forge + + +# ── SKIP_ACTIVATE parsing ──────────────────────────────────────────────────────── + + +def skip_set() -> set[str]: + """Uppercased SKIP_ACTIVATE entries (feature names or 0x ids), whitespace-stripped, empties dropped.""" + raw = os.environ.get("SKIP_ACTIVATE", "") + return {entry.strip().upper() for entry in raw.split(",") if entry.strip()} + + +def should_skip(name: str, fid: bytes, skip: set[str]) -> bool: + """True if a feature is named in SKIP_ACTIVATE by its canonical name or its raw 0x id (case-insensitive).""" + return name.upper() in skip or f"0X{fid.hex().upper()}" in skip + + +# ── Anvil lifecycle ────────────────────────────────────────────────────────────── + + +def assert_port_free(port: int) -> None: + """Die if something is already listening on the RPC port (mirrors the bash lsof guard).""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.5) + if sock.connect_ex(("127.0.0.1", port)) == 0: + die(f"port {port} is already in use. Set PORT= or kill the existing listener.") + + +@contextmanager +def anvil_running(anvil: Path, port: int, admin: str, log_path: Path) -> Iterator[Web3]: + """Launch anvil --base, yield a Web3 connected once the RPC is live, and always tear it down. + + Cleanup lives in the `finally`, so anvil is terminated whether the suite passes, fails, or raises — + the context-manager equivalent of the bash `trap ... EXIT`, plus a hard kill if it ignores SIGTERM. + """ + with open(log_path, "w") as logf: + proc = subprocess.Popen( + [str(anvil), "--base", "--base-activation-admin", admin, "--port", str(port)], + stdout=logf, + stderr=subprocess.STDOUT, + ) + try: + w3 = Web3(Web3.HTTPProvider(f"http://localhost:{port}")) + for _ in range(20): # poll for the RPC port to come up (up to 10s) + if proc.poll() is not None: + tail = "".join(log_path.read_text().splitlines(keepends=True)[-20:]) + die(f"anvil exited during startup; see {log_path}\n--- last 20 lines of {log_path} ---\n{tail}") + try: + if w3.eth.chain_id: + break + except Exception: # noqa: BLE001 - RPC not up yet; keep polling + pass + time.sleep(0.5) + else: + die(f"anvil did not answer RPC within 10s; see {log_path}") + log(f"anvil up (pid={proc.pid})") + yield w3 + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + + +# ── Activation ─────────────────────────────────────────────────────────────────── + + +def activate_features(w3: Web3, admin: str, skip: set[str]) -> None: + """Fund + impersonate the admin, then activate each gated feature not in SKIP_ACTIVATE. + + Anvil's impersonation lets us send activate() calls from the admin without its private key (real-chain + forks would substitute a funded signer). Each tx is awaited and its receipt status checked — the + structured equivalent of the bash `cast send | grep '^status'`. + """ + log("funding + impersonating activation admin…") + w3.provider.make_request("anvil_setBalance", [admin, hex(2**64 - 1)]) + w3.provider.make_request("anvil_impersonateAccount", [admin]) + + for name, fid in FEATURES: + if should_skip(name, fid, skip): + log(f"leaving feature un-activated: {name} 0x{fid.hex()} [SKIP_ACTIVATE]") + continue + log(f"activating feature {name} 0x{fid.hex()}") + data = Web3.to_hex(ACTIVATE_SELECTOR + fid) + try: + tx_hash = w3.eth.send_transaction({"from": admin, "to": ACTIVATION_REGISTRY, "data": data}) + receipt = w3.eth.wait_for_transaction_receipt(tx_hash, timeout=30) + except Exception as exc: # noqa: BLE001 - any RPC/tx failure is an environment problem + die(f"activation tx failed for {name} 0x{fid.hex()}: {type(exc).__name__}: {exc}") + if receipt["status"] != 1: + die(f"activation tx reverted for {name} 0x{fid.hex()} (status {receipt['status']})") + + +# ── Orchestration ──────────────────────────────────────────────────────────────── + + +def main(forge_args: list[str]) -> int: + port = int(os.environ.get("PORT", "8546")) + admin = Web3.to_checksum_address( + os.environ.get("ACTIVATION_ADMIN", "0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc") + ) + log_path = Path(os.environ.get("ANVIL_LOG", "/tmp/anvil.log")) + skip = skip_set() + + anvil, forge = discover_binaries() + assert_port_free(port) + + log(f"anvil: {anvil}") + log(f"forge: {forge}") + log(f"port: {port}") + log(f"activation admin: {admin}") + log(f"log file: {log_path}") + log(f"skip-activate: {os.environ.get('SKIP_ACTIVATE') or ''}") + + log("starting anvil…") + with anvil_running(anvil, port, admin, log_path) as w3: + activate_features(w3, admin, skip) + + rpc_url = f"http://localhost:{port}" + log(f"running forge test --fork-url {rpc_url} {' '.join(forge_args)}") + # LIVE_PRECOMPILES skips BaseTest's vm.etch of the mocks at the precompile addresses (so calls + # dispatch to the real Rust impls). FOUNDRY_PROFILE=fork enables [profile.fork] base=true (so + # forge's EVM installs the Base precompile set). + env = {**os.environ, "LIVE_PRECOMPILES": "true", "FOUNDRY_PROFILE": "fork"} + result = subprocess.run( + [str(forge), "test", "--fork-url", rpc_url, *forge_args], + cwd=REPO_ROOT, + env=env, + ) + + log(f"forge test exited {result.returncode}") + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/script/run-fork-tests.sh b/script/run-fork-tests.sh deleted file mode 100755 index 0ab7c08..0000000 --- a/script/run-fork-tests.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash -# run-fork-tests.sh — run the base-std unit suite against a local anvil that -# dispatches Base's Rust precompiles, validating the Solidity reference against -# the live Rust impl from base/base. -# -# Both binaries (anvil + forge) come from the base-anvil fork of foundry-rs, -# which adds a single `--base` flag to NetworkConfigs that installs the B-20 -# precompile suite into the EVM. Stock foundry binaries will NOT work. -# -# Workflow: -# 1. Launch anvil on $PORT with --base (registers Base precompiles). -# 2. Fund + impersonate the activation admin, activate the gated features. -# 3. Run forge --fork-url against anvil + the `fork` profile (which sets -# base = true so forge's own EVM also dispatches to Base precompiles). -# 4. Tear down anvil regardless of success / failure. -# -# Any extra arguments to this script are forwarded to `forge test`. Use them -# to scope the run (e.g. --match-contract, --match-test, -vvvv). -# -# Env vars (with defaults): -# ANVIL_BIN path to the patched anvil binary -# (default: ../base-anvil/target/release/anvil, falling -# back to debug if release is missing) -# FORGE_BIN path to the patched forge binary -# (default: `forge` next to ANVIL_BIN) -# PORT local RPC port for anvil (default: 8546) -# ACTIVATION_ADMIN address authorized to activate features -# (default: 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc, the -# canonical local-dev admin) -# ANVIL_LOG anvil stdout/stderr log path (default: /tmp/anvil.log) -# SKIP_ACTIVATE comma-separated feature names or 0x ids to leave -# un-activated (default: none, so every feature is activated). -# Use to exercise the inactive-feature dispatch path, e.g. -# SKIP_ACTIVATE=POLICY_REGISTRY to run the policy-registry -# inactive-dispatch regression tests. Names and ids are -# matched case-insensitively. -# -# Exit codes: -# 0 forge test exit 0 (all targeted tests pass) -# 1 forge test exit non-zero (at least one targeted test fails — the -# output is the cross-validation signal) -# 2 environment problem (missing binary, port in use, anvil failed to -# start, activation tx failed) - -set -euo pipefail - -# ── Layout ──────────────────────────────────────────────────────────────────── - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DEFAULT_ANVIL_RELEASE="$REPO_ROOT/../base-anvil/target/release/anvil" -DEFAULT_ANVIL_DEBUG="$REPO_ROOT/../base-anvil/target/debug/anvil" - -if [[ -z "${ANVIL_BIN:-}" ]]; then - if [[ -x "$DEFAULT_ANVIL_RELEASE" ]]; then - ANVIL_BIN="$DEFAULT_ANVIL_RELEASE" - elif [[ -x "$DEFAULT_ANVIL_DEBUG" ]]; then - ANVIL_BIN="$DEFAULT_ANVIL_DEBUG" - else - echo "ERROR: anvil binary not found. Expected at:" >&2 - echo " $DEFAULT_ANVIL_RELEASE" >&2 - echo " $DEFAULT_ANVIL_DEBUG" >&2 - echo "Build with: cd ../base-anvil && cargo build --release -p anvil -p forge" >&2 - echo "Or set ANVIL_BIN=/path/to/anvil." >&2 - exit 2 - fi -fi - -if [[ -z "${FORGE_BIN:-}" ]]; then - FORGE_BIN="$(dirname "$ANVIL_BIN")/forge" - if [[ ! -x "$FORGE_BIN" ]]; then - echo "ERROR: patched forge binary not found at $FORGE_BIN." >&2 - echo "Build with: cd $(dirname "$ANVIL_BIN")/../.. && cargo build --release -p forge" >&2 - echo "Or set FORGE_BIN=/path/to/forge." >&2 - echo "(System forge will NOT work — it lacks the --base injection." >&2 - echo " forge must come from the base-anvil fork of foundry-rs.)" >&2 - exit 2 - fi -fi - -PORT="${PORT:-8546}" -ACTIVATION_ADMIN="${ACTIVATION_ADMIN:-0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc}" -REGISTRY=0x8453000000000000000000000000000000000001 -LOG_FILE="${ANVIL_LOG:-/tmp/anvil.log}" - -# Feature IDs mirror the canonical set in test/lib/mocks/ActivationRegistryFeatureList.sol -# (the Solidity reference is the source of truth). If a feature is added there, append its ID here. -FEATURE_IDS=( - 0xcdcc772fe4cbdb1029f822861176d09e646db96723d4c1e82ddfdeb8163ef54c # B20_ASSET - 0xb582ebae03f16fee49a6763f78df482fb11ae73f103ed0d330bbe556aa90a43f # POLICY_REGISTRY - 0xecfa0def2c10020caaf65e6155aa69c84b24892aaef76eeac52e0e2b3a0b8601 # B20_STABLECOIN -) - -# Optional set of features to leave UN-activated (see header). Matched against -# both the canonical name and the raw id, case-insensitively. Default empty, so -# the standard cross-validation run activates everything as before. -SKIP_ACTIVATE="${SKIP_ACTIVATE:-}" - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -log() { echo "[run-fork-tests] $*" >&2; } -die() { echo "[run-fork-tests] ERROR: $*" >&2; exit 2; } - -# Canonical name for a feature id (mirrors the comments on FEATURE_IDS and the -# Solidity ActivationRegistryFeatureList). Empty string for an unknown id. -feature_name() { - case "$1" in - 0xcdcc772fe4cbdb1029f822861176d09e646db96723d4c1e82ddfdeb8163ef54c) echo B20_ASSET ;; - 0xb582ebae03f16fee49a6763f78df482fb11ae73f103ed0d330bbe556aa90a43f) echo POLICY_REGISTRY ;; - 0xecfa0def2c10020caaf65e6155aa69c84b24892aaef76eeac52e0e2b3a0b8601) echo B20_STABLECOIN ;; - *) echo "" ;; - esac -} - -# Returns 0 (skip) if feature id $1 is named in SKIP_ACTIVATE, by either its -# canonical name or its raw id. Case-insensitive; whitespace-tolerant. -should_skip_activate() { - [[ -z "$SKIP_ACTIVATE" ]] && return 1 - local fid="$1" name id_uc entry - name="$(feature_name "$fid")" - id_uc="$(printf '%s' "$fid" | tr '[:lower:]' '[:upper:]')" - local IFS=',' - for entry in $SKIP_ACTIVATE; do - entry="$(printf '%s' "$entry" | tr -d '[:space:]' | tr '[:lower:]' '[:upper:]')" - [[ -z "$entry" ]] && continue - if [[ "$entry" == "$name" || "$entry" == "$id_uc" ]]; then - return 0 - fi - done - return 1 -} - -rpc() { - local method="$1"; shift - local params="$1"; shift - curl -s -X POST -H "Content-Type: application/json" \ - --data "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":$params,\"id\":1}" \ - "http://localhost:$PORT" -} - -# ── Pre-flight ──────────────────────────────────────────────────────────────── - -command -v cast >/dev/null 2>&1 || die "cast not found (install foundry: https://getfoundry.sh)" -command -v curl >/dev/null 2>&1 || die "curl not found" - -if lsof -i ":$PORT" >/dev/null 2>&1; then - die "port $PORT is already in use. Set PORT= or kill the existing listener." -fi - -log "anvil: $ANVIL_BIN" -log "forge: $FORGE_BIN" -log "port: $PORT" -log "activation admin: $ACTIVATION_ADMIN" -log "log file: $LOG_FILE" -log "skip-activate: ${SKIP_ACTIVATE:-}" - -# ── Launch anvil ────────────────────────────────────────────────────────────── - -log "starting anvil…" -"$ANVIL_BIN" --base --base-activation-admin "$ACTIVATION_ADMIN" --port "$PORT" \ - > "$LOG_FILE" 2>&1 & -ANVIL_PID=$! -trap 'kill $ANVIL_PID 2>/dev/null; wait $ANVIL_PID 2>/dev/null; true' EXIT - -# Poll for the RPC port to come up (up to 10s). -for i in $(seq 1 20); do - if rpc eth_chainId '[]' 2>/dev/null | grep -q '"result"'; then - break - fi - sleep 0.5 - if ! kill -0 $ANVIL_PID 2>/dev/null; then - echo "--- last 20 lines of $LOG_FILE ---" >&2 - tail -20 "$LOG_FILE" >&2 - die "anvil exited during startup; see $LOG_FILE" - fi -done -log "anvil up (pid=$ANVIL_PID)" - -# ── Activate features ──────────────────────────────────────────────────────── -# Anvil's --unlocked + anvil_impersonateAccount lets us send activate() calls -# from the admin address without needing its private key. Real-chain forks -# would substitute --private-key + a funded signer. - -log "funding + impersonating activation admin…" -rpc anvil_setBalance "[\"$ACTIVATION_ADMIN\", \"0xffffffffffffffff\"]" > /dev/null -rpc anvil_impersonateAccount "[\"$ACTIVATION_ADMIN\"]" > /dev/null - -for fid in "${FEATURE_IDS[@]}"; do - if should_skip_activate "$fid"; then - log "leaving feature un-activated: $(feature_name "$fid") $fid [SKIP_ACTIVATE]" - continue - fi - log "activating feature $fid" - out=$(cast send --rpc-url "http://localhost:$PORT" --from "$ACTIVATION_ADMIN" \ - --unlocked "$REGISTRY" "activate(bytes32)" "$fid" 2>&1) || \ - die "activation tx failed for $fid:\n$out" - # status==1 line confirms inclusion + success - echo "$out" | grep -E "^status\b" | head -1 >&2 || die "no status in cast send output for $fid" -done - -# ── Run the test suite ──────────────────────────────────────────────────────── - -log "running forge test --fork-url http://localhost:$PORT $*" -cd "$REPO_ROOT" - -# LIVE_PRECOMPILES skips BaseTest's vm.etch of the mocks at the precompile -# addresses (so calls dispatch to the real Rust impls). FOUNDRY_PROFILE=fork -# enables the [profile.fork] base=true setting (so forge's EVM installs the -# Base precompile set). -LIVE_PRECOMPILES=true FOUNDRY_PROFILE=fork \ - "$FORGE_BIN" test --fork-url "http://localhost:$PORT" "$@" -forge_exit=$? - -log "forge test exited $forge_exit" -exit $forge_exit From 5ff186fa62345d30be114380c8ac6b54c06568ee Mon Sep 17 00:00:00 2001 From: katzman Date: Thu, 11 Jun 2026 11:29:34 -0700 Subject: [PATCH 2/2] cleanup --- .github/workflows/base-std-fork-tests.yml | 8 ++- FORK_TESTING.md | 6 +- Makefile | 28 ++++++++- script/fork/README.md | 2 + script/fork/__main__.py | 73 ++++------------------- script/smoke/README.md | 2 + 6 files changed, 51 insertions(+), 68 deletions(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index 0b7a7cb..6566050 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -122,10 +122,14 @@ jobs: "$BASE_ANVIL_DIR/target/release/anvil" --version "$BASE_ANVIL_DIR/target/release/forge" --version + - name: Set up Python 3.13 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + - name: Set up fork-test runner venv shell: bash - # The runner is `python -m fork` (web3 only); reuse the smoke venv target with the - # runner's preinstalled python3 instead of the python3.13 default. + # setup-python makes python3 == 3.13; make python-check enforces it. run: make smoke-setup PYTHON=python3 - name: Run base-std fork tests diff --git a/FORK_TESTING.md b/FORK_TESTING.md index 172bcf7..4bcf1e7 100644 --- a/FORK_TESTING.md +++ b/FORK_TESTING.md @@ -67,8 +67,10 @@ brew install lld # macOS; Linux uses mold per base-anvil's .cargo/config.toml curl -L https://foundry.paradigm.xyz | bash && foundryup # stock foundry, for `cast` (the manual probe below) ``` -The runner itself is Python ([`script/fork/`](script/fork/)), driven by `web3`. -Create its venv once (shared with the smoke suite): +The runner itself is Python ([`script/fork/`](script/fork/)), driven by `web3` +and requiring **Python 3.13**. Create its venv once (shared with the smoke +suite; `make smoke-setup` checks the version and prints install guidance if it's +missing): ```bash make smoke-setup diff --git a/Makefile b/Makefile index f95acaf..5ece7a1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ # Source the gitignored .env for the smoke recipes. LOAD_ENV = pre=$$(export -p); set -a; [ -f .env ] && . ./.env; set +a; eval "$$pre"; +# The smoke + fork runners require Python 3.13 (the version the shared venv is built and CI-tested +# against). `make smoke-setup` enforces this via `python-check` before creating the venv. Override the +# interpreter with `make smoke-setup PYTHON=/path/to/python3.13` if 3.13 isn't on your PATH as below. +REQUIRED_PYTHON = 3.13 PYTHON ?= python3.13 VENV = script/smoke/.venv # `smoke` and `fork` are packages under script/, so script/ is on the path. Both share the one venv @@ -8,7 +12,7 @@ VENV = script/smoke/.venv SMOKE_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m smoke FORK_RUN = $(LOAD_ENV) PYTHONPATH=script $(VENV)/bin/python -m fork -.PHONY: build coverage smoke smoke-all smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants smoke-setup fork-tests +.PHONY: build coverage smoke smoke-all smoke-factory smoke-asset smoke-stablecoin smoke-policy smoke-invariants python-check smoke-setup fork-tests # Generate an lcov coverage report and open it in the browser. # Scoped to src/ and test/lib/mocks/ (excludes test runner files and the smoke probe helper). @@ -18,8 +22,26 @@ coverage: open coverage/index.html -# One-time setup: create the smoketest venv and install web3. -smoke-setup: +# Verify the interpreter for the venv exists and is the required version. Runs before smoke-setup; +# prints install guidance and fails fast if Python $(REQUIRED_PYTHON) isn't available as $(PYTHON). +python-check: + @command -v $(PYTHON) >/dev/null 2>&1 || { \ + echo "ERROR: '$(PYTHON)' not found. The smoke + fork runners require Python $(REQUIRED_PYTHON)."; \ + echo "Install it, then re-run (or pass PYTHON=/path/to/python$(REQUIRED_PYTHON)):"; \ + echo " pyenv: pyenv install $(REQUIRED_PYTHON) && pyenv local $(REQUIRED_PYTHON)"; \ + echo " macOS: brew install python@$(REQUIRED_PYTHON)"; \ + echo " Debian: sudo add-apt-repository ppa:deadsnakes/ppa && sudo apt-get install -y python$(REQUIRED_PYTHON) python$(REQUIRED_PYTHON)-venv"; \ + exit 1; \ + } + @$(PYTHON) -c 'import sys; req=tuple(int(x) for x in "$(REQUIRED_PYTHON)".split(".")); \ + sys.exit(0) if sys.version_info[:len(req)] == req else \ + sys.exit("ERROR: %s is Python %d.%d.%d, but this project requires Python $(REQUIRED_PYTHON).x. " \ + "Install it or pass PYTHON=/path/to/python$(REQUIRED_PYTHON)." \ + % ("$(PYTHON)", sys.version_info.major, sys.version_info.minor, sys.version_info.micro))' + @echo "python-check: $(PYTHON) is $$($(PYTHON) --version 2>&1 | cut -d' ' -f2) (need $(REQUIRED_PYTHON).x) — ok" + +# One-time setup: verify Python $(REQUIRED_PYTHON), then create the smoke+fork venv and install web3. +smoke-setup: python-check $(PYTHON) -m venv $(VENV) $(VENV)/bin/python -m pip install --upgrade pip $(VENV)/bin/python -m pip install -r script/smoke/requirements.txt diff --git a/script/fork/README.md b/script/fork/README.md index 3e5b556..cb1eed0 100644 --- a/script/fork/README.md +++ b/script/fork/README.md @@ -29,6 +29,8 @@ hex table to drift from the Solidity `ActivationRegistryFeatureList` / Rust ## Running +Requires **Python 3.13** (`make smoke-setup` enforces it; override with `PYTHON=`). + ```bash make smoke-setup # one-time: create the shared venv + install web3 (shared with `make smoke`) diff --git a/script/fork/__main__.py b/script/fork/__main__.py index 41be127..512f566 100644 --- a/script/fork/__main__.py +++ b/script/fork/__main__.py @@ -1,44 +1,9 @@ """python -m fork [forge test args...] — run the base-std unit suite against a -local anvil that dispatches Base's Rust precompiles, validating the Solidity -reference against the live Rust impl from base/base. - -Both binaries (anvil + forge) come from the base-anvil fork of foundry-rs, which -adds a single `--base` flag to NetworkConfigs that installs the B-20 precompile -suite into the EVM. Stock foundry binaries will NOT work. - -Workflow: - 1. Launch anvil on $PORT with --base (registers Base precompiles). - 2. Fund + impersonate the activation admin, activate the gated features. - 3. Run forge --fork-url against anvil + the `fork` profile (which sets - base = true so forge's own EVM also dispatches to Base precompiles). - 4. Tear down anvil regardless of success / failure. - -Any extra arguments are forwarded to `forge test` (e.g. --match-contract, ---match-test, -vvvv). - -Env vars (with defaults): - ANVIL_BIN path to the patched anvil binary - (default: ../base-anvil/target/release/anvil, falling back to - debug if release is missing) - FORGE_BIN path to the patched forge binary (default: `forge` next to ANVIL_BIN) - PORT local RPC port for anvil (default: 8546) - ACTIVATION_ADMIN address authorized to activate features - (default: 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc, the - canonical local-dev admin) - ANVIL_LOG anvil stdout/stderr log path (default: /tmp/anvil.log) - SKIP_ACTIVATE comma-separated feature names or 0x ids to leave un-activated - (default: none, so every feature is activated). Use to exercise - the inactive-feature dispatch path, e.g. - SKIP_ACTIVATE=POLICY_REGISTRY to run the policy-registry - inactive-dispatch regression tests. Names and ids are matched - case-insensitively. - -Exit codes: - 0 forge test exit 0 (all targeted tests pass) - 1 forge test exit non-zero (at least one targeted test fails — the output is - the cross-validation signal) - 2 environment problem (missing binary, port in use, anvil failed to start, - activation tx failed) +local anvil that dispatches Base's Rust precompiles, cross-validating the +Solidity reference against the live Rust impl. + +Requires the patched anvil + forge from the base-anvil fork. Env vars, exit +codes, and the full workflow are documented in README.md. """ from __future__ import annotations @@ -54,9 +19,7 @@ from web3 import Web3 -# Feature ids are derived (not hardcoded hex) from the smoke package's config, which keccaks the canonical -# feature names — the same values the Solidity ActivationRegistryFeatureList and the Rust storage.rs use. -# Reusing them deletes the hand-maintained hex table the bash runner carried, so there is one source of truth. +# Feature ids: reuse the smoke package's derived keccak constants (one source of truth). from smoke.config import ( ACTIVATION_REGISTRY, FEATURE_B20_ASSET, @@ -64,13 +27,12 @@ FEATURE_POLICY_REGISTRY, ) -# __main__.py -> fork -> script -> project root, where forge writes `out/` and base-anvil sits alongside. +# __main__.py -> fork -> script -> project root. REPO_ROOT = Path(__file__).resolve().parents[2] -ENV_ERROR = 2 # environment-problem exit code (matches the bash contract). +ENV_ERROR = 2 # environment-problem exit code. -# Canonical (name, id) order, mirroring the FEATURE_IDS array the bash runner activated in turn. Names are -# the SKIP_ACTIVATE / log labels; ids gate each precompile via ActivationRegistry.activate(bytes32). +# (name, id) in activation order; names are the SKIP_ACTIVATE / log labels. FEATURES: list[tuple[str, bytes]] = [ ("B20_ASSET", bytes(FEATURE_B20_ASSET)), ("POLICY_REGISTRY", bytes(FEATURE_POLICY_REGISTRY)), @@ -155,11 +117,7 @@ def assert_port_free(port: int) -> None: @contextmanager def anvil_running(anvil: Path, port: int, admin: str, log_path: Path) -> Iterator[Web3]: - """Launch anvil --base, yield a Web3 connected once the RPC is live, and always tear it down. - - Cleanup lives in the `finally`, so anvil is terminated whether the suite passes, fails, or raises — - the context-manager equivalent of the bash `trap ... EXIT`, plus a hard kill if it ignores SIGTERM. - """ + """Launch anvil --base, yield a Web3 once the RPC is live, and tear it down on exit (any outcome).""" with open(log_path, "w") as logf: proc = subprocess.Popen( [str(anvil), "--base", "--base-activation-admin", admin, "--port", str(port)], @@ -194,12 +152,7 @@ def anvil_running(anvil: Path, port: int, admin: str, log_path: Path) -> Iterato def activate_features(w3: Web3, admin: str, skip: set[str]) -> None: - """Fund + impersonate the admin, then activate each gated feature not in SKIP_ACTIVATE. - - Anvil's impersonation lets us send activate() calls from the admin without its private key (real-chain - forks would substitute a funded signer). Each tx is awaited and its receipt status checked — the - structured equivalent of the bash `cast send | grep '^status'`. - """ + """Fund + impersonate the admin, then activate each gated feature not in SKIP_ACTIVATE.""" log("funding + impersonating activation admin…") w3.provider.make_request("anvil_setBalance", [admin, hex(2**64 - 1)]) w3.provider.make_request("anvil_impersonateAccount", [admin]) @@ -246,9 +199,7 @@ def main(forge_args: list[str]) -> int: rpc_url = f"http://localhost:{port}" log(f"running forge test --fork-url {rpc_url} {' '.join(forge_args)}") - # LIVE_PRECOMPILES skips BaseTest's vm.etch of the mocks at the precompile addresses (so calls - # dispatch to the real Rust impls). FOUNDRY_PROFILE=fork enables [profile.fork] base=true (so - # forge's EVM installs the Base precompile set). + # LIVE_PRECOMPILES: skip the mock etch; fork profile: base=true installs the precompiles. env = {**os.environ, "LIVE_PRECOMPILES": "true", "FOUNDRY_PROFILE": "fork"} result = subprocess.run( [str(forge), "test", "--fork-url", rpc_url, *forge_args], diff --git a/script/smoke/README.md b/script/smoke/README.md index f0138ba..d7d7d86 100644 --- a/script/smoke/README.md +++ b/script/smoke/README.md @@ -25,6 +25,8 @@ anyone for you: you supply the endpoint and two funded keys. ## Running +Requires **Python 3.13** (`make smoke-setup` enforces it; override with `PYTHON=`). + ```bash make smoke-setup # one-time: create the venv + install web3 cp .env.template .env # then set RPC_URL, DEPLOYER_PK, USER2_PK