From fac8bda05241d08c29f598a6d5295ecc7243f57d Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 17:05:51 +0800 Subject: [PATCH 01/14] (check): simplify node matrix to 3 concurrent cert-free chains Rework .ci/ci_check.sh from 5 sequential build-test-teardown rounds (v3.2.6 auth / v3.7.3 wasm / v3.7.3 ecdsa / v3.7.3 sm / v3.11.0) to 3 chains started once, concurrently, on disjoint ports: - v3.7.3 ecdsa (p2p 30300, rpc 20200) pinned compatibility - latest release ecdsa (p2p 30310, rpc 20210) - latest release sm (p2p 30320, rpc 20220) The latest tag is auto-resolved from the GitHub releases/latest redirect (guaranteeing binary assets exist; the newest git tag may have no release), with a gitee fallback; build_chain.sh falls back to the source tree for releases that no longer attach it as an asset. WASM testing is dropped and SM is only tested on the latest version. All chains disable SSL on the RPC endpoint (';disable_ssl=true' -> 'disable_ssl=true' on <=3.7.x, 'enable_ssl=true' -> 'enable_ssl=false' on newer templates; the [p2p] enable_ssl_verify key is unaffected), so the SDK connects certificate-free: the rendered config.toml sets enableSsl=false and no node certs are copied into conf/. integrationTest then runs three times, once per chain (useSMCrypto toggled for the SM round). Verified locally (macOS, both binaries via Rosetta): all three chains healthy with plain-HTTP JSON-RPC responding, and a real SDK integration test class ran against each chain (36/36 executed, 0 failures, 0 skips) with no certificates configured. --- .ci/ci_check.sh | 278 +++++++++++++++++++++++++++--------------------- AGENTS.md | 21 ++-- 2 files changed, 168 insertions(+), 131 deletions(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index 0c0623b66..d2befbe96 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -5,13 +5,6 @@ LOG_INFO() { local content=${1} echo -e "\033[32m ${content}\033[0m" } -check_basic() -{ -# check code format -# bash gradlew verifyGoogleJavaFormat -# build -bash gradlew build --info -} download_tassl() { @@ -26,136 +19,162 @@ fi curl -LO "https://github.com/FISCO-BCOS/LargeFiles/raw/master/tools/${package_name}.tar.gz" && tar -zxvf "${package_name}.tar.gz" && mv "${package_name}" tassl-1.1.1b && mkdir -p ~/.fisco && mv tassl-1.1.1b ~/.fisco/ } -download_build_chain() +get_sed_cmd() { - local tag="${1}" + local sed_cmd="sed -i" + if [ "$(uname)" == "Darwin" ];then + sed_cmd="sed -i .bkp" + fi + echo "$sed_cmd" +} + +get_latest_version() +{ + # The GitHub /releases/latest redirect is authoritative for a release that + # actually has downloadable binary assets (the newest git tag may not have + # a published release yet). Fall back to the gitee tag list if unreachable. + local tag + tag=$(curl -fsSI "https://github.com/FISCO-BCOS/FISCO-BCOS/releases/latest" | grep -i "^location:" | grep -oe "v[0-9]*\.[0-9]*\.[0-9]*" | head -n 1) if [ -z "${tag}" ]; then tag=$(curl -sS "https://gitee.com/api/v5/repos/FISCO-BCOS/FISCO-BCOS/tags" | grep -oe "\"name\":\"v[2-9]*\.[0-9]*\.[0-9]*\"" | cut -d \" -f 4 | sort -V | tail -n 1) fi - LOG_INFO "--- current tag: $tag" - curl -LO "https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/${tag}/build_chain.sh" && chmod u+x build_chain.sh + echo "${tag}" +} + +download_build_chain() +{ + local tag="${1}" + LOG_INFO "--- download build_chain.sh: ${tag} ---" + # older releases attach build_chain.sh as a release asset; newer releases + # only keep it in the source tree, so fall back to the raw file at that tag + curl -fsSL -o "build_chain-${tag}.sh" "https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/${tag}/build_chain.sh" \ + || curl -fsSL -o "build_chain-${tag}.sh" "https://raw.githubusercontent.com/FISCO-BCOS/FISCO-BCOS/${tag}/tools/BcosAirBuilder/build_chain.sh" + chmod u+x "build_chain-${tag}.sh" } download_binary() { local tag="${1}" - LOG_INFO "--- current tag: $tag" + LOG_INFO "--- download fisco-bcos binary: ${tag} ---" local package_name="fisco-bcos-linux-x86_64.tar.gz" if [ "$(uname)" == "Darwin" ];then package_name="fisco-bcos-macOS-x86_64.tar.gz" fi - curl -LO "https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/${tag}/${package_name}" && tar -zxvf "${package_name}" + curl -fsSLO "https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/${tag}/${package_name}" + # keep the binary named exactly 'fisco-bcos' (the generated start.sh expects + # that name), one sub-directory per version + mkdir -p "bins/${tag}" + tar -zxf "${package_name}" -C "bins/${tag}" && rm -f "${package_name}" + "./bins/${tag}/fisco-bcos" -v } -get_sed_cmd() +# build one 4-node air chain with a certificate-free (ssl disabled) rpc +# endpoint, then start it +build_chain_one() { - local sed_cmd="sed -i" - if [ "$(uname)" == "Darwin" ];then - sed_cmd="sed -i .bkp" - fi - echo "$sed_cmd" + local tag="${1}" + local outdir="${2}" + local ports="${3}" # "p2p_start_port,rpc_start_port" + local extra="${4}" # e.g. "-s" for a sm-crypto chain + LOG_INFO "--- build chain: version=${tag} outdir=${outdir} ports=${ports} ${extra} ---" + bash "build_chain-${tag}.sh" -l 127.0.0.1:4 -e "./bins/${tag}/fisco-bcos" -o "${outdir}" -p "${ports}" ${extra} + # turn off ssl on the rpc endpoint; the config key differs between versions: + # <= 3.7.x ships a commented ';disable_ssl=true' in [rpc] + # >= 3.16.x ships an explicit 'enable_ssl=true' in [rpc] + # ([p2p] uses a distinct 'enable_ssl_verify' key, so the sed cannot touch it) + local sed_cmd=$(get_sed_cmd) + local cfg + for cfg in "${outdir}"/127.0.0.1/node*/config.ini; do + ${sed_cmd} "s/;disable_ssl=true/disable_ssl=true/" "${cfg}" + ${sed_cmd} "s/enable_ssl=true/enable_ssl=false/" "${cfg}" + done + cat "${outdir}/127.0.0.1/node0/config.genesis" + bash "${outdir}/127.0.0.1/start_all.sh" } -prepare_environment() +wait_rpc_ready() { - ## prepare resources for integration test - mkdir -p src/integration-test/resources/ - mkdir -p conf - cp -r nodes/127.0.0.1/sdk/* conf - cp src/test/resources/config-example.toml src/integration-test/resources/config.toml - cp src/test/resources/clog.ini conf/ - cp src/test/resources/config-example.toml src/test/resources/config.toml - cp src/test/resources/log4j2.properties src/integration-test/resources/ - rm -rf src/integration-test/resources/abi - rm -rf src/integration-test/resources/bin - cp -r src/test/resources/ecdsa/abi src/integration-test/resources/abi - cp -r src/test/resources/ecdsa/bin src/integration-test/resources/bin - - sed_cmd=$(get_sed_cmd) - local node_type="${1}" - local use_sm="false" - local not_use_sm="true" - if [ "${node_type}" == "sm" ];then - use_sm="true" - not_use_sm="false" - rm -rf src/integration-test/resources/abi - rm -rf src/integration-test/resources/bin - cp -r src/test/resources/gm/abi src/integration-test/resources/abi - cp -r src/test/resources/gm/bin src/integration-test/resources/bin - fi - use_sm_str="useSMCrypto = \"${use_sm}\"" - ${sed_cmd} "s/useSMCrypto = \"${not_use_sm}\"/${use_sm_str}/g" ./src/integration-test/resources/config.toml - ${sed_cmd} "s/useSMCrypto = \"${not_use_sm}\"/${use_sm_str}/g" ./src/integration-test/resources/amop/config-subscriber-for-test.toml - ${sed_cmd} "s/useSMCrypto = \"${not_use_sm}\"/${use_sm_str}/g" ./src/integration-test/resources/amop/config-publisher-for-test.toml + local port="${1}" + local i + for i in $(seq 1 30); do + if curl -s -m 3 --noproxy "*" -o /dev/null "http://127.0.0.1:${port}"; then + LOG_INFO "--- rpc 127.0.0.1:${port} is ready ---" + return 0 + fi + sleep 2 + done + echo "rpc 127.0.0.1:${port} not ready after 60s" + return 1 } -prepare_wasm_environment() +# render the sdk configs for one round: no certificates (ssl disabled), +# pointed at the given chain's rpc ports +prepare_sdk_config() { - ## prepare resources for integration test - mkdir -p src/integration-wasm-test/resources/ - mkdir -p conf - cp -r nodes/127.0.0.1/sdk/* conf - cp src/test/resources/config-example.toml src/integration-wasm-test/resources/config.toml + local rpc_port="${1}" + local use_sm="${2}" # "true" / "false" + local sed_cmd=$(get_sed_cmd) + + mkdir -p src/integration-test/resources/ conf cp src/test/resources/clog.ini conf/ cp src/test/resources/config-example.toml src/test/resources/config.toml - cp src/test/resources/log4j2.properties src/integration-wasm-test/resources/ -} + cp src/test/resources/config-example.toml src/integration-test/resources/config.toml + cp src/test/resources/log4j2.properties src/integration-test/resources/ -build_node() -{ - local node_type="${1}" - local sed_cmd=$(get_sed_cmd) - if [ ! -f "get_account.sh" ];then - curl -LO https://raw.githubusercontent.com/FISCO-BCOS/console/master/tools/get_account.sh - fi - if [ ! -f "get_gm_account.sh" ];then - curl -LO https://raw.githubusercontent.com/FISCO-BCOS/console/master/tools/get_gm_account.sh - fi - if [ "${node_type}" == "wasm" ];then - bash build_chain.sh -l 127.0.0.1:4 -e ./fisco-bcos -w ${2} + rm -rf src/integration-test/resources/abi src/integration-test/resources/bin + if [ "${use_sm}" == "true" ];then + cp -r src/test/resources/gm/abi src/integration-test/resources/abi + cp -r src/test/resources/gm/bin src/integration-test/resources/bin else - bash build_chain.sh -l 127.0.0.1:4 -e ./fisco-bcos ${2} + cp -r src/test/resources/ecdsa/abi src/integration-test/resources/abi + cp -r src/test/resources/ecdsa/bin src/integration-test/resources/bin fi - ./nodes/127.0.0.1/fisco-bcos -v - cat nodes/127.0.0.1/node0/config.genesis - bash nodes/127.0.0.1/start_all.sh + + ${sed_cmd} "s/enableSsl = \"true\"/enableSsl = \"false\"/" ./src/integration-test/resources/config.toml + ${sed_cmd} "s/useSMCrypto = \"false\"/useSMCrypto = \"${use_sm}\"/" ./src/integration-test/resources/config.toml + ${sed_cmd} "s/127.0.0.1:20201/127.0.0.1:${rpc_port}/g" ./src/integration-test/resources/config.toml + + # amop test configs: restore from a pristine template each round, then point + # them at this round's chain; certPath is replaced by enableSsl=false since + # no certificates are needed + local f + for f in config-publisher-for-test.toml config-subscriber-for-test.toml; do + local p="src/integration-test/resources/amop/${f}" + if [ ! -f "${p}.tpl" ];then + cp "${p}" "${p}.tpl" + fi + cp "${p}.tpl" "${p}" + ${sed_cmd} "s/certPath = \"conf\"/enableSsl = \"false\"/" "${p}" + ${sed_cmd} "s/useSMCrypto = \"false\"/useSMCrypto = \"${use_sm}\"/" "${p}" + ${sed_cmd} "s/127.0.0.1:20200/127.0.0.1:${rpc_port}/g" "${p}" + ${sed_cmd} "s/127.0.0.1:20201/127.0.0.1:$((rpc_port + 1))/g" "${p}" + done } -clean_node() +# informational only: a burst of PBFT view-change timeouts right after start +# is normal; a persistently growing count means the chain never reached +# consensus and the round against it will fail +report_chain_health() { - bash nodes/127.0.0.1/stop_all.sh - rm -rf nodes - if [ "${1}" == "true" ]; then - rm -rf ./fisco-bcos* - fi + local outdir="${1}" + local cnt + cnt=$(grep -h "After onTimeout" "${outdir}"/127.0.0.1/node0/log/*.log 2>/dev/null | wc -l | tr -d ' ') + LOG_INFO "--- ${outdir}: node0 consensus view-change timeouts so far: ${cnt:-0} ---" } - # check integration-test for non-gm node -check_standard_node() +run_integration_round() { - rm -rf build dist - build_node "normal" "${3}" - prepare_environment "${2}" - ## run integration test + local name="${1}" + local rpc_port="${2}" + local use_sm="${3}" + LOG_INFO "------ integration round: ${name} (rpc ${rpc_port}, sm=${use_sm}) ------" + wait_rpc_ready "${rpc_port}" + prepare_sdk_config "${rpc_port}" "${use_sm}" bash gradlew clean integrationTest --info # if hs_err log exist, print it (cat hs_err_pid*.log) || true - ## clean - clean_node "${1}" } -check_wasm_node() -{ - rm -rf build dist - build_node "wasm" "${3}" - prepare_wasm_environment - ## run integration test - bash gradlew clean integrationWasmTest --info - (cat hs_err_pid*.log) || true - - ## clean - clean_node "${1}" -} LOG_INFO "------ check java version ---------" java -version @@ -164,27 +183,40 @@ ls -la export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8 download_tassl -LOG_INFO "------ download_binary: v3.2.6---------" -download_build_chain "v3.2.6" -download_binary "v3.2.6" -LOG_INFO "------ check_standard_node---------" -check_standard_node "false" "normal" "-A" -rm -rf ./bin - -LOG_INFO "------ download_build_chain: v3.7.3---------" -download_binary "v3.7.3" -download_build_chain "v3.7.3" -LOG_INFO "------ check_wasm_node---------" -check_wasm_node "false" -LOG_INFO "------ check_standard_node---------" -check_standard_node "false" "normal" -LOG_INFO "------ check_standard_node---------" -check_standard_node "true" "sm" "-s" -rm -rf ./bin - -LOG_INFO "------ download_binary: v3.11.0---------" -download_build_chain "v3.11.0" -download_binary "v3.11.0" -LOG_INFO "------ check_standard_node---------" -check_standard_node "true" "normal" -rm -rf ./bin \ No newline at end of file +if [ ! -f "get_account.sh" ];then + curl -LO https://raw.githubusercontent.com/FISCO-BCOS/console/master/tools/get_account.sh +fi +if [ ! -f "get_gm_account.sh" ];then + curl -LO https://raw.githubusercontent.com/FISCO-BCOS/console/master/tools/get_gm_account.sh +fi + +PINNED_VERSION="v3.7.3" +LATEST_VERSION=$(get_latest_version) +LOG_INFO "------ node versions: ${PINNED_VERSION} (pinned) + ${LATEST_VERSION} (latest) ---------" + +download_build_chain "${PINNED_VERSION}" +download_binary "${PINNED_VERSION}" +if [ "${LATEST_VERSION}" != "${PINNED_VERSION}" ];then + download_build_chain "${LATEST_VERSION}" + download_binary "${LATEST_VERSION}" +fi + +# three chains, started together on disjoint ports, all with certificate-free rpc +build_chain_one "${PINNED_VERSION}" "nodes_pinned" "30300,20200" +build_chain_one "${LATEST_VERSION}" "nodes_latest" "30310,20210" +build_chain_one "${LATEST_VERSION}" "nodes_latest_sm" "30320,20220" "-s" + +wait_rpc_ready 20200 +wait_rpc_ready 20210 +wait_rpc_ready 20220 + +run_integration_round "ecdsa @ ${PINNED_VERSION}" 20200 "false" +report_chain_health "nodes_latest" +run_integration_round "ecdsa @ ${LATEST_VERSION}" 20210 "false" +report_chain_health "nodes_latest_sm" +run_integration_round "sm @ ${LATEST_VERSION}" 20220 "true" + +# best-effort teardown +bash nodes_pinned/127.0.0.1/stop_all.sh || true +bash nodes_latest/127.0.0.1/stop_all.sh || true +bash nodes_latest_sm/127.0.0.1/stop_all.sh || true diff --git a/AGENTS.md b/AGENTS.md index 1794948e5..c4a83b519 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,14 +33,19 @@ the repo's git pre-commit hook (`copyHooks` task copies from `hooks/`). ### Integration tests require a running chain -`./gradlew integrationTest` (EVM/Solidity) and `./gradlew integrationWasmTest` (WASM/Liquid) **cannot -run without a live FISCO BCOS node** and generated SDK certificates in `conf/`. CI provisions this in -`.ci/ci_check.sh` (`prepare_environment`): it builds a local chain with `build_chain.sh`, copies -`nodes/127.0.0.1/sdk/*` into `conf/`, and writes `src/integration-test/resources/config.toml` -(toggling `useSMCrypto` for SM-crypto nodes). Do not expect these tasks to pass in a bare checkout. - -CI entrypoint is `.github/workflows/workflow.yml` → `.ci/ci_check.sh` (Linux/macOS run integration -tests against multiple node versions; Windows runs only `./gradlew.bat build`). +`./gradlew integrationTest` (EVM/Solidity) **cannot run without a live FISCO BCOS node**. CI +provisions this in `.ci/ci_check.sh`: it starts three local 4-node chains at once with +`build_chain.sh` on disjoint ports — a pinned v3.7.3 ECDSA chain (rpc 20200), an ECDSA chain on the +**latest release** (rpc 20210, tag auto-resolved from the GitHub `releases/latest` redirect), and an +SM-crypto chain on the latest release (rpc 20220). All chains run with **SSL disabled on the RPC +endpoint** (`disable_ssl=true` / `enable_ssl=false` in each node's `[rpc]` config), so the SDK +connects **without certificates** (`enableSsl = "false"` in the rendered +`src/integration-test/resources/config.toml`; `useSMCrypto` toggled per round). `integrationTest` +then runs once per chain. (`./gradlew integrationWasmTest` still exists but is no longer exercised +in CI.) Do not expect these tasks to pass in a bare checkout. + +CI entrypoint is `.github/workflows/workflow.yml` → `.ci/ci_check.sh` (Linux/macOS run the three +integration rounds; Windows runs only `./gradlew.bat build`). ### Native dependency note From d8df1a30508a87e95bb1898ff7f19df99312d58d Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 17:17:44 +0800 Subject: [PATCH 02/14] (check): validate latest-tag resolution, probe rpc with real JSON-RPC Address review feedback: fail fast with a clear message if the latest release tag cannot be resolved (instead of confusing download errors), and make wait_rpc_ready send an actual getBlockNumber JSON-RPC request and require a jsonrpc response body (a bare HTTP probe could report ready on any HTTP responder). Probe verified against a live local chain. --- .ci/ci_check.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index d2befbe96..1d0bb5472 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -97,7 +97,9 @@ wait_rpc_ready() local port="${1}" local i for i in $(seq 1 30); do - if curl -s -m 3 --noproxy "*" -o /dev/null "http://127.0.0.1:${port}"; then + if curl -s -m 3 --noproxy "*" -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"getBlockNumber","params":["group0",""],"id":1}' \ + "http://127.0.0.1:${port}" | grep -q "jsonrpc"; then LOG_INFO "--- rpc 127.0.0.1:${port} is ready ---" return 0 fi @@ -192,6 +194,10 @@ fi PINNED_VERSION="v3.7.3" LATEST_VERSION=$(get_latest_version) +if ! echo "${LATEST_VERSION}" | grep -qE "^v[0-9]+\.[0-9]+\.[0-9]+$"; then + echo "failed to resolve the latest FISCO BCOS release tag, got: '${LATEST_VERSION}'" + exit 1 +fi LOG_INFO "------ node versions: ${PINNED_VERSION} (pinned) + ${LATEST_VERSION} (latest) ---------" download_build_chain "${PINNED_VERSION}" From 00a7cbbddc30d761fe598f772791d5318e862112 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 17:58:59 +0800 Subject: [PATCH 03/14] (check): stop each chain after its round, run all rounds to completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The macOS run showed the v3.16.3 round failing 6 PrecompiledTest cases with receipt status -4008 (node-side execution timeout): the chain processed transactions fine for ~16 minutes and then stalled permanently. Root cause is runner resource exhaustion — macos-latest (7GB, arm64) cannot sustain 12 x86_64 nodes under Rosetta plus the Gradle JVM for the whole job. The same tests pass locally against an idle v3.16.3 chain. Keep the start-everything-upfront design but stop each chain as soon as its round finishes, so round 2 runs with 8 nodes and round 3 with 4. Also make rounds non-fatal individually: every round always runs, failures are aggregated in FAILED_ROUNDS and reported at the end (a flaky early round no longer hides the later rounds' results), with chain health reported before and after each round. --- .ci/ci_check.sh | 57 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index 1d0bb5472..e574fe986 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -164,17 +164,35 @@ report_chain_health() LOG_INFO "--- ${outdir}: node0 consensus view-change timeouts so far: ${cnt:-0} ---" } +# run one round; never aborts the script — a failed round is recorded in +# FAILED_ROUNDS so the remaining rounds still run and the job fails at the end run_integration_round() { local name="${1}" local rpc_port="${2}" local use_sm="${3}" + local outdir="${4}" LOG_INFO "------ integration round: ${name} (rpc ${rpc_port}, sm=${use_sm}) ------" - wait_rpc_ready "${rpc_port}" - prepare_sdk_config "${rpc_port}" "${use_sm}" - bash gradlew clean integrationTest --info - # if hs_err log exist, print it - (cat hs_err_pid*.log) || true + report_chain_health "${outdir}" + local round_status=0 + if wait_rpc_ready "${rpc_port}"; then + prepare_sdk_config "${rpc_port}" "${use_sm}" + bash gradlew clean integrationTest --info || round_status=1 + # if hs_err log exist, print it + (cat hs_err_pid*.log) || true + else + round_status=1 + fi + report_chain_health "${outdir}" + # stop this chain as soon as its round is done: later rounds do not touch it, + # and the runner (especially macOS, where the x86_64 nodes run under Rosetta) + # cannot sustain all 12 nodes plus the JVM for the whole job — chain2 stalled + # with execution timeouts mid-round when all three chains were kept running + bash "${outdir}/127.0.0.1/stop_all.sh" || true + if [ "${round_status}" -ne 0 ]; then + FAILED_ROUNDS="${FAILED_ROUNDS} [${name}]" + fi + return 0 } LOG_INFO "------ check java version ---------" @@ -212,17 +230,20 @@ build_chain_one "${PINNED_VERSION}" "nodes_pinned" "30300,20200" build_chain_one "${LATEST_VERSION}" "nodes_latest" "30310,20210" build_chain_one "${LATEST_VERSION}" "nodes_latest_sm" "30320,20220" "-s" +# chain1 is needed right away; chains 2/3 have the whole preceding rounds to +# finish booting, so a slow start there is only logged, not fatal — each round +# re-checks readiness itself wait_rpc_ready 20200 -wait_rpc_ready 20210 -wait_rpc_ready 20220 - -run_integration_round "ecdsa @ ${PINNED_VERSION}" 20200 "false" -report_chain_health "nodes_latest" -run_integration_round "ecdsa @ ${LATEST_VERSION}" 20210 "false" -report_chain_health "nodes_latest_sm" -run_integration_round "sm @ ${LATEST_VERSION}" 20220 "true" - -# best-effort teardown -bash nodes_pinned/127.0.0.1/stop_all.sh || true -bash nodes_latest/127.0.0.1/stop_all.sh || true -bash nodes_latest_sm/127.0.0.1/stop_all.sh || true +wait_rpc_ready 20210 || true +wait_rpc_ready 20220 || true + +FAILED_ROUNDS="" +run_integration_round "ecdsa @ ${PINNED_VERSION}" 20200 "false" "nodes_pinned" +run_integration_round "ecdsa @ ${LATEST_VERSION}" 20210 "false" "nodes_latest" +run_integration_round "sm @ ${LATEST_VERSION}" 20220 "true" "nodes_latest_sm" + +if [ -n "${FAILED_ROUNDS}" ]; then + echo "integration rounds failed:${FAILED_ROUNDS}" + exit 1 +fi +LOG_INFO "------ all integration rounds passed ---------" From 6b130e814287ef02bd1a2be91acd88fdce9be0d6 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 18:58:39 +0800 Subject: [PATCH 04/14] (sdk): stop SystemServicesExhaustive test poisoning the chain gas price Root cause of the CI integration flakiness (and of the long-standing 6 PrecompiledTest failures on newer node versions upstream): SystemServicesExhaustiveIntegrationTest.testSystemConfigManyKeys set tx_gas_price=1 on the live chain. On any node version that supports the key (>= ~3.6), every later transaction from the zero-balance test accounts can no longer be sealed (the SDK times out after 10s and surfaces a synthetic -4008 receipt), and the price cannot be restored because the restoring transaction itself would need gas. Whether a run passed depended purely on Gradle's test class execution order: classes running before the poisoning passed, classes after it failed - which is why identical scripts alternated between green and red, on both ubuntu and macOS at the same wall-clock offset. Set the value to 0 instead: it exercises exactly the same Numeric.toHexString conversion branch in SystemConfigService.setValueByKey with no side effect. Same guard for the tx_gas_price governance proposal in AuthGovernanceExhaustiveIntegrationTest (currently inert because the test account is not a governor, but poisonous the day it is one). Verified locally against a fresh v3.16.3 chain in the poisoning order (SystemServicesExhaustive first, then PrecompiledTest): both pass, and the chain reports tx_gas_price=0x0 set at block 24. --- .../auth/AuthGovernanceExhaustiveIntegrationTest.java | 5 ++++- .../SystemServicesExhaustiveIntegrationTest.java | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/auth/AuthGovernanceExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/auth/AuthGovernanceExhaustiveIntegrationTest.java index ffc5d84d1..384864b3c 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/auth/AuthGovernanceExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/auth/AuthGovernanceExhaustiveIntegrationTest.java @@ -273,8 +273,11 @@ public void testSetSysConfigProposalLifecycle() { } readBackAndRevoke(proposalId, "setSysConfig"); // tx_gas_price branch -> exercises the Numeric.toHexString(value) conversion path. + // MUST stay "0": if this account ever is a governor with quorum, a non-zero value + // would actually execute and poison the live chain (zero-balance accounts can no + // longer send transactions, and the price cannot be restored). try { - authManager.createSetSysConfigProposal("tx_gas_price", "1"); + authManager.createSetSysConfigProposal("tx_gas_price", "0"); } catch (Exception e) { System.out.println("createSetSysConfigProposal(gasPrice) ignored: " + e.getMessage()); } diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index 69feb25ea..393e8bbe0 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -255,7 +255,13 @@ public void testSystemConfigManyKeys() { if (SystemConfigService.AUTH_STATUS.equals(key)) { next = current; // re-set same to avoid flipping auth on the live chain } else if (SystemConfigService.TX_GAS_PRICE.equals(key)) { - next = "1"; // small positive, exercises Numeric.toHexString conversion + // MUST stay "0": it still exercises the Numeric.toHexString conversion + // branch, but a non-zero gas price poisons the live chain — every later + // transaction from the zero-balance test accounts can no longer be + // sealed (and the price cannot be restored, since the restoring + // transaction itself would need gas), failing whole test classes that + // happen to run after this one. + next = "0"; } else if (SystemConfigService.TX_GAS_LIMIT.equals(key)) { next = new BigInteger(current) From 3fc7ba51932d0668f4b9108e238cafc6df6610b1 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 20:24:38 +0800 Subject: [PATCH 05/14] (sdk): stop consensus-membership churn poisoning the shared chain Second chain-poisoner found behind the v3.16.x-only round failures (the first was the tx_gas_price one): - WrapperTxContractDeepIntegrationTest.testConsensusServiceAddRemoveAgainstRealNode did setWeight(2) + setTermWeight(1) + addObserver on a REAL sealer and never restored it. On nodes >= 3.12 the setTermWeight version gate passes, so the final addObserver executes and permanently demotes a sealer of the shared 4-node chain; on <= 3.11 setTermWeight throws into the catch block first, which is exactly why only the latest-version rounds died. Bisect on a live v3.16.3 chain confirmed: sealers 4 -> 3 after the class ran, then the chain stalls under load and every later transaction times out with -4008. - SystemServicesExhaustiveIntegrationTest.testConsensusFullLifecycleWithRealNode demoted a sealer and re-added it, but membership changes only take effect on a block boundary: on newer nodes the immediate addSealer returned receipt status 0 without taking effect, silently leaving 3 sealers. - SystemServicesExhaustiveIntegrationTest.testConsensusSetTermWeight left a real sealer with termWeight=1. Fixes keep the codec/version-gate/receipt-parsing coverage without mutating live consensus state: setWeight now re-sets the genesis weight (success receipt, zero net change), setTermWeight targets a bogus node id (error receipt), and the demote/promote lifecycle poll-verifies the node is actually back in the sealer list, retrying the addSealer until it takes effect. --- ...stemServicesExhaustiveIntegrationTest.java | 58 ++++++++++++++----- .../WrapperTxContractDeepIntegrationTest.java | 25 ++++++-- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index 393e8bbe0..74c0a3435 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -155,12 +155,36 @@ public void testConsensusFullLifecycleWithRealNode() { System.out.println("addObserver duplicate rejected: " + expected.getMessage()); } - // put it back to sealer - try { - RetCode back = consensus.addSealer(nodeId, BigInteger.ONE); - System.out.println("addSealer back: " + back.getCode()); - } catch (Exception ex) { - System.out.println("addSealer back: " + ex.getMessage()); + // Put it back to sealer — and VERIFY it is actually back. Consensus membership + // changes only take effect on a block boundary; on newer nodes an addSealer sent + // right after the addObserver can return receipt status 0 without ever taking + // effect, silently leaving the shared 4-node chain with only 3 sealers (no PBFT + // fault tolerance) and stalling it under load. So poll-verify and retry. + boolean restored = false; + for (int attempt = 0; attempt < 5 && !restored; attempt++) { + try { + RetCode back = consensus.addSealer(nodeId, BigInteger.ONE); + System.out.println("addSealer back: " + back.getCode()); + } catch (Exception ex) { + System.out.println("addSealer back: " + ex.getMessage()); + } + for (int i = 0; i < 5 && !restored; i++) { + Thread.sleep(1000); + List now = client.getSealerList().getResult(); + for (SealerList.Sealer s : now) { + if (nodeId.equals(s.getNodeID())) { + restored = true; + break; + } + } + } + } + System.out.println("sealer restored: " + restored); + if (!restored) { + System.out.println( + "WARNING: could not restore node " + + nodeId + + " to the sealer list; the chain is left degraded!"); } } catch (Exception e) { System.out.println("testConsensusFullLifecycleWithRealNode skipped: " + e.getMessage()); @@ -192,16 +216,18 @@ public void testConsensusRemoveNode() { public void testConsensusSetTermWeight() { try { ConsensusService consensus = new ConsensusService(client, keyPair); - List sealerList = client.getSealerList().getResult(); - if (sealerList != null && !sealerList.isEmpty()) { - String nodeId = sealerList.get(0).getNodeID(); - try { - RetCode r = consensus.setTermWeight(nodeId, BigInteger.ONE); - System.out.println("setTermWeight: " + r.getCode()); - } catch (Exception ex) { - // version gate / rpBFT disabled -> fine - System.out.println("setTermWeight unsupported: " + ex.getMessage()); - } + // bogus node id on purpose: drives the version gate, the encoder and the + // error-receipt parsing without changing a REAL sealer's term weight (a live + // consensus-parameter mutation is never restored and can destabilize the + // shared chain on rpBFT-capable node versions) + String bogusNode = + "4444444444444444444444444444444444444444444444444444444444444444"; + try { + RetCode r = consensus.setTermWeight(bogusNode, BigInteger.ONE); + System.out.println("setTermWeight: " + r.getCode()); + } catch (Exception ex) { + // version gate / rpBFT disabled / unknown node -> fine + System.out.println("setTermWeight unsupported: " + ex.getMessage()); } } catch (Exception e) { System.out.println("testConsensusSetTermWeight skipped: " + e.getMessage()); diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java index 0297f21b6..6247e1c50 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java @@ -717,15 +717,28 @@ public void testConsensusServiceAddRemoveAgainstRealNode() { try { ConsensusService service = new ConsensusService(client, keyPair); String node = realSealerNodeId(); - // exercise input/output decoders through the service against a real sealer node id + // Exercise the service codecs WITHOUT mutating the live consensus membership. + // This test previously did setWeight(2) + setTermWeight(1) + addObserver on a + // REAL sealer and never restored it. On nodes >= 3.12 (where setTermWeight passes + // its version gate instead of throwing into the catch block, which is why <= 3.11 + // chains were unaffected) that permanently demoted a sealer of the shared 4-node + // chain, leaving PBFT with no fault tolerance; the chain then stalled under load + // and every later transaction in the suite timed out with -4008. RetCode addSealer = service.addSealer(node, BigInteger.ONE); System.out.println("addSealer ret: " + (addSealer == null ? "null" : addSealer.getCode())); - RetCode setWeight = service.setWeight(node, BigInteger.valueOf(2)); + // same weight as genesis (1): success receipt + codecs, zero net change + RetCode setWeight = service.setWeight(node, BigInteger.ONE); System.out.println("setWeight ret: " + (setWeight == null ? "null" : setWeight.getCode())); - RetCode setTermWeight = service.setTermWeight(node, BigInteger.ONE); - System.out.println("setTermWeight ret: " + (setTermWeight == null ? "null" : setTermWeight.getCode())); - RetCode addObserver = service.addObserver(node); - System.out.println("addObserver ret: " + (addObserver == null ? "null" : addObserver.getCode())); + // bogus node id: still drives the version gate, the encoder and the error-receipt + // parsing, but cannot touch a real consensus node + String bogusTermNode = + "3333333333333333333333333333333333333333333333333333333333333333"; + try { + RetCode setTermWeight = service.setTermWeight(bogusTermNode, BigInteger.ONE); + System.out.println("setTermWeight ret: " + (setTermWeight == null ? "null" : setTermWeight.getCode())); + } catch (Exception ex) { + System.out.println("setTermWeight rejected: " + ex.getMessage()); + } Assert.assertTrue(true); } catch (Exception e) { System.out.println("testConsensusServiceAddRemoveAgainstRealNode skipped: " + e.getMessage()); From 43b63b180865689a1f58304b59628c409df2a44f Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 21:25:18 +0800 Subject: [PATCH 06/14] (sdk): async consensus test was removing a live sealer despite 'bogus' comment Third and final chain-poisoner behind the -4008 cascades: testConsensusAsyncCallbackVariants declared 'String bogus' with a comment claiming a bogus node id, but actually assigned realSealerNodeId(). The async consensus.remove(bogus, cb) therefore REMOVED a live sealer from the shared 4-node chain. Unlike setTermWeight there is no version gate on remove, which is why after the first two poisoners were fixed the v3.7.3 round started failing as well - whether a round survived depended only on whether Gradle happened to schedule this class before or after the strict assertion classes. Use a genuinely bogus 64-hex node id: the async submission paths, callbacks and decoders are still exercised via the error receipt, with zero live consensus mutation (same pattern as testConsensusRemoveInputOutputDecode). --- .../WrapperTxContractDeepIntegrationTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java index 6247e1c50..719a76649 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java @@ -688,8 +688,13 @@ public void testConsensusAsyncCallbackVariants() { ConsensusPrecompiled consensus = ConsensusPrecompiled.load( PrecompiledAddress.CONSENSUS_PRECOMPILED_ADDRESS, client, keyPair); - // bogus node id so the chain rejects but the async path + decoders still run - String bogus = realSealerNodeId(); + // A REALLY bogus node id so the chain rejects but the async path + decoders still + // run. This used to be realSealerNodeId() despite the comment: the async + // consensus.remove() below then REMOVED a live sealer from the shared 4-node chain + // (remove has no version gate, so every node version was affected), the chain + // stalled under load and every later transaction in the suite timed out with -4008. + String bogus = + "5555555555555555555555555555555555555555555555555555555555555555"; final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(2); TransactionCallback cb = From 344acbc55e4e95467f9f10f8180ff2a0e761931a Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 22:24:22 +0800 Subject: [PATCH 07/14] (sdk): consensus decode tests were demoting a live sealer too Fourth chain-poisoner: PrecompiledWrapperDecodeIntegrationTest's consensus input/output-decode tests shared pickNodeId(), which returned a REAL sealer id (getSealerList().get(0)). The addObserver decode test assumed the tx 'may be rejected by the chain', but demoting an existing sealer to observer succeeds - so the class silently dropped the shared 4-node chain to 3 sealers before the other consensus tests even ran (visible in CI as sealerList=3 at the start of SystemServicesExhaustive's lifecycle test), the chain stalled under load and later strict test classes failed with -4008. pickNodeId() now returns a well-formed bogus node id: all four decode tests assert only on transaction INPUT decoding, which behaves identically on the error receipt a bogus id produces (same pattern the remove decode test in the same class already used). --- .../PrecompiledWrapperDecodeIntegrationTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java index 9e51fb810..29b50461f 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java @@ -472,14 +472,13 @@ public void testTableConditionUpdateRemoveInputDecodeAndCount() { // ====================================================================== private static String pickNodeId() { - try { - List sealers = client.getSealerList().getResult(); - if (sealers != null && !sealers.isEmpty()) { - return sealers.get(0).getNodeID(); - } - } catch (Exception ignored) { - } - return null; + // A well-formed but BOGUS node id, on purpose. This used to return a REAL sealer id + // (getSealerList().get(0)): the addObserver decode test below then actually DEMOTED a + // live sealer of the shared 4-node chain (the "may be rejected" assumption was wrong), + // leaving PBFT with no fault tolerance and stalling the chain under load. The decode + // tests only assert on the transaction INPUT decoding, which works exactly the same on + // the error receipt a bogus node id produces. + return "6666666666666666666666666666666666666666666666666666666666666666"; } @Test From 8a204266a121d197811b0c274e348973c97caaf0 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Fri, 3 Jul 2026 23:14:25 +0800 Subject: [PATCH 08/14] (sdk): wait for demotion to apply before restore; avoid phantom consensus entries Two refinements after the previous run (v3.7.3 round green, v3.16.x rounds still dying 25s after the consensus lifecycle test): 1. The lifecycle restore was a false positive on newer nodes: membership changes apply on a block boundary, so right after addObserver the node is STILL in the sealer list - the immediate addSealer-back gets rejected with ALREADY_EXISTS, the membership poll passes on the stale list ('sealer restored: true' 2s after the demote), and the demotion lands afterwards, leaving 3 sealers. Now the test first waits until the demotion has actually been applied (node absent from the list, up to 10s) and only then re-adds and verifies, retrying until the change takes effect. 2. Newer nodes ACCEPT addSealer/addObserver/setTermWeight for a bogus node id with receipt status 0 (older nodes reject with -51100), creating a phantom consensus-table entry. The addSealer decode test now targets a REAL sealer (ALREADY_EXISTS rejection - guaranteed non-mutating), the setWeight decode test re-sets a real sealer's genesis weight (success, zero net change), and an @AfterClass hook best-effort removes the phantom entry the remaining bogus-id decode tests may leave in the table. --- ...ecompiledWrapperDecodeIntegrationTest.java | 40 +++++++++++++++++-- ...stemServicesExhaustiveIntegrationTest.java | 28 ++++++++++--- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java index 29b50461f..13158366b 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledWrapperDecodeIntegrationTest.java @@ -476,11 +476,37 @@ private static String pickNodeId() { // (getSealerList().get(0)): the addObserver decode test below then actually DEMOTED a // live sealer of the shared 4-node chain (the "may be rejected" assumption was wrong), // leaving PBFT with no fault tolerance and stalling the chain under load. The decode - // tests only assert on the transaction INPUT decoding, which works exactly the same on - // the error receipt a bogus node id produces. + // tests only assert on the transaction INPUT decoding, which works exactly the same + // whatever the receipt status is. (Newer nodes even accept the bogus id with status 0, + // but the phantom entry never appears in the effective sealer/observer lists; the + // @AfterClass cleanup below removes it from the consensus table anyway.) return "6666666666666666666666666666666666666666666666666666666666666666"; } + /** A real sealer id, ONLY for provably non-mutating ops (already-exists / same-value). */ + private static String realSealerId() { + try { + List sealers = client.getSealerList().getResult(); + if (sealers != null && !sealers.isEmpty()) { + return sealers.get(0).getNodeID(); + } + } catch (Exception ignored) { + } + return null; + } + + @org.junit.AfterClass + public static void cleanupPhantomConsensusEntry() { + // best-effort: drop the bogus consensus-table entry newer nodes accept + try { + ConsensusPrecompiled consensus = + ConsensusPrecompiled.load( + PrecompiledAddress.CONSENSUS_PRECOMPILED_ADDRESS, client, keyPair); + consensus.remove(pickNodeId()); + } catch (Exception ignored) { + } + } + @Test public void testConsensusAddObserverInputOutputDecode() { try { @@ -517,7 +543,10 @@ public void testConsensusAddObserverInputOutputDecode() { @Test public void testConsensusAddSealerInputOutputDecode() { try { - String nodeId = pickNodeId(); + // real sealer id: addSealer on an EXISTING sealer is rejected on every node + // version (ALREADY_EXISTS_IN_SEALER_LIST) - a guaranteed-non-mutating receipt. + // A bogus id is NOT safe here: newer nodes accept it and create a phantom entry. + String nodeId = realSealerId(); if (nodeId == null) { System.out.println("testConsensusAddSealerInputOutputDecode skipped: no node id"); Assert.assertTrue(true); @@ -548,7 +577,10 @@ public void testConsensusAddSealerInputOutputDecode() { @Test public void testConsensusSetWeightInputOutputDecode() { try { - String nodeId = pickNodeId(); + // real sealer id with its genesis weight (1): a success receipt with zero net + // change on every node version. A bogus id would create a phantom entry on + // newer nodes. + String nodeId = realSealerId(); if (nodeId == null) { System.out.println("testConsensusSetWeightInputOutputDecode skipped: no node id"); Assert.assertTrue(true); diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index 74c0a3435..b14c58ca0 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -155,12 +155,28 @@ public void testConsensusFullLifecycleWithRealNode() { System.out.println("addObserver duplicate rejected: " + expected.getMessage()); } - // Put it back to sealer — and VERIFY it is actually back. Consensus membership - // changes only take effect on a block boundary; on newer nodes an addSealer sent - // right after the addObserver can return receipt status 0 without ever taking - // effect, silently leaving the shared 4-node chain with only 3 sealers (no PBFT - // fault tolerance) and stalling it under load. So poll-verify and retry. - boolean restored = false; + // Membership changes only take effect on a block boundary. FIRST wait until the + // demotion has actually been applied (the node left the sealer list) — otherwise + // the restore below is a false positive: the immediate addSealer is rejected with + // ALREADY_EXISTS while the node is still listed, the membership check passes on + // the stale list, and the demotion lands afterwards leaving the shared 4-node + // chain with only 3 sealers (no PBFT fault tolerance) until it stalls under load. + boolean demoted = false; + for (int i = 0; i < 10 && !demoted; i++) { + Thread.sleep(1000); + demoted = true; + for (SealerList.Sealer s : client.getSealerList().getResult()) { + if (nodeId.equals(s.getNodeID())) { + demoted = false; + break; + } + } + } + System.out.println("demotion applied: " + demoted); + + // Put it back to sealer — and VERIFY it is actually back, retrying until the + // membership change takes effect. + boolean restored = !demoted; for (int attempt = 0; attempt < 5 && !restored; attempt++) { try { RetCode back = consensus.addSealer(nodeId, BigInteger.ONE); From 6d8935cb5e6aff11313bf19037a045a2857860db Mon Sep 17 00:00:00 2001 From: kyonRay Date: Sat, 4 Jul 2026 00:02:42 +0800 Subject: [PATCH 09/14] (sdk): stop demoting live sealers entirely - rejection paths only Even with the demotion verified as applied and the sealer list verified as restored ('demotion applied: true' / 'sealer restored: true' in the previous run), the v3.16.x chains still stalled ~25s after the lifecycle test: on newer nodes the demoted-then-restored node's consensus engine does not re-engage cleanly even though the sealer list shows it back, and the chain dies under load. Live consensus-membership mutation is fundamentally unsafe on a suite-shared chain. The lifecycle test now exercises only guaranteed-non-mutating real-node branches: addSealer on an existing sealer (ALREADY_EXISTS rejection) and setWeight with the genesis weight (success receipt, zero net change). The demote/restore success path is left to dedicated chain-per-test environments. --- ...stemServicesExhaustiveIntegrationTest.java | 77 ++++--------------- 1 file changed, 14 insertions(+), 63 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index b14c58ca0..17567eb17 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -112,10 +112,15 @@ public static void tearDown() { // ---------------------------------------------------------------------- /** - * Full add/remove lifecycle using REAL node ids from the chain. This drives every well-formed - * branch in ConsensusService (existsInNodeList, sealer/observer membership checks, sync-status - * threshold check, receipt parsing) instead of only the validation-error branches covered by - * the expanded test. + * Drives the well-formed real-node branches of ConsensusService (existsInNodeList, sealer + * membership checks, sync-status threshold check, receipt parsing) via REJECTION paths only. + * + *

This test used to demote a real sealer to observer and re-add it. That is NOT safe on a + * shared 4-node chain: on newer nodes (>= 3.12) the demoted-then-restored node's consensus + * engine does not re-engage cleanly even after the sealer list shows it restored, and the + * chain stalls under load ~25s later, failing every remaining transaction in the suite with + * -4008. Live consensus-membership mutation belongs in a dedicated chain-per-test setup, not + * a suite-shared chain. */ @Test public void testConsensusFullLifecycleWithRealNode() { @@ -140,67 +145,13 @@ public void testConsensusFullLifecycleWithRealNode() { System.out.println("addSealer existing sealer rejected: " + expected.getMessage()); } - // move it to observer (real, well-formed) - may succeed or revert + // setWeight with the node's genesis weight (1): a success receipt with zero net + // change — drives the real-node success path without mutating anything try { - RetCode obs = consensus.addObserver(nodeId); - System.out.println("addObserver real node: " + obs.getCode()); + RetCode sw = consensus.setWeight(nodeId, BigInteger.ONE); + System.out.println("setWeight same-value: " + sw.getCode()); } catch (Exception ex) { - System.out.println("addObserver real node: " + ex.getMessage()); - } - - // adding observer again should now hit ALREADY_EXISTS_IN_OBSERVER_LIST - try { - consensus.addObserver(nodeId); - } catch (Exception expected) { - System.out.println("addObserver duplicate rejected: " + expected.getMessage()); - } - - // Membership changes only take effect on a block boundary. FIRST wait until the - // demotion has actually been applied (the node left the sealer list) — otherwise - // the restore below is a false positive: the immediate addSealer is rejected with - // ALREADY_EXISTS while the node is still listed, the membership check passes on - // the stale list, and the demotion lands afterwards leaving the shared 4-node - // chain with only 3 sealers (no PBFT fault tolerance) until it stalls under load. - boolean demoted = false; - for (int i = 0; i < 10 && !demoted; i++) { - Thread.sleep(1000); - demoted = true; - for (SealerList.Sealer s : client.getSealerList().getResult()) { - if (nodeId.equals(s.getNodeID())) { - demoted = false; - break; - } - } - } - System.out.println("demotion applied: " + demoted); - - // Put it back to sealer — and VERIFY it is actually back, retrying until the - // membership change takes effect. - boolean restored = !demoted; - for (int attempt = 0; attempt < 5 && !restored; attempt++) { - try { - RetCode back = consensus.addSealer(nodeId, BigInteger.ONE); - System.out.println("addSealer back: " + back.getCode()); - } catch (Exception ex) { - System.out.println("addSealer back: " + ex.getMessage()); - } - for (int i = 0; i < 5 && !restored; i++) { - Thread.sleep(1000); - List now = client.getSealerList().getResult(); - for (SealerList.Sealer s : now) { - if (nodeId.equals(s.getNodeID())) { - restored = true; - break; - } - } - } - } - System.out.println("sealer restored: " + restored); - if (!restored) { - System.out.println( - "WARNING: could not restore node " - + nodeId - + " to the sealer list; the chain is left degraded!"); + System.out.println("setWeight same-value: " + ex.getMessage()); } } catch (Exception e) { System.out.println("testConsensusFullLifecycleWithRealNode skipped: " + e.getMessage()); From 7b7830afc61e35db5068049fb76e6b32cf47b8b7 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Sat, 4 Jul 2026 00:53:30 +0800 Subject: [PATCH 10/14] (sdk): sysconfig tests re-set current values instead of mutating live consensus params With every consensus-membership mutation eliminated, the v3.16.x rounds still stalled ~25-30s after SystemServicesExhaustive's sysconfig writes (v3.7.3 immune, same tests). The remaining live mutations were consensus_leader_period (current+1) - a PBFT reconfiguration that applies at the next epoch - and enabling the bugfix_revert feature switch mid-run. All sysconfig coverage tests now RE-SET the current value (full setValueByKey pipeline: validation predicates, tx_gas_price hex branch, submission, receipt parsing - zero behavior change), and the feature-switch write only happens if the feature is already enabled. Same treatment for the consensus_leader_period test in PrecompiledExpandedIntegrationTest. --- .../PrecompiledExpandedIntegrationTest.java | 6 ++- ...stemServicesExhaustiveIntegrationTest.java | 45 ++++++++++--------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledExpandedIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledExpandedIntegrationTest.java index f861f480a..b96993816 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledExpandedIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/PrecompiledExpandedIntegrationTest.java @@ -362,8 +362,10 @@ public void testSystemConfigConsensusPeriod() { String key = SystemConfigService.CONSENSUS_PERIOD; String current = client.getSystemConfigByKey(key).getSystemConfig().getValue(); System.out.println("consensus_leader_period current: " + current); - BigInteger updated = new BigInteger(current).add(BigInteger.ONE); - RetCode r = sysConfig.setValueByKey(key, updated.toString()); + // re-set the CURRENT value: full setValueByKey pipeline, zero behavior change. + // Actually changing the leader period reconfigures PBFT at the next epoch and + // can stall newer nodes on the shared live chain. + RetCode r = sysConfig.setValueByKey(key, current); System.out.println("setValueByKey consensus_leader_period: " + r.getCode()); String after = client.getSystemConfigByKey(key).getSystemConfig().getValue(); System.out.println("consensus_leader_period after: " + after); diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index 17567eb17..2b9bbab5b 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -243,25 +243,18 @@ public void testSystemConfigManyKeys() { String current = client.getSystemConfigByKey(key).getSystemConfig().getValue(); System.out.println(key + " current=" + current); - // pick a valid-ish next value per key - String next; - if (SystemConfigService.AUTH_STATUS.equals(key)) { - next = current; // re-set same to avoid flipping auth on the live chain - } else if (SystemConfigService.TX_GAS_PRICE.equals(key)) { - // MUST stay "0": it still exercises the Numeric.toHexString conversion - // branch, but a non-zero gas price poisons the live chain — every later - // transaction from the zero-balance test accounts can no longer be - // sealed (and the price cannot be restored, since the restoring - // transaction itself would need gas), failing whole test classes that - // happen to run after this one. - next = "0"; - } else if (SystemConfigService.TX_GAS_LIMIT.equals(key)) { - next = - new BigInteger(current) - .add(BigInteger.valueOf(1000)) - .toString(); - } else { - next = new BigInteger(current).add(BigInteger.ONE).toString(); + // ALWAYS re-set the CURRENT value. This still drives the full + // setValueByKey pipeline (validation predicates, the tx_gas_price + // Numeric.toHexString branch, tx submission, receipt parsing) but leaves + // the shared chain's behavior untouched. Changing live values here has + // repeatedly poisoned the suite: a non-zero tx_gas_price makes every + // zero-balance account unable to transact (and cannot be undone, since + // the undoing tx would itself need gas), and consensus parameters like + // consensus_leader_period apply at the next epoch and can stall newer + // nodes' PBFT minutes later. + String next = current; + if (SystemConfigService.TX_GAS_PRICE.equals(key)) { + next = "0"; // hex-conversion branch; 0 keeps transactions free } RetCode r = sysConfig.setValueByKey(key, next); System.out.println("set " + key + "=" + next + " -> " + r.getCode()); @@ -312,10 +305,18 @@ public void testSystemConfigFeatureKeys() { } catch (Exception expected) { System.out.println("unknown feature rejected: " + expected.getMessage()); } - // a real, known feature key (may or may not be enabled on the chain version) + // a real, known feature key: only RE-SET it if it is already enabled on this + // chain — enabling a feature switch mid-run changes execution semantics at the + // next block on the shared live chain try { - RetCode r = sysConfig.setValueByKey("bugfix_revert", "1"); - System.out.println("set bugfix_revert: " + r.getCode()); + String cur = + client.getSystemConfigByKey("bugfix_revert").getSystemConfig().getValue(); + if ("1".equals(cur)) { + RetCode r = sysConfig.setValueByKey("bugfix_revert", "1"); + System.out.println("set bugfix_revert: " + r.getCode()); + } else { + System.out.println("bugfix_revert not enabled, write skipped"); + } } catch (Exception ex) { System.out.println("set bugfix_revert: " + ex.getMessage()); } From acd55cc8d774f42daf2565820e772f0bd148cd6a Mon Sep 17 00:00:00 2001 From: kyonRay Date: Sat, 4 Jul 2026 02:03:21 +0800 Subject: [PATCH 11/14] (check): connect the SDK to two peers for RPC failover Aligned three consecutive macOS runs: the v3.16.3 round dies ~7.5 minutes in, between two vanilla CRUD transactions, regardless of which test is running - after every test-side state mutation had been eliminated. The same signature was reproduced locally: node0's RPC endpoint wedges (every request times out) while node1 keeps answering on the same healthy chain. The rendered SDK config pointed at a SINGLE node's RPC, so a one-node RPC wedge killed the entire round. Render peers=[node0, node1] like the historical CI config did, so the SDK fails over instead of the round dying with cascading -4008 timeouts. --- .ci/ci_check.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index e574fe986..1e80417df 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -134,7 +134,11 @@ prepare_sdk_config() ${sed_cmd} "s/enableSsl = \"true\"/enableSsl = \"false\"/" ./src/integration-test/resources/config.toml ${sed_cmd} "s/useSMCrypto = \"false\"/useSMCrypto = \"${use_sm}\"/" ./src/integration-test/resources/config.toml - ${sed_cmd} "s/127.0.0.1:20201/127.0.0.1:${rpc_port}/g" ./src/integration-test/resources/config.toml + # TWO peers for failover: on v3.16.x a single node's RPC endpoint can wedge under + # sustained load (observed both locally and on CI ~7.5 min into a round: every request + # to node0 times out while node1 keeps answering); with a second peer the SDK fails + # over instead of the whole round dying + ${sed_cmd} "s/peers=\[.*\]/peers=[\"127.0.0.1:${rpc_port}\", \"127.0.0.1:$((rpc_port + 1))\"]/" ./src/integration-test/resources/config.toml # amop test configs: restore from a pristine template each round, then point # them at this round's chain; certPath is replaced by enableSsl=false since From adae19c9bdb95f8a2a1a0784e2184469d979c079 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Tue, 7 Jul 2026 11:26:43 +0800 Subject: [PATCH 12/14] (check): dump per-node diagnostics on round failure, clean up phantom consensus entries --- .ci/ci_check.sh | 42 +++++++++++++++++++ ...stemServicesExhaustiveIntegrationTest.java | 22 ++++++++-- .../WrapperTxContractDeepIntegrationTest.java | 7 ++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index 1e80417df..9a0d7d9ee 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -168,6 +168,45 @@ report_chain_health() LOG_INFO "--- ${outdir}: node0 consensus view-change timeouts so far: ${cnt:-0} ---" } +rpc_call() +{ + local port="${1}" + local method="${2}" + curl -s -m 5 --noproxy "*" -H "Content-Type: application/json" \ + -d "{\"jsonrpc\":\"2.0\",\"method\":\"${method}\",\"params\":[\"group0\",\"\"],\"id\":1}" \ + "http://127.0.0.1:${port}" 2>/dev/null +} + +# only runs after a failed round: a "-4008 receipt timeout" alone cannot be +# attributed — this answers, per node, whether blocks still advance, whether +# txs are stuck in the txpool, what the committee looks like (a phantom +# bogus-id entry would show up in the sealer/observer lists), and what the +# node logs themselves report +dump_chain_diagnostics() +{ + local rpc_port="${1}" + local outdir="${2}" + local i port node + LOG_INFO "--- diagnostics for ${outdir} (rpc base ${rpc_port}) ---" + for i in 0 1 2 3; do + port=$((rpc_port + i)) + echo "[node${i} :${port}] blockNumber(t0): $(rpc_call "${port}" getBlockNumber)" + echo "[node${i} :${port}] pendingTxSize: $(rpc_call "${port}" getPendingTxSize)" + done + sleep 5 + for i in 0 1 2 3; do + port=$((rpc_port + i)) + echo "[node${i} :${port}] blockNumber(t+5s): $(rpc_call "${port}" getBlockNumber)" + done + echo "[node0] sealerList: $(rpc_call "${rpc_port}" getSealerList | head -c 2000)" + echo "[node0] observerList: $(rpc_call "${rpc_port}" getObserverList | head -c 1000)" + echo "[node0] consensusStatus: $(rpc_call "${rpc_port}" getConsensusStatus | head -c 3000)" + for node in node0 node1 node2 node3; do + echo "--- ${outdir}/${node}: last warning/error log lines ---" + grep -hE "^(warning|error)\|" "${outdir}/127.0.0.1/${node}/log/"*.log 2>/dev/null | tail -20 || true + done +} + # run one round; never aborts the script — a failed round is recorded in # FAILED_ROUNDS so the remaining rounds still run and the job fails at the end run_integration_round() @@ -188,6 +227,9 @@ run_integration_round() round_status=1 fi report_chain_health "${outdir}" + if [ "${round_status}" -ne 0 ]; then + dump_chain_diagnostics "${rpc_port}" "${outdir}" || true + fi # stop this chain as soon as its round is done: later rounds do not touch it, # and the runner (especially macOS, where the x86_64 nodes run under Rosetta) # cannot sustain all 12 nodes plus the JVM for the whole job — chain2 stalled diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java index 2b9bbab5b..c88b56c06 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/SystemServicesExhaustiveIntegrationTest.java @@ -195,6 +195,14 @@ public void testConsensusSetTermWeight() { } catch (Exception ex) { // version gate / rpBFT disabled / unknown node -> fine System.out.println("setTermWeight unsupported: " + ex.getMessage()); + } finally { + // some node versions (observed on 3.16.x) accept consensus ops for an + // unknown node id with a success receipt, leaving a phantom committee + // entry behind on the shared chain — always try to remove it again + try { + consensus.removeNode(bogusNode); + } catch (Exception ignored) { + } } } catch (Exception e) { System.out.println("testConsensusSetTermWeight skipped: " + e.getMessage()); @@ -207,14 +215,20 @@ public void testConsensusSetTermWeight() { public void testConsensusSetWeightInvalidNode() { try { ConsensusService consensus = new ConsensusService(client, keyPair); + String bogusNode = + "3333333333333333333333333333333333333333333333333333333333333333"; try { - RetCode r = - consensus.setWeight( - "3333333333333333333333333333333333333333333333333333333333333333", - BigInteger.valueOf(2)); + RetCode r = consensus.setWeight(bogusNode, BigInteger.valueOf(2)); System.out.println("setWeight invalid: " + r.getCode()); } catch (Exception ex) { System.out.println("setWeight invalid rejected: " + ex.getMessage()); + } finally { + // 3.16.x-style nodes accept this with a success receipt, which would put a + // phantom weight-2 sealer into the shared chain's committee — remove it + try { + consensus.removeNode(bogusNode); + } catch (Exception ignored) { + } } } catch (Exception e) { System.out.println("testConsensusSetWeightInvalidNode skipped: " + e.getMessage()); diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java index 719a76649..634398e02 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java @@ -743,6 +743,13 @@ public void testConsensusServiceAddRemoveAgainstRealNode() { System.out.println("setTermWeight ret: " + (setTermWeight == null ? "null" : setTermWeight.getCode())); } catch (Exception ex) { System.out.println("setTermWeight rejected: " + ex.getMessage()); + } finally { + // 3.16.x-style nodes accept consensus ops for an unknown node id with a + // success receipt, leaving a phantom committee entry on the shared chain + try { + service.removeNode(bogusTermNode); + } catch (Exception ignored) { + } } Assert.assertTrue(true); } catch (Exception e) { From b6a991960b717f7efd2c6ad32171d48bf41cce02 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Tue, 7 Jul 2026 13:10:48 +0800 Subject: [PATCH 13/14] (test): stop passing a BFS path as solidity deploy path, it poisons 3.16.x chains --- .../WrapperTxContractDeepIntegrationTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java index 634398e02..b97b570c5 100644 --- a/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java +++ b/src/integration-test/java/org/fisco/bcos/sdk/v3/test/precompiled/WrapperTxContractDeepIntegrationTest.java @@ -876,10 +876,17 @@ public void testProcessorDeployStringParamsPathOverloadsAndSignedDeploy() { AssembleTransactionProcessor processor = TransactionProcessorFactory.createAssembleTransactionProcessor(client, keyPair); - // deployAndGetResponseWithStringParams(abi, bin, params) — HelloWorld constructor empty + // deployAndGetResponseWithStringParams(abi, bin, params) — HelloWorld constructor empty. + // The 4th arg is the DEPLOY PATH and must stay "" on a Solidity chain: it is copied + // verbatim into the transaction's `to` field (only WASM/Liquid deploys use a path). + // Passing "HelloWorld" here put a non-hex `to` on the wire; the 3.16.x baseline + // scheduler throws from boost unhex while executing that block and PBFT retries the + // same poisoned proposal forever — one such tx permanently halted the whole chain + // (every later tx in the suite then timed out with -4008). Older executors (3.7.x) + // tolerate it, which is why only the latest-version CI rounds died. TransactionResponse sp1 = processor.deployAndGetResponseWithStringParams( - helloWorldAbi, helloWorldBin, new ArrayList(), "HelloWorld"); + helloWorldAbi, helloWorldBin, new ArrayList(), ""); System.out.println("deployWithStringParams status: " + sp1.getReturnCode()); // deployAndGetResponse(abi, signedData) — pre-signed deploy via createSignedConstructor From f78276325c46dfa12299b4382b53ac3845c502f2 Mon Sep 17 00:00:00 2001 From: kyonRay Date: Tue, 7 Jul 2026 13:43:16 +0800 Subject: [PATCH 14/14] (check): revert to single peer, poison-tx fix makes failover unneeded and it flakes testClient --- .ci/ci_check.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.ci/ci_check.sh b/.ci/ci_check.sh index 9a0d7d9ee..6b2995d20 100755 --- a/.ci/ci_check.sh +++ b/.ci/ci_check.sh @@ -134,11 +134,13 @@ prepare_sdk_config() ${sed_cmd} "s/enableSsl = \"true\"/enableSsl = \"false\"/" ./src/integration-test/resources/config.toml ${sed_cmd} "s/useSMCrypto = \"false\"/useSMCrypto = \"${use_sm}\"/" ./src/integration-test/resources/config.toml - # TWO peers for failover: on v3.16.x a single node's RPC endpoint can wedge under - # sustained load (observed both locally and on CI ~7.5 min into a round: every request - # to node0 times out while node1 keeps answering); with a second peer the SDK fails - # over instead of the whole round dying - ${sed_cmd} "s/peers=\[.*\]/peers=[\"127.0.0.1:${rpc_port}\", \"127.0.0.1:$((rpc_port + 1))\"]/" ./src/integration-test/resources/config.toml + # single peer on purpose: the round death we chased with a "failover" second peer was + # never a single-node RPC wedge — it was one poison transaction (non-hex `to`) halting + # the whole chain on 3.16.x, now fixed test-side. A second peer does not help a + # whole-chain stall and it makes BcosSDKTest.testClient flaky: getBlockByNumber and + # getBlockByHash can then land on different nodes whose node-local block fields + # (importTime, signatureList order) differ, breaking its block-equality assertion + ${sed_cmd} "s/peers=\[.*\]/peers=[\"127.0.0.1:${rpc_port}\"]/" ./src/integration-test/resources/config.toml # amop test configs: restore from a pristine template each round, then point # them at this round's chain; certPath is replaced by enableSsl=false since