From 99156680114033aa0818fa12bf532a3ef6359349 Mon Sep 17 00:00:00 2001 From: quasar Date: Thu, 12 Nov 2026 00:52:40 +0000 Subject: [PATCH 1/2] Add cross-platform noise benchmark comparison (issue 31) Add a new GitHub Actions workflow (Cross-Platform Noise Benchmark) that runs an additional benchmark of all OpenSimplex2F noise functions on all three platforms (Linux, Windows, macOS) with the same seed and the same input parameters, waits for all three jobs to complete, and then compares the produced result files byte-for-byte: the noise results must match perfectly on every platform. - tests/test_OpenSimplex2F_cross_platform.c: evaluates all nine noise functions (2D, 3D, 4D variants) on a fixed coordinate grid with three frequency scales (37107 values in total) and prints the exact IEEE-754 binary64 bit pattern of every result plus an FNV-1a checksum. - tests/CMakeLists.txt: new OpenSimplex2_cross_platform executable and the OpenSimplex2_cross_platform_benchmark target that runs it (no cqtdeployer deployment required). - CMakeLists.txt: disable FMA (-mno-fma) for GNU/Clang builds of the library and the benchmark so that arm64 (Apple Silicon) FMA contraction cannot change the least significant bits of the results. - .github/workflows/cross-platform-compare.yml: benchmark matrix job on ubuntu-latest, windows-latest and macos-latest (same Qt/libpng setup as ci.yml), artifact upload of the platform reports, and a compare job that waits for all three jobs (needs) and fails when the reports differ. --- .github/workflows/cross-platform-compare.yml | 185 ++++++++++ CMakeLists.txt | 12 + tests/CMakeLists.txt | 35 ++ tests/test_OpenSimplex2F_cross_platform.c | 364 +++++++++++++++++++ 4 files changed, 596 insertions(+) create mode 100644 .github/workflows/cross-platform-compare.yml create mode 100644 tests/test_OpenSimplex2F_cross_platform.c diff --git a/.github/workflows/cross-platform-compare.yml b/.github/workflows/cross-platform-compare.yml new file mode 100644 index 0000000..fefde78 --- /dev/null +++ b/.github/workflows/cross-platform-compare.yml @@ -0,0 +1,185 @@ +# Copyright (C) 2026 QuasarApp Group DOO +# Licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0), +# see the accompanying LICENSE file. + +name: Cross-Platform Noise Benchmark + +# Runs the cross-platform benchmark (tests/test_OpenSimplex2F_cross_platform.c) +# on all three platforms - Linux, Windows and macOS - with the same seed and +# the same input parameters, waits for all three jobs to complete (needs), +# and then compares the produced result files byte-for-byte. The noise +# functions must return bit-identical results for the same input parameters, +# so any difference between the platforms fails the comparison job. +on: + pull_request: + types: [opened, synchronize, reopened] + # Allows maintainers to trigger a manual run (e.g. to test the pipeline). + workflow_dispatch: + +jobs: + benchmark: + name: Benchmark (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + platform: linux + - os: windows-latest + platform: windows + - os: macos-latest + platform: macos + + steps: + # The project is plain C (no Qt code), but the CMake project + # (submodule/CMake/QuasarApp.cmake) looks for qmake at configure time, + # so Qt must be installed on every platform (same setup as ci.yml). + - name: Install Qt (Windows) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'Windows' + with: + version: '6.10.3' # aqt has bug with latest qt versions in windows + target: 'desktop' + arch: 'win64_mingw' + cache: true + tools: 'tools_mingw1310' + aqtversion: ==3.3.* + + - name: Install Qt (Linux) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'Linux' + with: + version: '6.12.0' + target: 'desktop' + arch: 'linux_gcc_64' + cache: true + aqtversion: ==3.3.* + + - name: Install Qt (macOS) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'macOS' + with: + version: '6.10.3' + target: 'desktop' + arch: 'clang_64' + cache: true + aqtversion: ==3.3.* + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + # The project links against libpng (find_package(PNG REQUIRED) in + # tests/CMakeLists.txt), so it must be available on every platform. + - name: Install libpng (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libpng-dev + + # The aqt MinGW does not ship libpng, so install the MinGW libpng from + # MSYS2 and point CMake at it (same setup as ci.yml). + - name: Install libpng (Windows) + id: msys2 + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + install: mingw-w64-x86_64-libpng + + # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the + # compiled binaries would not find libpng16-16.dll (and libz-1.dll) at + # runtime and fail with 0xc0000135 (STATUS_DLL_NOT_FOUND). + - name: Add MSYS2 mingw64 bin to PATH (Windows) + if: runner.os == 'Windows' + run: | + Add-Content "$env:GITHUB_PATH" "${{ steps.msys2.outputs.msys2-location }}/mingw64/bin" + + # libpng is part of the Xcode command line tools on macOS, nothing to + # install. + + - name: Configure CMake (Windows) + if: runner.os == 'Windows' + run: > + cmake -B ${{ github.workspace }}/build + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=gcc + -DCMAKE_PREFIX_PATH=${{ steps.msys2.outputs.msys2-location }}/mingw64 + -S ${{ github.workspace }} + + - name: Configure CMake (Linux and macOS) + if: runner.os != 'Windows' + run: > + cmake -B ${{ github.workspace }}/build + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_PREFIX_PATH=${{ env.QT_ROOT_DIR }} + -S ${{ github.workspace }} + + - name: Build + run: cmake --build ${{ github.workspace }}/build --config Release --parallel 4 + # Runs the OpenSimplex2_cross_platform_benchmark target, which evaluates + # all noise functions with the same input parameters and writes + # noise_benchmark_results.txt into build/cross_platform. + - name: Run cross-platform benchmark + shell: bash + run: | + cmake --build ${{ github.workspace }}/build --target OpenSimplex2_cross_platform_benchmark --config Release + head -n 8 ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt + tail -n 2 ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt + + # Publish the platform report for the comparison job. + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: noise-benchmark-${{ matrix.platform }} + path: ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt + if-no-files-found: error + + compare: + name: Compare results (linux + windows + macos) + runs-on: ubuntu-latest + # Waits for the benchmark jobs of all three platforms to complete. + needs: benchmark + + steps: + # Downloads all uploaded reports into results//. + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + path: results + + - name: Compare the results byte-for-byte + run: | + set -euo pipefail + linux=results/noise-benchmark-linux/noise_benchmark_results.txt + windows=results/noise-benchmark-windows/noise_benchmark_results.txt + macos=results/noise-benchmark-macos/noise_benchmark_results.txt + for f in "$linux" "$windows" "$macos"; do + [ -s "$f" ] || { echo "FAIL: missing or empty result file: $f"; exit 1; } + done + echo "--- FNV-1a report checksums ---" + grep '^checksum=' "$linux" "$windows" "$macos" + echo "--- sha256 of the complete report files ---" + sha256sum "$linux" "$windows" "$macos" + echo "--- head of the report ---" + head -n 8 "$linux" + echo "--- tail of the report ---" + tail -n 2 "$linux" + if ! cmp -s "$linux" "$windows"; then + echo "FAIL: the linux and windows noise results differ" + diff -u "$linux" "$windows" | head -n 50 || true + exit 1 + fi + if ! cmp -s "$linux" "$macos"; then + echo "FAIL: the linux and macos noise results differ" + diff -u "$linux" "$macos" | head -n 50 || true + exit 1 + fi + echo "OK: all three platforms produced bit-identical noise results for the same input parameters" + diff --git a/CMakeLists.txt b/CMakeLists.txt index d42ad59..8e082ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,18 @@ target_compile_options(${CURRENT_PROJECT}C PRIVATE -Wall -Wextra --pedantic) target_link_libraries(${CURRENT_PROJECT}C PRIVATE m) target_link_libraries(${CURRENT_PROJECT} INTERFACE ${CURRENT_PROJECT}C) +# The cross-platform benchmark (see tests/test_OpenSimplex2F_cross_platform.c +# and .github/workflows/cross-platform-compare.yml) compares the noise output +# of the library byte-for-byte between Linux, Windows and macOS. The noise +# code runs inside this target, so the same floating point guarantees apply: +# arm64 compilers (Apple Silicon) may contract a*b+c into FMA instructions, +# which changes the least significant bits of the results. Disable FMA for +# the GNU/Clang builds (on x86-64 this is a no-op) so that the same input +# parameters always produce bit-identical noise values on every platform. +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${CURRENT_PROJECT}C PRIVATE -mno-fma) +endif() + option(OPENSIMPLEX2_TESTS "This option disables or enables tests of the ${PROJECT_NAME} project" ON) initAll() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 46f59b5..4a37413 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,3 +21,38 @@ target_link_libraries(${CURRENT_PROJECT} PRIVATE OpenSimplex2Interface m PNG::PN initTests() addTests(${PROJECT_NAME} ${CURRENT_PROJECT}) + +# Cross-platform noise benchmark (issue #31): evaluates every OpenSimplex2F +# noise function with the same seed and the same input parameters on every +# platform and prints the exact binary64 bit pattern of every result. The +# .github/workflows/cross-platform-compare.yml workflow runs it on Linux, +# Windows and macOS and compares the produced report byte-for-byte. +set(CROSS_PLATFORM_BENCHMARK "${PROJECT_NAME}_cross_platform") + +add_executable(${CROSS_PLATFORM_BENCHMARK} + "${CMAKE_CURRENT_SOURCE_DIR}/test_OpenSimplex2F_cross_platform.c" +) +target_include_directories(${CROSS_PLATFORM_BENCHMARK} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") +target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -Wall -Wextra --pedantic) +target_link_libraries(${CROSS_PLATFORM_BENCHMARK} PRIVATE OpenSimplex2Interface m) + +# The benchmark report must be bit-identical on all platforms, but arm64 +# compilers (Apple Silicon) may contract a*b+c into FMA instructions, which +# changes the least significant bits of the results. Disable FMA for the +# GNU/Clang builds (on x86-64 this is a no-op). +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -mno-fma) +endif() + +# The cross-platform comparison workflow +# (.github/workflows/cross-platform-compare.yml) builds and runs this target +# on Linux, Windows and macOS. It is intentionally not part of the QuasarApp +# `test` target, so no cqtdeployer deployment is required. The benchmark +# writes noise_benchmark_results.txt into the build/cross_platform working +# directory (the executable creates it). +add_custom_target(${PROJECT_NAME}_cross_platform_benchmark + COMMAND $ "${CMAKE_BINARY_DIR}/cross_platform/noise_benchmark_results.txt" + DEPENDS ${CROSS_PLATFORM_BENCHMARK} + COMMENT "Running the OpenSimplex2F cross-platform noise benchmark" + VERBATIM) + diff --git a/tests/test_OpenSimplex2F_cross_platform.c b/tests/test_OpenSimplex2F_cross_platform.c new file mode 100644 index 0000000..b07bd80 --- /dev/null +++ b/tests/test_OpenSimplex2F_cross_platform.c @@ -0,0 +1,364 @@ +/* + * Copyright (C) 2026 QuasarApp Group DOO + * Licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0), + * see the accompanying LICENSE file. + */ + +/* + * Cross-platform benchmark for all OpenSimplex2F noise functions. + * + * Evaluates every public noise function (2D, 3D, 4D, all variants) with the + * same seed and the same fixed input parameters on every platform, and + * prints the exact IEEE-754 binary64 bit pattern (16 hex digits) of every + * result. The report is plain, platform-independent ASCII text, so the + * result files produced by the CI jobs on Linux, Windows and macOS can be + * compared byte-for-byte: given the same input parameters the noise results + * must match perfectly on all platforms (see + * .github/workflows/cross-platform-compare.yml). + * + * The report is written to noise_benchmark_results.txt in the current + * working directory (or to the path given as the first command line + * argument, whose parent directory is created when missing) and printed + * to stdout as well. + * + * Exit code: 0 if the report was written and every noise value is inside + * the defined range [-1, 1], otherwise 1. + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#define MAKE_DIR(path) _mkdir(path) +#else +#include +#include +#define MAKE_DIR(path) mkdir((path), 0755) +#endif + +#include "OpenSimplex2F.h" + +#define BENCHMARK_SEED 77374LL /* Same seed as tests/test_OpenSimplex2F.c. */ +#define RESULT_FILE "noise_benchmark_results.txt" + +#define NOISE_LIMIT 1.0 +#define NOISE_EPSILON 1e-6 + +/* FNV-1a 64-bit: one deterministic checksum summarizing the whole report. */ +static uint64_t fnv1a64(uint64_t hash, const char *data, size_t length) +{ + size_t i; + for (i = 0; i < length; i++) + hash = (hash ^ (uint8_t) data[i]) * 0x100000001B3ULL; + return hash; +} + +/* Exact IEEE-754 binary64 representation of a double. All CI platforms + * (x86-64 Linux, x86-64 Windows, arm64 macOS) use binary64 doubles, so the + * bit pattern identifies the result unambiguously. */ +static uint64_t double_bits(double value) +{ + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* The report is assembled in a heap buffer so it can be written to stdout + * and to the result file with exactly the same bytes. */ +struct report { + char *data; + size_t length; + size_t capacity; + int failed; +}; + +static void report_put(struct report *r, const char *text, size_t length) +{ + if (r->failed) + return; + if (r->length + length + 1 > r->capacity) { + size_t new_capacity = r->capacity ? r->capacity * 2 : (1u << 20); + char *data; + while (r->length + length + 1 > new_capacity) + new_capacity *= 2; + data = realloc(r->data, new_capacity); + if (!data) { + fprintf(stderr, "FAIL: out of memory\n"); + r->failed = 1; + return; + } + r->data = data; + r->capacity = new_capacity; + } + memcpy(r->data + r->length, text, length); + r->length += length; + r->data[r->length] = '\0'; +} + +static void report_printf(struct report *r, const char *format, ...) +{ + char line[256]; + va_list args; + int length; + + va_start(args, format); + length = vsnprintf(line, sizeof(line), format, args); + va_end(args); + + if (length < 0) { + fprintf(stderr, "FAIL: vsnprintf failed\n"); + r->failed = 1; + return; + } + if ((size_t) length >= sizeof(line)) + length = (int) sizeof(line) - 1; + report_put(r, line, (size_t) length); +} + +/* Every noise function from the public header, grouped by dimension. */ +typedef double (*noise2_fn)(struct OpenSimplex2F_context *, double, double); +typedef double (*noise3_fn)(struct OpenSimplex2F_context *, double, double, double); +typedef double (*noise4_fn)(struct OpenSimplex2F_context *, double, double, double, double); + +static const struct { + const char *name; + noise2_fn fn; +} noise2_functions[] = { + { "OpenSimplex2F_noise2", OpenSimplex2F_noise2 }, + { "OpenSimplex2F_noise2_XBeforeY", OpenSimplex2F_noise2_XBeforeY }, +}; + +static const struct { + const char *name; + noise3_fn fn; +} noise3_functions[] = { + { "OpenSimplex2F_noise3_Classic", OpenSimplex2F_noise3_Classic }, + { "OpenSimplex2F_noise3_XYBeforeZ", OpenSimplex2F_noise3_XYBeforeZ }, + { "OpenSimplex2F_noise3_XZBeforeY", OpenSimplex2F_noise3_XZBeforeY }, +}; + +static const struct { + const char *name; + noise4_fn fn; +} noise4_functions[] = { + { "OpenSimplex2F_noise4_Classic", OpenSimplex2F_noise4_Classic }, + { "OpenSimplex2F_noise4_XYBeforeZW", OpenSimplex2F_noise4_XYBeforeZW }, + { "OpenSimplex2F_noise4_XZBeforeYW", OpenSimplex2F_noise4_XZBeforeYW }, + { "OpenSimplex2F_noise4_XYZBeforeW", OpenSimplex2F_noise4_XYZBeforeW }, +}; + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + +/* The frequency scales multiply the base coordinates: they place the sample + * points in different lattice cells, so every branch of the noise functions + * is exercised. */ +static const double scales[] = { 1.0, 2.3, 5.7 }; + +static int value_in_range(double value) +{ + return (value >= -NOISE_LIMIT - NOISE_EPSILON) && + (value <= NOISE_LIMIT + NOISE_EPSILON); +} + +static void report_value(struct report *r, const char *name, double scale, + double value, const char *coords) +{ + if (!value_in_range(value)) { + fprintf(stderr, "FAIL: %s (scale=%f, %s) returned %f outside the expected range [-1, 1]\n", + name, scale, coords, value); + r->failed = 1; + } + report_printf(r, "%s scale=%f %s bits=%016" PRIx64 "\n", + name, scale, coords, double_bits(value)); +} + +static void benchmark2(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, j, i, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (j = 0; j <= 16; j++) { + for (i = 0; i <= 16; i++) { + /* i / 8.0 - 1.0 is exact in binary64, so the + * input parameters are bit-identical on every + * platform. */ + double y = (double) j / 8.0 - 1.0; + double sy = y * scales[s]; + for (f = 0; f < ARRAY_SIZE(noise2_functions); f++) { + double x = (double) i / 8.0 - 1.0; + double sx = x * scales[s]; + char coords[96]; + snprintf(coords, sizeof(coords), "x=%f y=%f", sx, sy); + report_value(r, noise2_functions[f].name, scales[s], + noise2_functions[f].fn(ctx, sx, sy), coords); + } + } + } + } +} + +static void benchmark3(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, k, j, i, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (k = 0; k <= 8; k++) { + for (j = 0; j <= 8; j++) { + for (i = 0; i <= 8; i++) { + double z = (double) k / 8.0 - 1.0; + double y = (double) j / 8.0 - 1.0; + double x = (double) i / 8.0 - 1.0; + double sz = z * scales[s]; + double sy = y * scales[s]; + double sx = x * scales[s]; + char coords[128]; + snprintf(coords, sizeof(coords), "x=%f y=%f z=%f", sx, sy, sz); + for (f = 0; f < ARRAY_SIZE(noise3_functions); f++) + report_value(r, noise3_functions[f].name, scales[s], + noise3_functions[f].fn(ctx, sx, sy, sz), coords); + } + } + } + } +} + +static void benchmark4(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, w, z, y, x, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (w = 0; w <= 6; w++) { + for (z = 0; z <= 6; z++) { + for (y = 0; y <= 6; y++) { + for (x = 0; x <= 6; x++) { + double dw = (double) w / 4.0 - 1.0; + double dz = (double) z / 4.0 - 1.0; + double dy = (double) y / 4.0 - 1.0; + double dx = (double) x / 4.0 - 1.0; + double sw = dw * scales[s]; + double sz = dz * scales[s]; + double sy = dy * scales[s]; + double sx = dx * scales[s]; + char coords[160]; + snprintf(coords, sizeof(coords), + "x=%f y=%f z=%f w=%f", sx, sy, sz, sw); + for (f = 0; f < ARRAY_SIZE(noise4_functions); f++) + report_value(r, noise4_functions[f].name, scales[s], + noise4_functions[f].fn(ctx, sx, sy, sz, sw), coords); + } + } + } + } + } +} + +/* Create the parent directory of path when it is missing. */ +static int make_parent_dir(const char *path) +{ + char *dir = strdup(path); + char *slash; + + if (!dir) + return -1; + slash = strrchr(dir, '/'); +#ifdef _WIN32 + if (!slash) + slash = strrchr(dir, '\\'); +#endif + if (slash && *slash != '\0') { + *slash = '\0'; + if (dir[0] != '\0' && MAKE_DIR(dir) != 0 && errno != EEXIST) { + int e = errno; + free(dir); + errno = e; + return -1; + } + } else if (!slash) { + free(dir); + return 0; /* no directory part, nothing to create */ + } + free(dir); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct OpenSimplex2F_context *ctx = NULL; + struct report report; + const char *result_file = RESULT_FILE; + uint64_t values2, values3, values4, values; + uint64_t checksum; + FILE *file; + int result = 0; + + if (argc > 1 && argv[1][0] != '\0') + result_file = argv[1]; + + values2 = ARRAY_SIZE(noise2_functions) * 17u * 17u * ARRAY_SIZE(scales); + values3 = ARRAY_SIZE(noise3_functions) * 9u * 9u * 9u * ARRAY_SIZE(scales); + values4 = ARRAY_SIZE(noise4_functions) * 7u * 7u * 7u * 7u * ARRAY_SIZE(scales); + values = values2 + values3 + values4; + + memset(&report, 0, sizeof(report)); + + if (OpenSimplex2F(BENCHMARK_SEED, &ctx) != 0) { + fprintf(stderr, "FAIL: OpenSimplex2F(%" PRId64 ") failed to create the context\n", + (int64_t) BENCHMARK_SEED); + return 1; + } + + report_printf(&report, "OpenSimplex2F cross-platform benchmark\n"); + report_printf(&report, "seed=%" PRId64 "\n", (int64_t) BENCHMARK_SEED); + + benchmark2(&report, ctx); + benchmark3(&report, ctx); + benchmark4(&report, ctx); + report_printf(&report, + "values2=%" PRIu64 " values3=%" PRIu64 " values4=%" PRIu64 + " values=%" PRIu64 "\n", + (uint64_t) values2, (uint64_t) values3, (uint64_t) values4, + (uint64_t) values); + + if (report.failed) + result = 1; + + checksum = fnv1a64(0xcbf29ce484222325ULL, report.data, report.length); + if (!result) + report_printf(&report, "checksum=%016" PRIx64 "\n", checksum); + + /* Always print the full report so the CI log shows the results. */ + fputs(report.data, stdout); + fflush(stdout); + + if (make_parent_dir(result_file) != 0) { + fprintf(stderr, "FAIL: mkdir(parent of %s): %s\n", result_file, strerror(errno)); + result = 1; + } + + file = fopen(result_file, "w"); + if (!file) { + fprintf(stderr, "FAIL: fopen(%s): %s\n", result_file, strerror(errno)); + result = 1; + } else { + if (fwrite(report.data, 1, report.length, file) != report.length) { + fprintf(stderr, "FAIL: write(%s): %s\n", result_file, strerror(errno)); + result = 1; + } + if (fclose(file) != 0) { + fprintf(stderr, "FAIL: fclose(%s): %s\n", result_file, strerror(errno)); + result = 1; + } + } + + OpenSimplex2F_free(ctx); + OpenSimplex2F_shutdown(); + free(report.data); + + printf("Result: %s (%" PRIu64 " values, seed %" PRId64 ")\n", + result ? "FAIL" : "PASS", (uint64_t) values, (int64_t) BENCHMARK_SEED); + return result; +} + From 579453b6dacb36dfe874264df277cf6370647b1c Mon Sep 17 00:00:00 2001 From: quasar Date: Thu, 12 Nov 2026 17:32:12 +0000 Subject: [PATCH 2/2] Fix cross-platform benchmark CI failures and CodeQL findings (issue 31) The first CI run (43130583493) of the Cross-Platform Noise Benchmark workflow failed on the windows-latest and macos-latest benchmark jobs (compare skipped): - macos-latest: Apple Clang rejects -mno-fma (unsupported option for target arm64-apple-darwin), so the FMA guard broke the build on the only platform where it was needed. Use the portable -ffp-contract=off (supported by both GCC and Clang, no-op on x86-64) for the library and the benchmark target. - windows-latest: the 'Run cross-platform benchmark' step runs under bash (Git Bash), where the Windows workspace path (D:\a\...) is mangled by backslash escaping ('D:/a/.../aOpenSimplex2OpenSimplex2/build is not a directory'). Normalize GITHUB_WORKSPACE to forward slashes first (a no-op on Linux and macOS). - The report was written with fopen(path, w) (text mode), which converts every newline to CRLF in the Windows C runtime and would have broken the byte-for-byte comparison even after the path fix. Write in binary mode ("wb") so the report is byte-identical on all three platforms. - CodeQL ('Workflow does not contain permissions'): limit the GITHUB_TOKEN of the benchmark job to contents:read + actions:write (checkout + upload-artifact) and the compare job to actions:read (download-artifact). Verified locally (Linux x86-64, GCC 13, Qt 6.12.0 prefix): build + benchmark PASS with 37107 values and checksum bb84b67c9c4e1e11 (unchanged by the flag swap, matching the ubuntu CI job), re-run byte-identical, PNG test target 3/3 passed, and the compare script passes with three identical reports and fails on a tampered one. --- .github/workflows/cross-platform-compare.yml | 21 +++++++++++++++++--- CMakeLists.txt | 10 ++++++---- tests/CMakeLists.txt | 8 +++++--- tests/test_OpenSimplex2F_cross_platform.c | 5 ++++- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cross-platform-compare.yml b/.github/workflows/cross-platform-compare.yml index fefde78..c4dc090 100644 --- a/.github/workflows/cross-platform-compare.yml +++ b/.github/workflows/cross-platform-compare.yml @@ -20,6 +20,12 @@ jobs: benchmark: name: Benchmark (${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Limit the GITHUB_TOKEN to what this job needs (CodeQL: "Workflow does + # not contain permissions"): contents: read for the checkout, + # actions: write for uploading the benchmark report artifact. + permissions: + contents: read + actions: write strategy: fail-fast: false @@ -129,9 +135,14 @@ jobs: - name: Run cross-platform benchmark shell: bash run: | - cmake --build ${{ github.workspace }}/build --target OpenSimplex2_cross_platform_benchmark --config Release - head -n 8 ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt - tail -n 2 ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt + # The Windows workspace path (D:\a\...) contains backslashes, which + # the bash shell (Git Bash on the windows runner) would interpret as + # escape characters and mangle the path. Normalize the workspace + # path to forward slashes (a no-op on Linux and macOS). + ws="${GITHUB_WORKSPACE//\\//}" + cmake --build "$ws/build" --target OpenSimplex2_cross_platform_benchmark --config Release + head -n 8 "$ws/build/cross_platform/noise_benchmark_results.txt" + tail -n 2 "$ws/build/cross_platform/noise_benchmark_results.txt" # Publish the platform report for the comparison job. - name: Upload benchmark results @@ -144,6 +155,10 @@ jobs: compare: name: Compare results (linux + windows + macos) runs-on: ubuntu-latest + # Limit the GITHUB_TOKEN to what this job needs: actions: read for + # downloading the benchmark report artifacts. + permissions: + actions: read # Waits for the benchmark jobs of all three platforms to complete. needs: benchmark diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e082ff..4a27d96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,11 +36,13 @@ target_link_libraries(${CURRENT_PROJECT} INTERFACE ${CURRENT_PROJECT}C) # of the library byte-for-byte between Linux, Windows and macOS. The noise # code runs inside this target, so the same floating point guarantees apply: # arm64 compilers (Apple Silicon) may contract a*b+c into FMA instructions, -# which changes the least significant bits of the results. Disable FMA for -# the GNU/Clang builds (on x86-64 this is a no-op) so that the same input -# parameters always produce bit-identical noise values on every platform. +# which changes the least significant bits of the results. Disable FMA +# contraction for the GNU/Clang builds (on x86-64 this is a no-op) so that +# the same input parameters always produce bit-identical noise values on +# every platform. -ffp-contract=off is supported by both GCC and Clang +# (-mno-fma is rejected by Apple Clang on arm64 macOS). if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(${CURRENT_PROJECT}C PRIVATE -mno-fma) + target_compile_options(${CURRENT_PROJECT}C PRIVATE -ffp-contract=off) endif() option(OPENSIMPLEX2_TESTS "This option disables or enables tests of the ${PROJECT_NAME} project" ON) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4a37413..0f46d7f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,10 +38,12 @@ target_link_libraries(${CROSS_PLATFORM_BENCHMARK} PRIVATE OpenSimplex2Interface # The benchmark report must be bit-identical on all platforms, but arm64 # compilers (Apple Silicon) may contract a*b+c into FMA instructions, which -# changes the least significant bits of the results. Disable FMA for the -# GNU/Clang builds (on x86-64 this is a no-op). +# changes the least significant bits of the results. Disable FMA contraction +# for the GNU/Clang builds (on x86-64 this is a no-op). -ffp-contract=off is +# supported by both GCC and Clang (-mno-fma is rejected by Apple Clang on +# arm64 macOS). if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -mno-fma) + target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -ffp-contract=off) endif() # The cross-platform comparison workflow diff --git a/tests/test_OpenSimplex2F_cross_platform.c b/tests/test_OpenSimplex2F_cross_platform.c index b07bd80..51bd086 100644 --- a/tests/test_OpenSimplex2F_cross_platform.c +++ b/tests/test_OpenSimplex2F_cross_platform.c @@ -338,7 +338,10 @@ int main(int argc, char *argv[]) result = 1; } - file = fopen(result_file, "w"); + /* Binary mode: the report must be byte-for-byte identical on every + * platform, but in text mode the Windows C runtime converts every "\n" + * to "\r\n" on write, which would break the byte-for-byte comparison. */ + file = fopen(result_file, "wb"); if (!file) { fprintf(stderr, "FAIL: fopen(%s): %s\n", result_file, strerror(errno)); result = 1;