diff --git a/.github/workflows/cross-platform-compare.yml b/.github/workflows/cross-platform-compare.yml new file mode 100644 index 0000000..c4dc090 --- /dev/null +++ b/.github/workflows/cross-platform-compare.yml @@ -0,0 +1,200 @@ +# Copyright (C) 2026 QuasarApp Group DOO +# Licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0), +# see the accompanying LICENSE file. + +name: Cross-Platform Noise Benchmark + +# Runs the cross-platform benchmark (tests/test_OpenSimplex2F_cross_platform.c) +# on all three platforms - Linux, Windows and macOS - with the same seed and +# the same input parameters, waits for all three jobs to complete (needs), +# and then compares the produced result files byte-for-byte. The noise +# functions must return bit-identical results for the same input parameters, +# so any difference between the platforms fails the comparison job. +on: + pull_request: + types: [opened, synchronize, reopened] + # Allows maintainers to trigger a manual run (e.g. to test the pipeline). + workflow_dispatch: + +jobs: + benchmark: + name: Benchmark (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + # Limit the GITHUB_TOKEN to what this job needs (CodeQL: "Workflow does + # not contain permissions"): contents: read for the checkout, + # actions: write for uploading the benchmark report artifact. + permissions: + contents: read + actions: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + platform: linux + - os: windows-latest + platform: windows + - os: macos-latest + platform: macos + + steps: + # The project is plain C (no Qt code), but the CMake project + # (submodule/CMake/QuasarApp.cmake) looks for qmake at configure time, + # so Qt must be installed on every platform (same setup as ci.yml). + - name: Install Qt (Windows) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'Windows' + with: + version: '6.10.3' # aqt has bug with latest qt versions in windows + target: 'desktop' + arch: 'win64_mingw' + cache: true + tools: 'tools_mingw1310' + aqtversion: ==3.3.* + + - name: Install Qt (Linux) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'Linux' + with: + version: '6.12.0' + target: 'desktop' + arch: 'linux_gcc_64' + cache: true + aqtversion: ==3.3.* + + - name: Install Qt (macOS) + uses: jurplel/install-qt-action@v4.3.1 + if: runner.os == 'macOS' + with: + version: '6.10.3' + target: 'desktop' + arch: 'clang_64' + cache: true + aqtversion: ==3.3.* + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + # The project links against libpng (find_package(PNG REQUIRED) in + # tests/CMakeLists.txt), so it must be available on every platform. + - name: Install libpng (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libpng-dev + + # The aqt MinGW does not ship libpng, so install the MinGW libpng from + # MSYS2 and point CMake at it (same setup as ci.yml). + - name: Install libpng (Windows) + id: msys2 + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + install: mingw-w64-x86_64-libpng + + # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the + # compiled binaries would not find libpng16-16.dll (and libz-1.dll) at + # runtime and fail with 0xc0000135 (STATUS_DLL_NOT_FOUND). + - name: Add MSYS2 mingw64 bin to PATH (Windows) + if: runner.os == 'Windows' + run: | + Add-Content "$env:GITHUB_PATH" "${{ steps.msys2.outputs.msys2-location }}/mingw64/bin" + + # libpng is part of the Xcode command line tools on macOS, nothing to + # install. + + - name: Configure CMake (Windows) + if: runner.os == 'Windows' + run: > + cmake -B ${{ github.workspace }}/build + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=gcc + -DCMAKE_PREFIX_PATH=${{ steps.msys2.outputs.msys2-location }}/mingw64 + -S ${{ github.workspace }} + + - name: Configure CMake (Linux and macOS) + if: runner.os != 'Windows' + run: > + cmake -B ${{ github.workspace }}/build + -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_PREFIX_PATH=${{ env.QT_ROOT_DIR }} + -S ${{ github.workspace }} + + - name: Build + run: cmake --build ${{ github.workspace }}/build --config Release --parallel 4 + # Runs the OpenSimplex2_cross_platform_benchmark target, which evaluates + # all noise functions with the same input parameters and writes + # noise_benchmark_results.txt into build/cross_platform. + - name: Run cross-platform benchmark + shell: bash + run: | + # The Windows workspace path (D:\a\...) contains backslashes, which + # the bash shell (Git Bash on the windows runner) would interpret as + # escape characters and mangle the path. Normalize the workspace + # path to forward slashes (a no-op on Linux and macOS). + ws="${GITHUB_WORKSPACE//\\//}" + cmake --build "$ws/build" --target OpenSimplex2_cross_platform_benchmark --config Release + head -n 8 "$ws/build/cross_platform/noise_benchmark_results.txt" + tail -n 2 "$ws/build/cross_platform/noise_benchmark_results.txt" + + # Publish the platform report for the comparison job. + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: noise-benchmark-${{ matrix.platform }} + path: ${{ github.workspace }}/build/cross_platform/noise_benchmark_results.txt + if-no-files-found: error + + compare: + name: Compare results (linux + windows + macos) + runs-on: ubuntu-latest + # Limit the GITHUB_TOKEN to what this job needs: actions: read for + # downloading the benchmark report artifacts. + permissions: + actions: read + # Waits for the benchmark jobs of all three platforms to complete. + needs: benchmark + + steps: + # Downloads all uploaded reports into results//. + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + path: results + + - name: Compare the results byte-for-byte + run: | + set -euo pipefail + linux=results/noise-benchmark-linux/noise_benchmark_results.txt + windows=results/noise-benchmark-windows/noise_benchmark_results.txt + macos=results/noise-benchmark-macos/noise_benchmark_results.txt + for f in "$linux" "$windows" "$macos"; do + [ -s "$f" ] || { echo "FAIL: missing or empty result file: $f"; exit 1; } + done + echo "--- FNV-1a report checksums ---" + grep '^checksum=' "$linux" "$windows" "$macos" + echo "--- sha256 of the complete report files ---" + sha256sum "$linux" "$windows" "$macos" + echo "--- head of the report ---" + head -n 8 "$linux" + echo "--- tail of the report ---" + tail -n 2 "$linux" + if ! cmp -s "$linux" "$windows"; then + echo "FAIL: the linux and windows noise results differ" + diff -u "$linux" "$windows" | head -n 50 || true + exit 1 + fi + if ! cmp -s "$linux" "$macos"; then + echo "FAIL: the linux and macos noise results differ" + diff -u "$linux" "$macos" | head -n 50 || true + exit 1 + fi + echo "OK: all three platforms produced bit-identical noise results for the same input parameters" + diff --git a/CMakeLists.txt b/CMakeLists.txt index d42ad59..4a27d96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,20 @@ target_compile_options(${CURRENT_PROJECT}C PRIVATE -Wall -Wextra --pedantic) target_link_libraries(${CURRENT_PROJECT}C PRIVATE m) target_link_libraries(${CURRENT_PROJECT} INTERFACE ${CURRENT_PROJECT}C) +# The cross-platform benchmark (see tests/test_OpenSimplex2F_cross_platform.c +# and .github/workflows/cross-platform-compare.yml) compares the noise output +# of the library byte-for-byte between Linux, Windows and macOS. The noise +# code runs inside this target, so the same floating point guarantees apply: +# arm64 compilers (Apple Silicon) may contract a*b+c into FMA instructions, +# which changes the least significant bits of the results. Disable FMA +# contraction for the GNU/Clang builds (on x86-64 this is a no-op) so that +# the same input parameters always produce bit-identical noise values on +# every platform. -ffp-contract=off is supported by both GCC and Clang +# (-mno-fma is rejected by Apple Clang on arm64 macOS). +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${CURRENT_PROJECT}C PRIVATE -ffp-contract=off) +endif() + option(OPENSIMPLEX2_TESTS "This option disables or enables tests of the ${PROJECT_NAME} project" ON) initAll() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 46f59b5..0f46d7f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,3 +21,40 @@ target_link_libraries(${CURRENT_PROJECT} PRIVATE OpenSimplex2Interface m PNG::PN initTests() addTests(${PROJECT_NAME} ${CURRENT_PROJECT}) + +# Cross-platform noise benchmark (issue #31): evaluates every OpenSimplex2F +# noise function with the same seed and the same input parameters on every +# platform and prints the exact binary64 bit pattern of every result. The +# .github/workflows/cross-platform-compare.yml workflow runs it on Linux, +# Windows and macOS and compares the produced report byte-for-byte. +set(CROSS_PLATFORM_BENCHMARK "${PROJECT_NAME}_cross_platform") + +add_executable(${CROSS_PLATFORM_BENCHMARK} + "${CMAKE_CURRENT_SOURCE_DIR}/test_OpenSimplex2F_cross_platform.c" +) +target_include_directories(${CROSS_PLATFORM_BENCHMARK} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") +target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -Wall -Wextra --pedantic) +target_link_libraries(${CROSS_PLATFORM_BENCHMARK} PRIVATE OpenSimplex2Interface m) + +# The benchmark report must be bit-identical on all platforms, but arm64 +# compilers (Apple Silicon) may contract a*b+c into FMA instructions, which +# changes the least significant bits of the results. Disable FMA contraction +# for the GNU/Clang builds (on x86-64 this is a no-op). -ffp-contract=off is +# supported by both GCC and Clang (-mno-fma is rejected by Apple Clang on +# arm64 macOS). +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${CROSS_PLATFORM_BENCHMARK} PRIVATE -ffp-contract=off) +endif() + +# The cross-platform comparison workflow +# (.github/workflows/cross-platform-compare.yml) builds and runs this target +# on Linux, Windows and macOS. It is intentionally not part of the QuasarApp +# `test` target, so no cqtdeployer deployment is required. The benchmark +# writes noise_benchmark_results.txt into the build/cross_platform working +# directory (the executable creates it). +add_custom_target(${PROJECT_NAME}_cross_platform_benchmark + COMMAND $ "${CMAKE_BINARY_DIR}/cross_platform/noise_benchmark_results.txt" + DEPENDS ${CROSS_PLATFORM_BENCHMARK} + COMMENT "Running the OpenSimplex2F cross-platform noise benchmark" + VERBATIM) + diff --git a/tests/test_OpenSimplex2F_cross_platform.c b/tests/test_OpenSimplex2F_cross_platform.c new file mode 100644 index 0000000..51bd086 --- /dev/null +++ b/tests/test_OpenSimplex2F_cross_platform.c @@ -0,0 +1,367 @@ +/* + * Copyright (C) 2026 QuasarApp Group DOO + * Licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0), + * see the accompanying LICENSE file. + */ + +/* + * Cross-platform benchmark for all OpenSimplex2F noise functions. + * + * Evaluates every public noise function (2D, 3D, 4D, all variants) with the + * same seed and the same fixed input parameters on every platform, and + * prints the exact IEEE-754 binary64 bit pattern (16 hex digits) of every + * result. The report is plain, platform-independent ASCII text, so the + * result files produced by the CI jobs on Linux, Windows and macOS can be + * compared byte-for-byte: given the same input parameters the noise results + * must match perfectly on all platforms (see + * .github/workflows/cross-platform-compare.yml). + * + * The report is written to noise_benchmark_results.txt in the current + * working directory (or to the path given as the first command line + * argument, whose parent directory is created when missing) and printed + * to stdout as well. + * + * Exit code: 0 if the report was written and every noise value is inside + * the defined range [-1, 1], otherwise 1. + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#define MAKE_DIR(path) _mkdir(path) +#else +#include +#include +#define MAKE_DIR(path) mkdir((path), 0755) +#endif + +#include "OpenSimplex2F.h" + +#define BENCHMARK_SEED 77374LL /* Same seed as tests/test_OpenSimplex2F.c. */ +#define RESULT_FILE "noise_benchmark_results.txt" + +#define NOISE_LIMIT 1.0 +#define NOISE_EPSILON 1e-6 + +/* FNV-1a 64-bit: one deterministic checksum summarizing the whole report. */ +static uint64_t fnv1a64(uint64_t hash, const char *data, size_t length) +{ + size_t i; + for (i = 0; i < length; i++) + hash = (hash ^ (uint8_t) data[i]) * 0x100000001B3ULL; + return hash; +} + +/* Exact IEEE-754 binary64 representation of a double. All CI platforms + * (x86-64 Linux, x86-64 Windows, arm64 macOS) use binary64 doubles, so the + * bit pattern identifies the result unambiguously. */ +static uint64_t double_bits(double value) +{ + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* The report is assembled in a heap buffer so it can be written to stdout + * and to the result file with exactly the same bytes. */ +struct report { + char *data; + size_t length; + size_t capacity; + int failed; +}; + +static void report_put(struct report *r, const char *text, size_t length) +{ + if (r->failed) + return; + if (r->length + length + 1 > r->capacity) { + size_t new_capacity = r->capacity ? r->capacity * 2 : (1u << 20); + char *data; + while (r->length + length + 1 > new_capacity) + new_capacity *= 2; + data = realloc(r->data, new_capacity); + if (!data) { + fprintf(stderr, "FAIL: out of memory\n"); + r->failed = 1; + return; + } + r->data = data; + r->capacity = new_capacity; + } + memcpy(r->data + r->length, text, length); + r->length += length; + r->data[r->length] = '\0'; +} + +static void report_printf(struct report *r, const char *format, ...) +{ + char line[256]; + va_list args; + int length; + + va_start(args, format); + length = vsnprintf(line, sizeof(line), format, args); + va_end(args); + + if (length < 0) { + fprintf(stderr, "FAIL: vsnprintf failed\n"); + r->failed = 1; + return; + } + if ((size_t) length >= sizeof(line)) + length = (int) sizeof(line) - 1; + report_put(r, line, (size_t) length); +} + +/* Every noise function from the public header, grouped by dimension. */ +typedef double (*noise2_fn)(struct OpenSimplex2F_context *, double, double); +typedef double (*noise3_fn)(struct OpenSimplex2F_context *, double, double, double); +typedef double (*noise4_fn)(struct OpenSimplex2F_context *, double, double, double, double); + +static const struct { + const char *name; + noise2_fn fn; +} noise2_functions[] = { + { "OpenSimplex2F_noise2", OpenSimplex2F_noise2 }, + { "OpenSimplex2F_noise2_XBeforeY", OpenSimplex2F_noise2_XBeforeY }, +}; + +static const struct { + const char *name; + noise3_fn fn; +} noise3_functions[] = { + { "OpenSimplex2F_noise3_Classic", OpenSimplex2F_noise3_Classic }, + { "OpenSimplex2F_noise3_XYBeforeZ", OpenSimplex2F_noise3_XYBeforeZ }, + { "OpenSimplex2F_noise3_XZBeforeY", OpenSimplex2F_noise3_XZBeforeY }, +}; + +static const struct { + const char *name; + noise4_fn fn; +} noise4_functions[] = { + { "OpenSimplex2F_noise4_Classic", OpenSimplex2F_noise4_Classic }, + { "OpenSimplex2F_noise4_XYBeforeZW", OpenSimplex2F_noise4_XYBeforeZW }, + { "OpenSimplex2F_noise4_XZBeforeYW", OpenSimplex2F_noise4_XZBeforeYW }, + { "OpenSimplex2F_noise4_XYZBeforeW", OpenSimplex2F_noise4_XYZBeforeW }, +}; + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + +/* The frequency scales multiply the base coordinates: they place the sample + * points in different lattice cells, so every branch of the noise functions + * is exercised. */ +static const double scales[] = { 1.0, 2.3, 5.7 }; + +static int value_in_range(double value) +{ + return (value >= -NOISE_LIMIT - NOISE_EPSILON) && + (value <= NOISE_LIMIT + NOISE_EPSILON); +} + +static void report_value(struct report *r, const char *name, double scale, + double value, const char *coords) +{ + if (!value_in_range(value)) { + fprintf(stderr, "FAIL: %s (scale=%f, %s) returned %f outside the expected range [-1, 1]\n", + name, scale, coords, value); + r->failed = 1; + } + report_printf(r, "%s scale=%f %s bits=%016" PRIx64 "\n", + name, scale, coords, double_bits(value)); +} + +static void benchmark2(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, j, i, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (j = 0; j <= 16; j++) { + for (i = 0; i <= 16; i++) { + /* i / 8.0 - 1.0 is exact in binary64, so the + * input parameters are bit-identical on every + * platform. */ + double y = (double) j / 8.0 - 1.0; + double sy = y * scales[s]; + for (f = 0; f < ARRAY_SIZE(noise2_functions); f++) { + double x = (double) i / 8.0 - 1.0; + double sx = x * scales[s]; + char coords[96]; + snprintf(coords, sizeof(coords), "x=%f y=%f", sx, sy); + report_value(r, noise2_functions[f].name, scales[s], + noise2_functions[f].fn(ctx, sx, sy), coords); + } + } + } + } +} + +static void benchmark3(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, k, j, i, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (k = 0; k <= 8; k++) { + for (j = 0; j <= 8; j++) { + for (i = 0; i <= 8; i++) { + double z = (double) k / 8.0 - 1.0; + double y = (double) j / 8.0 - 1.0; + double x = (double) i / 8.0 - 1.0; + double sz = z * scales[s]; + double sy = y * scales[s]; + double sx = x * scales[s]; + char coords[128]; + snprintf(coords, sizeof(coords), "x=%f y=%f z=%f", sx, sy, sz); + for (f = 0; f < ARRAY_SIZE(noise3_functions); f++) + report_value(r, noise3_functions[f].name, scales[s], + noise3_functions[f].fn(ctx, sx, sy, sz), coords); + } + } + } + } +} + +static void benchmark4(struct report *r, struct OpenSimplex2F_context *ctx) +{ + size_t s, w, z, y, x, f; + for (s = 0; s < ARRAY_SIZE(scales); s++) { + for (w = 0; w <= 6; w++) { + for (z = 0; z <= 6; z++) { + for (y = 0; y <= 6; y++) { + for (x = 0; x <= 6; x++) { + double dw = (double) w / 4.0 - 1.0; + double dz = (double) z / 4.0 - 1.0; + double dy = (double) y / 4.0 - 1.0; + double dx = (double) x / 4.0 - 1.0; + double sw = dw * scales[s]; + double sz = dz * scales[s]; + double sy = dy * scales[s]; + double sx = dx * scales[s]; + char coords[160]; + snprintf(coords, sizeof(coords), + "x=%f y=%f z=%f w=%f", sx, sy, sz, sw); + for (f = 0; f < ARRAY_SIZE(noise4_functions); f++) + report_value(r, noise4_functions[f].name, scales[s], + noise4_functions[f].fn(ctx, sx, sy, sz, sw), coords); + } + } + } + } + } +} + +/* Create the parent directory of path when it is missing. */ +static int make_parent_dir(const char *path) +{ + char *dir = strdup(path); + char *slash; + + if (!dir) + return -1; + slash = strrchr(dir, '/'); +#ifdef _WIN32 + if (!slash) + slash = strrchr(dir, '\\'); +#endif + if (slash && *slash != '\0') { + *slash = '\0'; + if (dir[0] != '\0' && MAKE_DIR(dir) != 0 && errno != EEXIST) { + int e = errno; + free(dir); + errno = e; + return -1; + } + } else if (!slash) { + free(dir); + return 0; /* no directory part, nothing to create */ + } + free(dir); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct OpenSimplex2F_context *ctx = NULL; + struct report report; + const char *result_file = RESULT_FILE; + uint64_t values2, values3, values4, values; + uint64_t checksum; + FILE *file; + int result = 0; + + if (argc > 1 && argv[1][0] != '\0') + result_file = argv[1]; + + values2 = ARRAY_SIZE(noise2_functions) * 17u * 17u * ARRAY_SIZE(scales); + values3 = ARRAY_SIZE(noise3_functions) * 9u * 9u * 9u * ARRAY_SIZE(scales); + values4 = ARRAY_SIZE(noise4_functions) * 7u * 7u * 7u * 7u * ARRAY_SIZE(scales); + values = values2 + values3 + values4; + + memset(&report, 0, sizeof(report)); + + if (OpenSimplex2F(BENCHMARK_SEED, &ctx) != 0) { + fprintf(stderr, "FAIL: OpenSimplex2F(%" PRId64 ") failed to create the context\n", + (int64_t) BENCHMARK_SEED); + return 1; + } + + report_printf(&report, "OpenSimplex2F cross-platform benchmark\n"); + report_printf(&report, "seed=%" PRId64 "\n", (int64_t) BENCHMARK_SEED); + + benchmark2(&report, ctx); + benchmark3(&report, ctx); + benchmark4(&report, ctx); + report_printf(&report, + "values2=%" PRIu64 " values3=%" PRIu64 " values4=%" PRIu64 + " values=%" PRIu64 "\n", + (uint64_t) values2, (uint64_t) values3, (uint64_t) values4, + (uint64_t) values); + + if (report.failed) + result = 1; + + checksum = fnv1a64(0xcbf29ce484222325ULL, report.data, report.length); + if (!result) + report_printf(&report, "checksum=%016" PRIx64 "\n", checksum); + + /* Always print the full report so the CI log shows the results. */ + fputs(report.data, stdout); + fflush(stdout); + + if (make_parent_dir(result_file) != 0) { + fprintf(stderr, "FAIL: mkdir(parent of %s): %s\n", result_file, strerror(errno)); + result = 1; + } + + /* Binary mode: the report must be byte-for-byte identical on every + * platform, but in text mode the Windows C runtime converts every "\n" + * to "\r\n" on write, which would break the byte-for-byte comparison. */ + file = fopen(result_file, "wb"); + if (!file) { + fprintf(stderr, "FAIL: fopen(%s): %s\n", result_file, strerror(errno)); + result = 1; + } else { + if (fwrite(report.data, 1, report.length, file) != report.length) { + fprintf(stderr, "FAIL: write(%s): %s\n", result_file, strerror(errno)); + result = 1; + } + if (fclose(file) != 0) { + fprintf(stderr, "FAIL: fclose(%s): %s\n", result_file, strerror(errno)); + result = 1; + } + } + + OpenSimplex2F_free(ctx); + OpenSimplex2F_shutdown(); + free(report.data); + + printf("Result: %s (%" PRIu64 " values, seed %" PRId64 ")\n", + result ? "FAIL" : "PASS", (uint64_t) values, (int64_t) BENCHMARK_SEED); + return result; +} +