From 06b4de4fc0a6ebfab6c42bd4c4b967d75469d360 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 12 Jan 2026 14:43:12 -0500 Subject: [PATCH 1/4] gh-143750: Compile OpenSSL with TSan for TSan CI - Also fix "Install dependencies" step so that we use the installed Clang. We can use clang-20 on both ASan and TSan now. --- .github/workflows/reusable-san.yml | 52 +++++++++++++++++++----------- Tools/ssl/multissltests.py | 8 +++++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 405fa36454a62bf..cb91471a8fa78a6 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -28,6 +28,8 @@ jobs: }} runs-on: ubuntu-26.04 timeout-minutes: 60 + env: + OPENSSL_VER: 3.5.4 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -38,28 +40,41 @@ jobs: run: | sudo ./.github/workflows/posix-deps-apt.sh # On ubuntu-26.04 image, clang is clang-21 by default - - if [ "${SANITIZER}" = "TSan" ]; then - # Reduce ASLR to avoid TSan crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - fi - - - name: Sanitizer option setup - run: | - if [ "${SANITIZER}" = "TSan" ]; then - echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - else - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" - fi echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" + - name: TSan option setup + if: inputs.sanitizer == 'TSan' + run: | + # Reduce ASLR to avoid TSan crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" env: - SANITIZER: ${{ inputs.sanitizer }} SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: UBSan option setup + if: inputs.sanitizer != 'TSan' + run: | + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build (TSan)' + id: cache-openssl + if: inputs.sanitizer == 'TSan' + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ env.IMAGE_OS_VERSION }}-multissl-openssl-tsan-${{ env.OPENSSL_VER }} + - name: Install OpenSSL (TSan) + if: steps.cache-openssl.outputs.cache-hit != 'true' && inputs.sanitizer == 'TSan' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux --tsan - name: Configure CPython run: >- ./configure @@ -70,6 +85,7 @@ jobs: || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug + ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython run: make -j4 diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 1a213187b897d1d..d5e38993d971dfe 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -163,6 +163,12 @@ dest='keep_sources', help="Keep original sources for debugging." ) +parser.add_argument( + '--tsan', + action='store_true', + dest='tsan', + help="Build with thread sanitizer. (Disables fips in OpenSSL 3.x)." +) class AbstractBuilder(object): @@ -317,6 +323,8 @@ def _build_src(self, config_args=()): """Now build openssl""" log.info("Running build in {}".format(self.build_dir)) cwd = self.build_dir + if self.args.tsan: + config_args += ("-fsanitize=thread",) cmd = [ "./config", *config_args, "shared", "--debug", From c25b7851ebea56ce0f06b201971a22f274842c3e Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 8 Jul 2026 15:20:56 +0530 Subject: [PATCH 2/4] Revert "Temporarily allow the TSan (free-threading) job to fail (#152173)" This reverts commit 4f39efc47de9b1bc1cd9dfa88060efb4ff104d01. --- .github/workflows/build.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98df879413c72ee..b541dbd0c23d326 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -554,6 +554,7 @@ jobs: - Thread free-threading: - false + - true sanitizer: - TSan include: @@ -565,17 +566,6 @@ jobs: sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} - # XXX: Temporarily allow this job to fail to not block PRs. - build-san-free-threading: - # ${{ '' } is a hack to nest jobs under the same sidebar category. - name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] - needs: build-context - if: needs.build-context.outputs.run-ubuntu == 'true' - uses: ./.github/workflows/reusable-san.yml - with: - sanitizer: TSan - free-threading: true - cross-build-linux: name: Cross build Linux runs-on: ubuntu-26.04 @@ -679,7 +669,6 @@ jobs: - test-hypothesis - build-asan - build-san - - build-san-free-threading - cross-build-linux - cifuzz if: always() @@ -691,7 +680,6 @@ jobs: allowed-failures: >- build-android, build-emscripten, - build-san-free-threading, build-windows-msi, build-ubuntu-ssltests, test-hypothesis, From c89f2a4001105f9c04c26a8fbea28175b7ea7f2a Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 8 Jul 2026 16:14:53 +0530 Subject: [PATCH 3/4] make CI optional again for now --- .github/workflows/build.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b541dbd0c23d326..98df879413c72ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -554,7 +554,6 @@ jobs: - Thread free-threading: - false - - true sanitizer: - TSan include: @@ -566,6 +565,17 @@ jobs: sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} + # XXX: Temporarily allow this job to fail to not block PRs. + build-san-free-threading: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' + uses: ./.github/workflows/reusable-san.yml + with: + sanitizer: TSan + free-threading: true + cross-build-linux: name: Cross build Linux runs-on: ubuntu-26.04 @@ -669,6 +679,7 @@ jobs: - test-hypothesis - build-asan - build-san + - build-san-free-threading - cross-build-linux - cifuzz if: always() @@ -680,6 +691,7 @@ jobs: allowed-failures: >- build-android, build-emscripten, + build-san-free-threading, build-windows-msi, build-ubuntu-ssltests, test-hypothesis, From bc62a6a0add8abb0ea32b27da7cd7dd121ba8ef2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 8 Jul 2026 19:47:17 +0530 Subject: [PATCH 4/4] address review --- .github/workflows/reusable-san.yml | 37 ++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index cb91471a8fa78a6..cae51fad2c586ca 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -17,6 +17,7 @@ permissions: env: FORCE_COLOR: 1 + OPENSSL_VER: 3.5.7 jobs: build-san-reusable: @@ -28,8 +29,6 @@ jobs: }} runs-on: ubuntu-26.04 timeout-minutes: 60 - env: - OPENSSL_VER: 3.5.4 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -45,21 +44,27 @@ jobs: - name: TSan option setup if: inputs.sanitizer == 'TSan' run: | - # Reduce ASLR to avoid TSan crashing - sudo sysctl -w vm.mmap_rnd_bits=28 + sudo sysctl -w vm.mmap_rnd_bits=28 # Reduce ASLR to avoid TSan crashing + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${SUPPRESSIONS_SUFFIX}.txt handle_segv=0" >> "$GITHUB_ENV" + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + SUPPRESSIONS_SUFFIX: >- + ${{ fromJSON(inputs.free-threading) && '_free_threading' || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - env: - SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + }} - name: UBSan option setup if: inputs.sanitizer != 'TSan' - run: | - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" + run: >- + echo + "UBSAN_OPTIONS=${SAN_LOG_OPTION} + halt_on_error=1 + suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" + >> "$GITHUB_ENV" env: SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - name: Add ccache to PATH @@ -73,8 +78,16 @@ jobs: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ env.IMAGE_OS_VERSION }}-multissl-openssl-tsan-${{ env.OPENSSL_VER }} - name: Install OpenSSL (TSan) - if: steps.cache-openssl.outputs.cache-hit != 'true' && inputs.sanitizer == 'TSan' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux --tsan + if: >- + inputs.sanitizer == 'TSan' + && steps.cache-openssl.outputs.cache-hit != 'true' + run: >- + python3 Tools/ssl/multissltests.py + --steps=library + --base-directory="${MULTISSL_DIR}" + --openssl="${OPENSSL_VER}" + --system=Linux + --tsan - name: Configure CPython run: >- ./configure