From b5f52b6b7d8fa1486e6c9f51ea65a15a655568de Mon Sep 17 00:00:00 2001
From: AndrewQuijano <andrewquijano92@gmail.com>
Date: Tue, 6 Oct 2026 18:19:36 +0300
Subject: [PATCH 1/2] Adding code for Fuzzbench to work on NYU HPC

---
 common/gcloud.py                              |  11 +
 docker/dispatcher-image/Dockerfile            |   1 +
 experiment/build/builder.py                   |   2 +
 experiment/build/singularity_build.py         |  51 ++++
 .../runner-startup-script-template.sh         |  38 +++
 experiment/scheduler.py                       |   1 +
 singularity/README.md                         | 226 ++++++++++++++++++
 singularity/build_sifs.sh                     |  59 +++++
 singularity/fuzzbench.sbatch                  |  43 ++++
 singularity/run_experiment.sh                 | 107 +++++++++
 10 files changed, 539 insertions(+)
 create mode 100644 experiment/build/singularity_build.py
 create mode 100644 singularity/README.md
 create mode 100644 singularity/build_sifs.sh
 create mode 100644 singularity/fuzzbench.sbatch
 create mode 100644 singularity/run_experiment.sh

diff --git a/common/gcloud.py b/common/gcloud.py
index cc03a2ff7..42910a81c 100644
--- a/common/gcloud.py
+++ b/common/gcloud.py
@@ -14,7 +14,9 @@
 """Google cloud related code."""
 
 import enum
+import os
 import posixpath
+import shutil
 import subprocess
 from typing import List, Optional
 
@@ -129,6 +131,15 @@ def set_default_project(cloud_project: str):
 def run_local_instance(startup_script: Optional[str] = None) -> bool:
     """Does the equivalent of "create_instance" for local experiments, runs
     |startup_script| in the background."""
+    spool_dir = os.getenv('FUZZBENCH_SPOOL_DIR')
+    if spool_dir:
+        # A Singularity container can't start other containers, so hand the
+        # script to singularity/run_experiment.sh, which runs it on the host.
+        spooled_script = os.path.join(spool_dir,
+                                      os.path.basename(startup_script))
+        shutil.copy(startup_script, spooled_script + '.tmp')
+        os.rename(spooled_script + '.tmp', spooled_script)
+        return True
     command = ['/bin/bash', startup_script]
     # pylint: disable=consider-using-with
     subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
diff --git a/docker/dispatcher-image/Dockerfile b/docker/dispatcher-image/Dockerfile
index af733ea9f..078f08964 100644
--- a/docker/dispatcher-image/Dockerfile
+++ b/docker/dispatcher-image/Dockerfile
@@ -21,6 +21,7 @@ WORKDIR $WORK
 
 # Install runtime dependencies for benchmarks, easy json parsing, and fuzzbench.
 RUN apt-get update -y && apt-get install -y \
+    git \
     jq \
     libglib2.0-0 \
     libxml2 \
diff --git a/experiment/build/builder.py b/experiment/build/builder.py
index d66ae30d3..d922c2ade 100644
--- a/experiment/build/builder.py
+++ b/experiment/build/builder.py
@@ -39,6 +39,8 @@
 
 if not experiment_utils.is_local_experiment():
     import experiment.build.gcb_build as buildlib
+elif os.getenv('SIF_DIR'):
+    import experiment.build.singularity_build as buildlib
 else:
     import experiment.build.local_build as buildlib
 
diff --git a/experiment/build/singularity_build.py b/experiment/build/singularity_build.py
new file mode 100644
index 000000000..52cfaf653
--- /dev/null
+++ b/experiment/build/singularity_build.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for "building" in Singularity experiments. The images are built with
+Docker ahead of time and converted to SIF files by singularity/build_sifs.sh,
+so this only checks that those files exist in $SIF_DIR."""
+
+import os
+import shutil
+import subprocess
+from typing import Tuple
+
+from common import logs
+from experiment.build import local_build
+
+logger = logs.Logger()  # pylint: disable=invalid-name
+
+
+def build_base_images() -> Tuple[int, str]:
+    """Nothing to build, the runner SIFs already contain the base images."""
+    return 0, ''
+
+
+def build_coverage(benchmark):
+    """Copy the coverage build archive made by build_sifs.sh to where the
+    measurer expects it."""
+    local_build.make_shared_coverage_binaries_dir()
+    archive = os.path.join(os.environ['SIF_DIR'], 'coverage',
+                           f'coverage-build-{benchmark}.tar.gz')
+    shutil.copy(archive, local_build.get_shared_coverage_binaries_dir())
+
+
+def build_fuzzer_benchmark(fuzzer: str, benchmark: str) -> bool:
+    """Check that the runner SIF for |fuzzer| and |benchmark| exists."""
+    sif = os.path.join(os.environ['SIF_DIR'], 'runners', fuzzer,
+                       f'{benchmark}.sif')
+    if not os.path.exists(sif):
+        logger.error('Runner SIF not found: %s.', sif)
+        raise subprocess.CalledProcessError(1, ['test', '-f', sif])
+    return True
diff --git a/experiment/resources/runner-startup-script-template.sh b/experiment/resources/runner-startup-script-template.sh
index 5ef1e40bf..e616b8c0f 100644
--- a/experiment/resources/runner-startup-script-template.sh
+++ b/experiment/resources/runner-startup-script-template.sh
@@ -13,6 +13,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{% if sif_dir %}
+# Singularity/Apptainer: singularity/run_experiment.sh runs this on the host.
+# SIF images are read-only, so the corpus and the runner's working directory
+# (results/, corpus-archives/) go to a writable trial directory at /trial.
+TRIAL_DIR=${FUZZBENCH_TRIALS_DIR:-/tmp/fuzzbench-trials}/{{experiment}}/{{trial_id}}
+mkdir -p $TRIAL_DIR/corpus $TRIAL_DIR/seeds
+cat > $TRIAL_DIR/env <<EOF
+INSTANCE_NAME={{instance_name}}
+FUZZER={{fuzzer}}
+BENCHMARK={{benchmark}}
+EXPERIMENT={{experiment}}
+TRIAL_ID={{trial_id}}
+TRIAL_GROUP_NUM={{trial_group_num}}
+MICRO_EXPERIMENT={{micro_experiment}}
+MAX_TOTAL_TIME={{max_total_time}}
+SNAPSHOT_PERIOD={{snapshot_period}}
+NO_SEEDS={{no_seeds}}
+NO_DICTIONARIES={{no_dictionaries}}
+OSS_FUZZ_CORPUS={{oss_fuzz_corpus}}
+CUSTOM_SEED_CORPUS_DIR={{custom_seed_corpus_dir}}
+DOCKER_REGISTRY={{docker_registry}}
+EXPERIMENT_FILESTORE={{experiment_filestore}}
+REPORT_FILESTORE={{report_filestore}}
+FUZZ_TARGET={{fuzz_target}}
+PRIVATE={{private}}
+LOCAL_EXPERIMENT={{local_experiment}}
+OUTPUT_CORPUS_DIR=/trial/corpus
+SEED_CORPUS_DIR=/trial/seeds
+EOF
+
+{% if cpuset %}taskset -c {{cpuset}} {% endif %}singularity run \
+--cleanenv --no-home --writable-tmpfs \
+--bind $TRIAL_DIR:/trial --pwd /trial \
+--bind {{experiment_filestore}} --bind {{report_filestore}} \
+--env-file $TRIAL_DIR/env \
+{{sif_dir}}/runners/{{fuzzer}}/{{benchmark}}.sif 2>&1 | tee $TRIAL_DIR/runner-log.txt
+exit
+{% endif %}
 # Configure the host.
 
 # Make everything ptrace-able.
diff --git a/experiment/scheduler.py b/experiment/scheduler.py
index b189eabb6..3e1f346e2 100644
--- a/experiment/scheduler.py
+++ b/experiment/scheduler.py
@@ -779,6 +779,7 @@ def render_startup_script_template(  # pylint: disable=too-many-arguments
         'private': experiment_config['private'],
         'cpuset': cpuset,
         'custom_seed_corpus_dir': experiment_config['custom_seed_corpus_dir'],
+        'sif_dir': os.getenv('SIF_DIR'),
     }
 
     if not local_experiment:
diff --git a/singularity/README.md b/singularity/README.md
new file mode 100644
index 000000000..eaf99dfad
--- /dev/null
+++ b/singularity/README.md
@@ -0,0 +1,226 @@
+# Running FuzzBench images with Singularity/Apptainer
+
+HPC clusters like NYU's don't allow Docker, only Singularity (or Apptainer, its renamed fork, which
+installs a `singularity` command too). The plan:
+
+1. **Build** the FuzzBench images with Docker somewhere you have root (your laptop via WSL2, or CI).
+2. **Convert** each image to a `.sif` file.
+3. **Run** the `.sif` files with Singularity on the cluster. The cluster never has to build anything.
+
+Sections 1–6 test everything on a Windows laptop (8 cores, 16 GB) in WSL2. Section 7 runs it on
+NYU Torch. The code changes are listed at the end.
+
+## 1. Set up WSL2 (Windows → Linux)
+
+Singularity is Linux-only. On Windows it runs inside WSL2.
+
+In PowerShell (as admin):
+
+```powershell
+wsl --install -d Ubuntu-22.04   # 22.04 ships Python 3.10, which FuzzBench expects
+```
+
+Limit what WSL can use so builds don't freeze Windows. Create `%UserProfile%\.wslconfig`:
+
+```ini
+[wsl2]
+memory=10GB      # leave ~6 GB for Windows
+processors=6     # leave 2 cores for Windows
+swap=8GB
+```
+
+Then run `wsl --shutdown` and reopen Ubuntu.
+
+**Clone the repo inside WSL** (e.g. `~/fuzzbench`), not under `/mnt/c/...`. Builds on the OneDrive or
+Windows filesystem are very slow and have permission problems.
+
+Plan on roughly **40 GB of free disk**. The base, builder and runner images for one fuzzer and one
+benchmark come to about 10–15 GB, and each `.sif` adds another 1–3 GB.
+
+## 2. Install Docker and Apptainer in WSL
+
+```bash
+# Docker (or turn on Docker Desktop's WSL integration instead)
+sudo apt-get update && sudo apt-get install -y docker.io make rsync
+sudo usermod -aG docker $USER && newgrp docker
+
+# Apptainer (provides the `singularity` command)
+sudo add-apt-repository -y ppa:apptainer/ppa
+sudo apt-get update && sudo apt-get install -y apptainer
+singularity --version
+```
+
+Check which version the cluster has (`singularity --version` on a login node). SIF files built with
+Apptainer 1.x generally run on SingularityCE 3.x/4.x.
+
+Test it:
+
+```bash
+singularity exec docker://ubuntu:22.04 cat /etc/os-release
+```
+
+## 3. Build one fuzzer×benchmark with Docker
+
+Use a small benchmark and a simple fuzzer. `libpng_libpng_read_fuzzer` builds quickly.
+
+```bash
+cd ~/fuzzbench
+make install-dependencies
+export FUZZER=afl BENCHMARK=libpng_libpng_read_fuzzer
+
+make build-$FUZZER-$BENCHMARK          # runner image (builds base + builder images too)
+make build-coverage-$BENCHMARK         # coverage image, which the measurer needs
+make test-run-$FUZZER-$BENCHMARK       # Docker check: fuzzes for ~20s
+```
+
+Don't use `make -j` on this laptop, because clang builds use a lot of memory. On the cluster you'll
+only use the output files.
+
+## 4. Convert to SIF
+
+```bash
+mkdir -p ~/sifs/runners/$FUZZER ~/sifs/builders/coverage
+singularity build ~/sifs/runners/$FUZZER/$BENCHMARK.sif \
+    docker-daemon://gcr.io/fuzzbench/runners/$FUZZER/$BENCHMARK:latest
+singularity build ~/sifs/builders/coverage/$BENCHMARK.sif \
+    docker-daemon://gcr.io/fuzzbench/builders/coverage/$BENCHMARK:latest
+```
+
+`docker-daemon://` needs an explicit tag, and `latest` is what local builds produce. To get the files to
+the cluster, copy them with `scp`/`rsync`. You can also push the images to a registry (Docker Hub,
+GHCR) and run `singularity pull docker://...` on the cluster.
+
+## 5. Smoke test: fuzz under Singularity (works today)
+
+This does the same thing as `make test-run-...`, but with Singularity. Singularity images are
+read-only, and FuzzBench's runner writes into `/out`. So the test points the corpus and the working
+directory at a writable host folder bound to `/trial`.
+
+```bash
+mkdir -p ~/fb-smoke/{corpus,seeds}
+cat > ~/fb-smoke/env <<EOF
+FUZZ_OUTSIDE_EXPERIMENT=1
+FORCE_LOCAL=1
+TRIAL_ID=1
+FUZZER=$FUZZER
+BENCHMARK=$BENCHMARK
+FUZZ_TARGET=libpng_read_fuzzer
+MAX_TOTAL_TIME=120
+SNAPSHOT_PERIOD=30
+OUTPUT_CORPUS_DIR=/trial/corpus
+SEED_CORPUS_DIR=/trial/seeds
+EOF
+
+singularity run \
+    --cleanenv --no-home --writable-tmpfs \
+    --bind ~/fb-smoke:/trial --pwd /trial \
+    --env-file ~/fb-smoke/env \
+    ~/sifs/runners/$FUZZER/$BENCHMARK.sif
+```
+
+The test passes if AFL's output streams for about 2 minutes and `~/fb-smoke/corpus` fills up. Some
+warnings are expected and harmless when you aren't root:
+- `nice: cannot set niceness: Permission denied`
+- writes to `/proc/sys/kernel/...` failing
+
+Here's why each flag matters. They'll be the same on the cluster:
+- `--cleanenv`: stops your host environment (`PYTHONPATH`, SLURM variables, ...) from leaking into
+  the container. The image's own `ENV` values (`OUT`, `ROOT_DIR`, `PYTHONPATH=/src`) are still set.
+- `--no-home`: stops your dotfiles and home-directory Python packages from shadowing the image's.
+- `--writable-tmpfs`: a small (≈64 MB) scratch layer over the image, for fuzzers that write outside
+  `/trial`. Large data must go to a bind mount.
+- `--pwd /trial`: the runner writes `corpus-archives/` and `results/` to its current directory, and
+  Singularity ignores the Dockerfile's `WORKDIR`.
+
+Before moving on, repeat this test for each fuzzer you plan to use. Some fuzzers assume root or a
+writable `/out` in ways the AFL family doesn't.
+
+## 6. Full local experiment under Singularity (WSL)
+
+`singularity/build_sifs.sh` does steps 3–4 for every fuzzer×benchmark and also writes the dispatcher
+SIF and the coverage archive the measurer needs:
+
+```bash
+cd ~/fuzzbench
+bash singularity/build_sifs.sh ~/sifs "afl" "libpng_libpng_read_fuzzer"
+# -> ~/sifs/dispatcher-image.sif
+#    ~/sifs/runners/afl/libpng_libpng_read_fuzzer.sif
+#    ~/sifs/coverage/coverage-build-libpng_libpng_read_fuzzer.tar.gz
+```
+
+Write `~/fb-exp/experiment-config.yaml` (unquoted absolute paths, the script greps them):
+
+```yaml
+trials: 2
+max_total_time: 1800        # 30 min, enough to see a report
+snapshot_period: 300        # measure every 5 min instead of 15
+docker_registry: gcr.io/fuzzbench
+experiment_filestore: /home/<you>/fb-exp/data
+report_filestore: /home/<you>/fb-exp/report
+local_experiment: true
+```
+
+Run it (`-a` allows uncommitted changes):
+
+```bash
+SIF_DIR=~/sifs bash singularity/run_experiment.sh ~/fb-exp/experiment-config.yaml test-sing -a \
+    --fuzzers afl --benchmarks libpng_libpng_read_fuzzer
+```
+
+What to check while it runs:
+- `ls /tmp/fuzzbench-$USER-test-sing/spool/`: one `*.sh.started` + `*.sh.log` per trial.
+- `/tmp/fuzzbench-$USER-test-sing/trials/test-sing/<trial>/runner-log.txt`: AFL output.
+- `~/fb-exp/data/test-sing/experiment-folders/`: corpus snapshots appear every 5 min.
+- `~/fb-exp/report/test-sing/index.html`: the report (first one after ~1–2 snapshot periods).
+
+The script exits when the dispatcher finishes (after `max_total_time` plus the final measurement).
+
+Don't pass `--runners-cpus`/`--measurers-cpus` on the cluster: they pin to CPU ids starting at 0,
+which may be outside the CPUs SLURM gave your job. Locally they're fine.
+
+## 7. On NYU Torch
+
+```bash
+# From WSL: copy the repo (same commit!) and the SIFs. Use the data transfer node.
+rsync -av --exclude .venv ~/fuzzbench/ <netid>@dtn.torch.hpc.nyu.edu:/scratch/<netid>/fuzzbench/
+rsync -av ~/sifs/ <netid>@dtn.torch.hpc.nyu.edu:/scratch/<netid>/sifs/
+```
+
+On Torch, write `/scratch/$USER/fb-exp/experiment-config.yaml` (same as above, with
+`/scratch/<netid>/fb-exp/...` paths). First try it interactively on a compute node (not the login
+node) with a short `max_total_time: 900`:
+
+```bash
+srun --account=<your torch_pr_... account> --cpus-per-task=4 --mem=16G --time=01:00:00 --pty bash
+cd /scratch/$USER/fuzzbench
+SIF_DIR=/scratch/$USER/sifs bash singularity/run_experiment.sh \
+    /scratch/$USER/fb-exp/experiment-config.yaml hpc-test -a \
+    --fuzzers afl --benchmarks libpng_libpng_read_fuzzer
+```
+
+Then edit `singularity/fuzzbench.sbatch` (account, CPUs, memory, time, fuzzers/benchmarks) and
+`sbatch singularity/fuzzbench.sbatch`. Don't use the preemption partitions: a requeued job restarts
+the experiment from scratch.
+
+## How it works (the code changes)
+
+The Singularity path is on only when `SIF_DIR` is set (by `run_experiment.sh`), so Docker and cloud
+experiments behave exactly as before. No `--fakeroot` or nested containers needed.
+
+| Where | Change |
+|---|---|
+| `singularity/run_experiment.sh` (new) | Runs `run_experiment.py` (with `MANUAL_EXPERIMENT=1`, so it only copies source and config to the filestore) and then the dispatcher inside `dispatcher-image.sif`, mirroring `LocalDispatcher`. Also runs a host-side loop that executes trial startup scripts the dispatcher drops into a spool dir. |
+| `common/gcloud.py::run_local_instance` | If `FUZZBENCH_SPOOL_DIR` is set, copy the trial startup script there instead of running it (a container can't start sibling containers; there's no docker.sock). |
+| `experiment/resources/runner-startup-script-template.sh` + 1 kwarg in `scheduler.py` | `{% if sif_dir %}`: `singularity run` the runner SIF with the flags from step 5; `--cpuset-cpus` becomes `taskset -c`. |
+| `experiment/build/singularity_build.py` (new) + 2 lines in `builder.py` | "Build" backend: checks the runner SIFs exist and copies the prebuilt coverage archive to the filestore. |
+| `docker/dispatcher-image/Dockerfile` | Adds `git`, which `run_experiment.py` needs. |
+| `singularity/build_sifs.sh`, `singularity/fuzzbench.sbatch` (new) | Build/convert on the laptop; the SLURM job. |
+
+## Cluster facts (checked 2026-09-30)
+
+- Torch has Apptainer 1.5.3 (`singularity` is an alias). `--env-file`, `--no-home` and
+  `--writable-tmpfs` all exist. SIFs built by Apptainer 1.3+ on WSL run on it.
+- `singularity --fakeroot` errored because `--fakeroot` is an option of `exec`/`run`/`build`, not
+  of the top-level command. The correct test is `singularity exec --fakeroot docker://alpine id`
+  (it prints `uid=0(root)` if allowed). This setup doesn't need it either way.
+- Login and compute nodes run Red Hat; build nothing on login nodes. Jobs must pass `--account`.
diff --git a/singularity/build_sifs.sh b/singularity/build_sifs.sh
new file mode 100644
index 000000000..4dc3b6af3
--- /dev/null
+++ b/singularity/build_sifs.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds everything a Singularity experiment needs, on a machine with Docker
+# and Apptainer/Singularity (e.g. WSL2). Output layout, which
+# experiment/build/singularity_build.py and the runner startup script expect:
+#   SIF_DIR/dispatcher-image.sif
+#   SIF_DIR/runners/FUZZER/BENCHMARK.sif
+#   SIF_DIR/coverage/coverage-build-BENCHMARK.tar.gz
+#
+# Usage: singularity/build_sifs.sh SIF_DIR "FUZZER..." "BENCHMARK..."
+
+set -euo pipefail
+
+if [ $# -ne 3 ]; then
+  echo "Usage: $0 SIF_DIR \"FUZZER...\" \"BENCHMARK...\"" >&2
+  exit 1
+fi
+SIF_DIR=$(realpath -m "$1")
+FUZZERS=$2
+BENCHMARKS=$3
+REGISTRY=gcr.io/fuzzbench
+
+cd "$(dirname "$0")/.."
+mkdir -p "$SIF_DIR/coverage" "$SIF_DIR/runners"
+
+make dispatcher-image
+singularity build --force "$SIF_DIR/dispatcher-image.sif" \
+  "docker-daemon://$REGISTRY/dispatcher-image:latest"
+
+for benchmark in $BENCHMARKS; do
+  make "build-coverage-$benchmark"
+  # Same archive local_build.py::copy_coverage_binaries makes.
+  docker run --rm -v "$SIF_DIR/coverage:/host-out" \
+    "$REGISTRY/builders/coverage/$benchmark" /bin/bash -c \
+    "cd /out; tar -czf /host-out/coverage-build-$benchmark.tar.gz * /src /work"
+
+  for fuzzer in $FUZZERS; do
+    make "build-$fuzzer-$benchmark"
+    mkdir -p "$SIF_DIR/runners/$fuzzer"
+    singularity build --force "$SIF_DIR/runners/$fuzzer/$benchmark.sif" \
+      "docker-daemon://$REGISTRY/runners/$fuzzer/$benchmark:latest"
+  done
+done
+
+echo "Done. Copy $SIF_DIR to the cluster, e.g.:"
+echo "  rsync -av $SIF_DIR/ <netid>@dtn.torch.hpc.nyu.edu:/scratch/<netid>/sifs/"
diff --git a/singularity/fuzzbench.sbatch b/singularity/fuzzbench.sbatch
new file mode 100644
index 000000000..ea4872e1f
--- /dev/null
+++ b/singularity/fuzzbench.sbatch
@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs a whole FuzzBench experiment (dispatcher + all trials) on one NYU Torch
+# node. Edit the account, resources and the variables below, then:
+#   sbatch singularity/fuzzbench.sbatch
+#
+# Size --cpus-per-task as (#fuzzers x #benchmarks x trials) + 2 for the
+# dispatcher/measurer, and --time as max_total_time plus ~1-2 hours.
+
+#SBATCH --job-name=fuzzbench
+#SBATCH --account=torch_pr_CHANGEME
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --time=04:00:00
+#SBATCH --output=%x-%j.log
+
+set -euo pipefail
+
+export SIF_DIR=/scratch/$USER/sifs
+FUZZBENCH_DIR=/scratch/$USER/fuzzbench
+CONFIG=/scratch/$USER/fb-exp/experiment-config.yaml
+NAME=hpc-$SLURM_JOB_ID
+
+cd "$FUZZBENCH_DIR"
+singularity --version
+bash singularity/run_experiment.sh "$CONFIG" "$NAME" -a \
+  --fuzzers afl \
+  --benchmarks libpng_libpng_read_fuzzer
diff --git a/singularity/run_experiment.sh b/singularity/run_experiment.sh
new file mode 100644
index 000000000..525408e23
--- /dev/null
+++ b/singularity/run_experiment.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs a local FuzzBench experiment with Singularity/Apptainer instead of
+# Docker, all on this machine. This does what run_experiment.py's
+# LocalDispatcher does, plus a host-side loop that starts the trial containers
+# the dispatcher asks for (a container can't start other containers).
+#
+# Usage:
+#   SIF_DIR=/path/to/sifs singularity/run_experiment.sh CONFIG NAME \
+#       [run_experiment.py args, e.g. -a --fuzzers afl --benchmarks ...]
+
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+  echo "Usage: SIF_DIR=... $0 CONFIG NAME [run_experiment.py args...]" >&2
+  exit 1
+fi
+CONFIG=$(realpath "$1")
+NAME=$2
+shift 2
+SIF_DIR=$(realpath "${SIF_DIR:?Set SIF_DIR to the directory build_sifs.sh wrote}")
+FUZZBENCH_DIR=$(realpath "$(dirname "$0")/..")
+DISPATCHER_SIF=$SIF_DIR/dispatcher-image.sif
+
+yaml_value() {
+  grep "^$1:" "$2" | head -n1 | awk '{print $2}'
+}
+EXPERIMENT_FILESTORE=$(yaml_value experiment_filestore "$CONFIG")
+REPORT_FILESTORE=$(yaml_value report_filestore "$CONFIG")
+mkdir -p "$EXPERIMENT_FILESTORE" "$REPORT_FILESTORE"
+
+# Fast node-local disk for the dispatcher's /work, the trial directories and
+# the spool of trial startup scripts.
+SCRATCH=${TMPDIR:-/tmp}/fuzzbench-$USER-$NAME
+rm -rf "$SCRATCH"
+mkdir -p "$SCRATCH/config" "$SCRATCH/work" "$SCRATCH/spool" "$SCRATCH/trials"
+# Copy the config so only it (not its whole directory) goes to the filestore.
+cp "$CONFIG" "$SCRATCH/config/experiment.yaml"
+export FUZZBENCH_TRIALS_DIR=$SCRATCH/trials
+
+in_dispatcher() {
+  singularity exec --cleanenv --no-home --writable-tmpfs \
+    --bind "$FUZZBENCH_DIR" --bind "$SIF_DIR" --bind "$SCRATCH" \
+    --bind "$SCRATCH/work:/work" \
+    --bind "$EXPERIMENT_FILESTORE" --bind "$REPORT_FILESTORE" \
+    "$@"
+}
+
+echo "== Copying source and config to the experiment filestore."
+in_dispatcher --pwd "$FUZZBENCH_DIR" "$DISPATCHER_SIF" \
+  env MANUAL_EXPERIMENT=1 PYTHONPATH="$FUZZBENCH_DIR" \
+  python3 experiment/run_experiment.py \
+  --experiment-config "$SCRATCH/config/experiment.yaml" \
+  --experiment-name "$NAME" "$@"
+
+# run_experiment.py fills in defaults, so read the values from its output.
+FINAL_CONFIG=$EXPERIMENT_FILESTORE/$NAME/input/config/experiment.yaml
+cat > "$SCRATCH/dispatcher.env" <<EOF
+LOCAL_EXPERIMENT=True
+INSTANCE_NAME=d-$NAME
+EXPERIMENT=$NAME
+SQL_DATABASE_URL='sqlite:///$EXPERIMENT_FILESTORE/local.db?check_same_thread=False'
+EXPERIMENT_FILESTORE=$EXPERIMENT_FILESTORE
+REPORT_FILESTORE=$REPORT_FILESTORE
+SNAPSHOT_PERIOD=$(yaml_value snapshot_period "$FINAL_CONFIG")
+DOCKER_REGISTRY=$(yaml_value docker_registry "$FINAL_CONFIG")
+CONCURRENT_BUILDS=4
+WORKER_POOL_NAME=
+SIF_DIR=$SIF_DIR
+FUZZBENCH_SPOOL_DIR=$SCRATCH/spool
+EOF
+
+echo "== Starting trial launcher (logs in $SCRATCH/spool)."
+(
+  while true; do
+    for script in "$SCRATCH"/spool/*.sh; do
+      [ -e "$script" ] || continue
+      mv "$script" "$script.started"
+      bash "$script.started" > "$script.log" 2>&1 &
+    done
+    sleep 5
+  done
+) &
+LAUNCHER_PID=$!
+trap 'kill $LAUNCHER_PID 2>/dev/null' EXIT
+
+echo "== Starting dispatcher. Report: $REPORT_FILESTORE/$NAME/index.html"
+# Same command LocalDispatcher runs in its Docker container.
+in_dispatcher --pwd /work --env-file "$SCRATCH/dispatcher.env" \
+  "$DISPATCHER_SIF" /bin/bash -c \
+  'rsync -r "${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && '\
+'mkdir ${WORK}/src && '\
+'tar -xzf ${WORK}/src.tar.gz -C ${WORK}/src && '\
+'PYTHONPATH=${WORK}/src python3 ${WORK}/src/experiment/dispatcher.py'

From f1f78b80d6db44fd85635f271eebf7e2b3d5f5ff Mon Sep 17 00:00:00 2001
From: AndrewQuijano <andrewquijano92@gmail.com>
Date: Tue, 6 Oct 2026 18:20:18 +0300
Subject: [PATCH 2/2] Claude helped me out on the conversion from Docker to
 singularity

---
 CLAUDE.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 CLAUDE.md

diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 000000000..579a50c82
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,110 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+FuzzBench is a service for benchmarking fuzzers against real-world (OSS-Fuzz) benchmarks. It builds
+every fuzzer × benchmark combination as Docker images, runs many timed trials, periodically measures
+coverage/crashes from the trials' corpora, and generates statistical HTML reports. Everything is Python
+3.10 plus a lot of Docker; it's Linux-only in practice (the Makefile uses `/bin/bash` and
+`.venv/bin/activate`), so on Windows, file edits are fine but builds and tests need Linux, WSL, or CI.
+
+Full docs live in `docs/` (Jekyll site published at google.github.io/fuzzbench). The most useful ones are
+`docs/getting-started/adding_a_new_fuzzer.md`, `docs/developing-fuzzbench/adding_a_new_benchmark.md`,
+and `docs/running-a-local-experiment/running_a_local_experiment.md`.
+
+## Commands
+
+All `make` targets create/use `.venv` from `requirements.txt` automatically (`make install-dependencies`).
+
+- `make presubmit`: runs every default check on files changed vs. `origin/master`: license header,
+  yapf format check, pylint, pytype, pytest, and fuzzer/benchmark name validation. CI runs
+  `FUZZBENCH_TEST_INTEGRATION=1 make presubmit`.
+- Single checks: `make format` (yapf in-place), `make lint`, `make typecheck`, `make licensecheck`,
+  `make test`. Or call `python3 presubmit.py <check> [--all-files]`. Without `--all-files` they only
+  look at files changed vs. master (via `src_analysis/diff_utils.py`).
+- Run one test: `source .venv/bin/activate && python3 -m pytest -vv path/to/test_file.py::test_name`.
+  Tests sit next to the code as `test_*.py`. The root `conftest.py` forces `SQL_DATABASE_URL=sqlite://`
+  (in-memory) and provides the `db`, `environ`, `experiment`, `use_local_filestore` and `use_gsutil`
+  fixtures. Mark slow tests with `@pytest.mark.slow`.
+- Integration tests that need Docker or GCB are gated by env vars such as `FUZZBENCH_TEST_INTEGRATION`,
+  `TEST_INTEGRATION_ALL`, `TEST_BUILD_CHANGED_FUZZERS` and `TEST_BUILD_CHANGED_BENCHMARKS`
+  (`python3 presubmit.py test_changed_integrations`, which isn't a default check).
+- Fuzzer/benchmark Docker targets come from the generated `docker/generated.mk`
+  (`docker/generate_makefile.py` + `docker/image_types.yaml`):
+  - `make build-$FUZZER-$BENCHMARK`, `make build-$FUZZER-all`
+  - `make run-$FUZZER-$BENCHMARK` (fuzz interactively), `make test-run-$FUZZER-$BENCHMARK` (short
+    smoke run)
+  - `make debug-builder-$FUZZER-$BENCHMARK` and `make debug-$FUZZER-$BENCHMARK` (shell into the
+    builder/runner)
+- Local experiment:
+  `PYTHONPATH=. python3 experiment/run_experiment.py --experiment-config <cfg.yaml> --experiment-name <name> --fuzzers afl libfuzzer --benchmarks freetype2_ftfuzzer ...`
+  (the config needs `local_experiment: true` and absolute `experiment_filestore`/`report_filestore`).
+  Alternatively, use `make run-experiment` for the docker-compose setup (`compose/fuzzbench.yaml`).
+- End-to-end test: `make run-end-to-end-test` (docker-compose with `compose/e2e-test.yaml`).
+- Docs: `make docs-serve` (Jekyll, needs Ruby/bundler).
+
+Style: yapf (`.style.yapf`) and pylint (`.pylintrc`). Every `.py`, `.sh`, `.c`, etc. file and every
+`Dockerfile` needs the Apache 2.0 license header; `benchmarks/`, `database/alembic/` and any
+`third_party/` directory are exempt.
+
+## Architecture
+
+### Integrations (most contributions land here)
+- **`fuzzers/<name>/`**: `builder.Dockerfile` (builds the fuzzer on top of a benchmark's builder
+  image), `runner.Dockerfile` (the runtime image), and `fuzzer.py`, which exposes `build()` (compiles
+  the benchmark with the fuzzer's instrumentation, usually through `fuzzers/utils.py::build_benchmark`
+  with CC/CXX/CFLAGS and `FUZZER_LIB` set) and `fuzz(input_corpus, output_corpus, target_binary)`,
+  plus an optional `get_stats`. Variants usually import and reuse a parent fuzzer's `fuzzer.py` (e.g.
+  `aflplusplus_*` reuse `aflplusplus`). `fuzzers/coverage/` is special: it's the clang source-based
+  coverage build the measurer uses, not a real fuzzer. Fuzzer and benchmark names are validated by
+  presubmit (`common/fuzzer_utils.py`, `common/benchmark_utils.py`).
+- **`benchmarks/<name>/`**: an OSS-Fuzz-style `Dockerfile` (usually with a `build.sh`), `benchmark.yaml` (`project`,
+  `fuzz_target`, `commit`, `commit_date`, optional `type: bug` / `unsupported_fuzzers`, etc.) and
+  optional seeds. `benchmarks/oss_fuzz_benchmark_integration.py` imports new ones from OSS-Fuzz.
+
+### Docker image graph
+`docker/image_types.yaml` is the single source of truth for how images layer:
+`base-image` → `{benchmark}-project-builder` (the benchmark's Dockerfile) →
+`{fuzzer}-{benchmark}-builder-intermediate` (the fuzzer's `builder.Dockerfile`) →
+`{fuzzer}-{benchmark}-builder` (`docker/benchmark-builder/Dockerfile`, which checks out the pinned
+commit and runs `fuzzer_build` → `fuzzer.py::build()`) → runner images
+(`docker/benchmark-runner/Dockerfile` on top of the fuzzer's `runner.Dockerfile`). The same YAML feeds
+both `docker/generated.mk` for local builds and `experiment/build/generate_cloudbuild.py` for GCB, so
+edit the YAML instead of either output.
+
+### Experiment pipeline (`experiment/`)
+1. `run_experiment.py` validates the config/fuzzers/benchmarks, then starts the **dispatcher**,
+   either as a local Docker container or as a GCE instance (`common/gce.py`, `experiment/cloud`).
+2. `dispatcher.py::dispatcher_main` builds all images (`experiment/build/builder.py`, choosing
+   `local_build.py` or `gcb_build.py`), then runs three things concurrently:
+   - `scheduler.py::schedule_loop`: starts and stops trial instances/containers and preempts, retries
+     or finishes trials.
+   - `measurer/measure_manager.py::measure_main`: pulls corpus snapshots from the filestore every
+     cycle, runs them through the coverage build (`run_coverage.py`, `coverage_utils.py`) and crash
+     reproduction (`run_crashes.py`), and writes `Snapshot`/`Crash` rows. Measure workers can run
+     remotely, coordinated through a queue (`common/queue_utils.py`, `measure_worker.py`).
+   - `reporter.py::output_report`: calls `analysis/generate_report.py` periodically.
+3. `runner.py` runs inside each trial container. It calls the fuzzer's `fuzz()` and syncs corpus
+   snapshots to the experiment filestore every cycle.
+
+State lives in two places. The **database** (`database/models.py`, SQLAlchemy: `Experiment`, `Trial`,
+`Snapshot`, `Crash`) is Postgres in production and SQLite in tests, with Alembic migrations in
+`database/alembic/`; add a migration whenever you change a model. The **filestore**
+(`common/filestore_utils.py`) sends paths either to `gsutil.py` (GCS) or to `local_filestore.py`,
+depending on whether the experiment is local. Most runtime configuration comes from environment
+variables (`EXPERIMENT`, `EXPERIMENT_FILESTORE`, `LOCAL_EXPERIMENT`, `CLOUD_PROJECT`, `DOCKER_REGISTRY`,
+...), read through `common/experiment_utils.py` and `common/environment.py`.
+
+### Other top-level pieces
+- `analysis/`: report generation (`generate_report.py`, `experiment_results.py`, `stat_tests.py`,
+  `plotting.py`, Jinja templates in `report_templates/`). You can run it standalone against a DB or
+  CSV to make custom reports (`docs/developing-fuzzbench/custom_analysis_and_reports.md`).
+- `service/`: the automated FuzzBench service. `experiment-requests.yaml` is the queue of requested
+  experiments that `gcbrun_experiment.py` / `run_experiment_cloudbuild.yaml` launch; `core-fuzzers.yaml`
+  lists the default fuzzers.
+- `src_analysis/`: works out which fuzzers/benchmarks a diff touches (used by presubmit and by
+  `.github/workflows/build_and_test_run_fuzzer_benchmarks.py` so CI only builds changed integrations).
+- `common/`: shared utilities (subprocess wrapper `new_process.py`, logging, retry, config parsing,
+  gcloud/gsutil wrappers).