From ef2c7d22b45c9d45729546ac2e9ff3cf54d339d6 Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 4 Aug 2026 11:30:48 -0600 Subject: [PATCH 1/4] initial draft --- .gitignore | 6 + Makefile | 74 +++ PLAN.md | 176 ++++++ README.md | 106 +++- bin/formascute | 570 ++++++++++++++++++ conf/alpine.reference.config | 73 +++ conf/characterization.config | 18 + conf/experiments/batched_jobs.config | 4 + conf/experiments/independent_jobs.config | 4 + .../nf1_featurization_batched.config | 7 + .../nf1_featurization_independent.config | 7 + conf/experiments/nf1_queue_size_20.config | 12 + conf/experiments/queue_size_20.config | 7 + docs/clone-ready.md | 39 ++ docs/hpc-quickstart.md | 95 +++ docs/nf1-pipeline-target.md | 67 ++ docs/production-checklist.md | 62 ++ examples/curc_alpine.config | 51 ++ examples/run_sarek.sh | 39 ++ nextflow.config | 57 ++ workflows/synthetic.nf | 184 ++++++ 21 files changed, 1656 insertions(+), 2 deletions(-) create mode 100644 Makefile create mode 100644 PLAN.md create mode 100755 bin/formascute create mode 100644 conf/alpine.reference.config create mode 100644 conf/characterization.config create mode 100644 conf/experiments/batched_jobs.config create mode 100644 conf/experiments/independent_jobs.config create mode 100644 conf/experiments/nf1_featurization_batched.config create mode 100644 conf/experiments/nf1_featurization_independent.config create mode 100644 conf/experiments/nf1_queue_size_20.config create mode 100644 conf/experiments/queue_size_20.config create mode 100644 docs/clone-ready.md create mode 100644 docs/hpc-quickstart.md create mode 100644 docs/nf1-pipeline-target.md create mode 100644 docs/production-checklist.md create mode 100644 examples/curc_alpine.config create mode 100644 examples/run_sarek.sh create mode 100644 nextflow.config create mode 100644 workflows/synthetic.nf diff --git a/.gitignore b/.gitignore index 83972fa..f4698b9 100644 --- a/.gitignore +++ b/.gitignore @@ -181,6 +181,12 @@ dmypy.json # Cython debug symbols cython_debug/ +# Nextflow and FormaScute run outputs +.nextflow/ +.nextflow.log* +work/ +results/ + # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..09190fc --- /dev/null +++ b/Makefile @@ -0,0 +1,74 @@ +SHELL := /usr/bin/env bash + +EXPERIMENT ?= independent_jobs +ITEMS ?= 16 +BATCH_SIZE ?= +RUN_ID ?= +PROFILE ?= alpine +ACCOUNT ?= +PARTITION ?= amilan +QOS ?= normal + +ARGS := +ifneq ($(strip $(RUN_ID)),) +ARGS += --run-id $(RUN_ID) +endif +ifneq ($(strip $(ITEMS)),) +ARGS += --items $(ITEMS) +endif +ifneq ($(strip $(BATCH_SIZE)),) +ARGS += --batch-size $(BATCH_SIZE) +endif + +.PHONY: help check doctor preflight smoke run submit submit-dry-run list-experiments + +help: + @bash bin/formascute --help + @printf '\nMake targets:\n' + @printf ' make check\n' + @printf ' make doctor\n' + @printf ' make preflight ACCOUNT=amc-general\n' + @printf ' make smoke\n' + @printf ' make run EXPERIMENT=independent_jobs ITEMS=16\n' + @printf ' make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general\n' + @printf ' make submit-dry-run EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general\n' + @printf ' make list-experiments\n' + @printf '\nMake variables:\n' + @printf ' EXPERIMENT Experiment name from conf/experiments. Default: independent_jobs\n' + @printf ' ITEMS Number of synthetic work items. Default: 16\n' + @printf ' BATCH_SIZE Work items per Nextflow task. Default: experiment default\n' + @printf ' RUN_ID Output directory name under results/. Default: UTC timestamp\n' + @printf ' PROFILE Nextflow profile for make run. Default: alpine\n' + @printf ' ACCOUNT Slurm account/allocation for make submit. Default: none\n' + @printf ' PARTITION Slurm partition for make submit. Default: amilan\n' + @printf ' QOS Slurm QoS for make submit. Default: normal\n' + @printf '\nEnvironment overrides:\n' + @printf ' FORMASCUTE_NEXTFLOW_MODULE Module loaded in generated Slurm scripts\n' + @printf ' FORMASCUTE_CONTAINER_MODULE Container module loaded in generated Slurm scripts\n' + @printf ' FORMASCUTE_SCRATCH Shared scratch root; default /scratch/alpine/$$USER\n' + @printf ' FORMASCUTE_PROJECT Project/cache root; default /projects/$$USER\n' + +check: + @bash bin/formascute check + +doctor: + @bash bin/formascute doctor + +preflight: + @bash bin/formascute preflight $(if $(RUN_ID),--run-id $(RUN_ID)) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" + +smoke: + @bash bin/formascute run-local-smoke $(if $(RUN_ID),--run-id $(RUN_ID)) + +run: + @FORMASCUTE_ACCOUNT="$(ACCOUNT)" FORMASCUTE_PARTITION="$(PARTITION)" FORMASCUTE_QOS="$(QOS)" \ + bash bin/formascute run $(EXPERIMENT) --profile $(PROFILE) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" + +submit: + @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" + +submit-dry-run: + @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" --dry-run + +list-experiments: + @find conf/experiments -maxdepth 1 -type f -name '*.config' | sed 's#conf/experiments/##; s#\.config##' | sort diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..f4ce019 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,176 @@ +# FormaScute + +**FormaScute** is a reproducible characterization framework for +understanding how scientific workflows execute on HPC systems. + +> *Reveal the shape of workflow execution.* + +## Philosophy + +FormaScute is **not** a benchmarking suite. + +It is a characterization framework that helps answer engineering +questions about workflow execution by running small, reproducible +experiments against a known-working HPC configuration. + +The project begins from a **known-good Alpine Nextflow deployment** +rather than inventing a new execution model. The provided `examples/run_sarek.sh` +launcher and `examples/curc_alpine.config` serve as the initial reference +implementation. + +Each experiment changes **exactly one variable** while leaving +everything else untouched. + +## Goals + +- Understand execution behavior rather than maximize benchmark scores. +- Reach scientific results sooner by understanding scheduler behavior. +- Produce evidence that HPC engineers can interpret. +- Generate reproducible observations and targeted questions. +- Keep every experiment small, deterministic, and open source. + +## Baseline + +Start from the existing Alpine deployment: + +- Coordinator launched with `sbatch` +- Nextflow loads as a module +- Singularity/Apptainer environment +- Existing `NXF_WORK`, `NXF_TEMP`, and `NXF_HOME` +- Existing Slurm executor configuration +- Existing queue, QoS, account, retry, and submission settings + +Treat this configuration as the **control**. + +## Repository layout + +``` text +formascute/ +├── README.md +├── PLAN.md +├── nextflow.config +├── conf/ +│ ├── alpine.reference.config +│ ├── characterization.config +│ └── experiments/ +├── workflows/ +├── modules/ +├── bin/ +├── experiments/ +├── reports/ +├── results/ +└── docs/ +``` + +## Experiment philosophy + +Each experiment: + +1. Starts from the Alpine reference configuration. +2. Applies one configuration mutation. +3. Executes the same synthetic workflow. +4. Validates output correctness. +5. Collects Nextflow + Slurm metrics. +6. Produces observations. +7. Generates questions for HPC engineers if behavior is unexplained. + +## Canonical synthetic workflow + + Manifest + │ + Fan out N work items + │ + Aggregate + │ + Validate + +Synthetic work items support: + +- CPU work +- Memory allocation +- I/O +- Sleep +- Controlled failures + +No biological data are required. + +## Initial execution strategies + +- One Slurm job per Nextflow process +- Batched work items +- Slurm job arrays +- Persistent Nextflow coordinator + +## Initial characterization dimensions + +Only modify one at a time. + +- queueSize +- submitRateLimit +- batching +- concurrency +- QoS +- partition +- retries +- work directory placement +- scratch usage + +## Safety + +Every experiment must define: + +- maximum jobs +- maximum concurrent jobs +- maximum CPUs +- maximum memory +- maximum walltime +- maximum output size +- maximum retries + +Experiments terminate immediately if limits are exceeded. + +## Metrics + +Capture: + +- Nextflow trace +- Timeline +- Report +- DAG +- Slurm accounting +- Queue wait +- Runtime +- Retry behavior +- Output correctness + +Primary metric: + +> Time to validated scientific output. + +## Deliverables + +Each run produces: + + results/ + └── run-id/ + ├── experiment.yaml + ├── trace.tsv + ├── report.html + ├── timeline.html + ├── slurm.tsv + ├── validation.json + ├── summary.json + └── questions.md + +## Questions for HPC engineers + +FormaScute should never guess. + +When observations cannot be explained, generate concise engineering +questions supported by reproducible evidence. + +## Long-term vision + +FormaScute becomes a reusable characterization framework that any +institution can point at an HPC cluster to understand workflow behavior +before running production-scale scientific analyses. diff --git a/README.md b/README.md index db00f55..30bb67b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,104 @@ -# formascute -Exploring the structure of HPC workflow execution through small, reproducible experiments. +# FormaScute + +Reveal the shape of workflow execution. + +FormaScute takes its name from shape and scute: the patterned plates on a turtle +shell. The project is about seeing the shape of the shell before we need it, +making each layer of workflow execution visible before production science +depends on it. 🐢 + +FormaScute runs small, reproducible characterization experiments for workflow +engines on HPC systems. It starts from a known-working deployment, changes one +execution setting at a time, validates correctness, and records enough +provenance to make scheduler and filesystem behavior discussable with HPC +engineers. + +FormaScute is not a benchmarking suite. The goal is to understand execution +behavior before running production science. + +## Quick Start + +From the repository root: + +```bash +make check +make doctor +make smoke +``` + +On an HPC system with Nextflow and Slurm: + +```bash +module load nextflow +module load singularity +make check +make doctor +make preflight ACCOUNT=amc-general +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general +make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general +``` + +Run outputs are written to `results//`. +Preflight output is written to `results//preflight/`. + +## Entry Points + +- `make check`: verify the cloned repository has the expected local files and a + writable `results/` directory. +- `make doctor`: show available runtime tools. +- `make preflight ACCOUNT=amc-general`: record Slurm, module, filesystem, and + runtime context without submitting work. +- `make smoke`: run a tiny local Nextflow check. +- `make run EXPERIMENT=independent_jobs ITEMS=16`: run directly. +- `make submit EXPERIMENT=independent_jobs ITEMS=16`: submit the Nextflow + coordinator with `sbatch`. +- `make submit-dry-run EXPERIMENT=independent_jobs ITEMS=16`: generate the + coordinator script and preflight files without submitting to Slurm. +- `bin/formascute doctor`: show available runtime tools. +- `bin/formascute run-local-smoke`: run a tiny local Nextflow check. +- `bin/formascute run EXPERIMENT --profile alpine`: run an experiment with the + Slurm baseline profile. +- `bin/formascute submit EXPERIMENT`: submit the Nextflow coordinator with + `sbatch`. + +No installation step is required. Clone the repository on the HPC system and run +the Make targets from the repository root. If `make` is unavailable, use the +matching `bash bin/formascute` commands directly. + +Available initial experiments: + +- `independent_jobs` +- `batched_jobs` +- `queue_size_20` +- `nf1_featurization_independent` +- `nf1_featurization_batched` +- `nf1_queue_size_20` + +## Arguments + +- `EXPERIMENT`: experiment config name from `conf/experiments/`, without + `.config`. +- `ITEMS`: number of synthetic work items to run. +- `BATCH_SIZE`: number of work items per Nextflow task. Defaults to `1`, + `8` for `batched_jobs`, and `4` for `nf1_featurization_batched`. +- `RUN_ID`: output directory name under `results/`. Defaults to a UTC timestamp. +- `PROFILE`: Nextflow profile for `make run`. Defaults to `alpine`. +- `ACCOUNT`: Slurm account/allocation for `make submit`. Defaults to no explicit + account. +- `PARTITION`: Slurm partition for `make submit`. Defaults to `amilan`. +- `QOS`: Slurm QoS for `make submit`. Defaults to `normal`. + +Useful environment overrides: + +- `FORMASCUTE_NEXTFLOW_MODULE`: module loaded by generated Slurm scripts. + Defaults to `nextflow`. +- `FORMASCUTE_CONTAINER_MODULE`: container module loaded by generated Slurm + scripts. Defaults to `singularity`. +- `FORMASCUTE_SCRATCH`: shared scratch root. Defaults to `/scratch/alpine/$USER`. +- `FORMASCUTE_PROJECT`: project/cache root. Defaults to `/projects/$USER`. + +See `docs/clone-ready.md` for the first clone-and-run sequence, +`docs/hpc-quickstart.md` for HPC usage notes, +`docs/production-checklist.md` for first-run validation, +`docs/nf1-pipeline-target.md` for the target pipeline, and `PLAN.md` for the +project direction. diff --git a/bin/formascute b/bin/formascute new file mode 100755 index 0000000..8c04d10 --- /dev/null +++ b/bin/formascute @@ -0,0 +1,570 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +usage() { + cat <<'USAGE' +FormaScute + +Usage: + bin/formascute doctor + bin/formascute check + bin/formascute preflight [--run-id ID] [--account NAME] [--partition NAME] [--qos NAME] + bin/formascute run-local-smoke [--run-id ID] + bin/formascute run EXPERIMENT [--profile PROFILE] [--run-id ID] [--items N] [--batch-size N] [--account NAME] [--partition NAME] [--qos NAME] + bin/formascute submit EXPERIMENT [--run-id ID] [--items N] [--batch-size N] [--account NAME] [--partition NAME] [--qos NAME] [--dry-run] + +Examples: + bin/formascute doctor + bin/formascute check + bin/formascute preflight --account amc-general + bin/formascute run-local-smoke + bin/formascute run independent_jobs --profile alpine --items 16 + bin/formascute run batched_jobs --profile alpine --items 16 --batch-size 4 + bin/formascute submit independent_jobs --items 16 --account amc-general + +Experiments: + independent_jobs + batched_jobs + queue_size_20 + nf1_featurization_independent + nf1_featurization_batched + nf1_queue_size_20 + +Options: + EXPERIMENT Experiment config name from conf/experiments without .config. + --profile Nextflow profile to use for direct runs. Default: alpine. + --run-id Output directory name under results/. Default: UTC timestamp. + --items Number of synthetic work items. Default: 16. + --batch-size Work items per Nextflow task. Default: 1, 8 for batched_jobs, or 4 for nf1_featurization_batched. + --account Slurm account/allocation. Default: FORMASCUTE_ACCOUNT or none. + --partition Slurm partition. Default: FORMASCUTE_PARTITION or amilan. + --qos Slurm QoS. Default: FORMASCUTE_QOS or normal. + +Environment: + FORMASCUTE_NEXTFLOW_MODULE Module loaded by generated Slurm scripts. Default: nextflow. + FORMASCUTE_CONTAINER_MODULE Module loaded by generated Slurm scripts. Default: singularity. + FORMASCUTE_SCRATCH Shared scratch root. Default: /scratch/alpine/$USER. + FORMASCUTE_PROJECT Project root for caches. Default: /projects/$USER. +USAGE +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +run_id() { + date -u +"%Y%m%dT%H%M%SZ" +} + +validate_name() { + local label="$1" + local value="$2" + if [[ -n "$value" && ! "$value" =~ ^[A-Za-z0-9_.@:+-]+$ ]]; then + echo "$label contains unsupported characters: $value" >&2 + exit 2 + fi +} + +validate_positive_int() { + local label="$1" + local value="$2" + local max="$3" + if [[ ! "$value" =~ ^[0-9]+$ || "$value" -lt 1 || "$value" -gt "$max" ]]; then + echo "$label must be an integer from 1 to $max" >&2 + exit 2 + fi +} + +run_capture() { + local output="$1" + shift + { + printf '$' + printf ' %q' "$@" + printf '\n' + "$@" + } > "$output" 2>&1 || { + local status=$? + printf '\nexit_status: %s\n' "$status" >> "$output" + return 0 + } +} + +check_repo() { + local failed=0 + local required=( + "Makefile" + "nextflow.config" + "conf/alpine.reference.config" + "conf/characterization.config" + "workflows/synthetic.nf" + "docs/hpc-quickstart.md" + "docs/production-checklist.md" + "docs/nf1-pipeline-target.md" + ) + + echo "FormaScute repository check" + for path in "${required[@]}"; do + if [[ -e "$ROOT_DIR/$path" ]]; then + echo "ok: $path" + else + echo "missing: $path" >&2 + failed=1 + fi + done + + if bash -n "$ROOT_DIR/bin/formascute"; then + echo "ok: bin/formascute syntax" + else + failed=1 + fi + + if find "$ROOT_DIR/conf/experiments" -maxdepth 1 -type f -name '*.config' | grep -q .; then + echo "ok: experiment configs found" + else + echo "missing: experiment configs" >&2 + failed=1 + fi + + mkdir -p "$ROOT_DIR/results" + if [[ -w "$ROOT_DIR/results" ]]; then + echo "ok: results directory writable" + else + echo "not writable: results" >&2 + failed=1 + fi + + return "$failed" +} + +write_experiment_file() { + local outdir="$1" + local experiment="$2" + local profile="$3" + local id="$4" + local items="$5" + local batch_size="$6" + local account="$7" + local partition="$8" + local qos="$9" + + mkdir -p "$outdir" + cat > "$outdir/experiment.yaml" </dev/null || true)" +workflow: "workflows/synthetic.nf" +EOF +} + +preflight() { + local id="$1" + local account="$2" + local partition="$3" + local qos="$4" + local outdir="$ROOT_DIR/results/$id/preflight" + + validate_name "run-id" "$id" + validate_name "account" "$account" + validate_name "partition" "$partition" + validate_name "qos" "$qos" + + mkdir -p "$outdir" + + { + echo "run_id: $id" + echo "created_utc: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" + echo "repo: $ROOT_DIR" + echo "host: $(hostname 2>/dev/null || true)" + echo "user: ${USER:-}" + echo "pwd: $(pwd)" + echo "account: $account" + echo "partition: $partition" + echo "qos: $qos" + echo "git_commit: $(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || true)" + } > "$outdir/summary.txt" + + env | sort > "$outdir/environment.txt" + run_capture "$outdir/nextflow-version.txt" nextflow -version + run_capture "$outdir/python-version.txt" python3 --version + + if have sinfo; then + run_capture "$outdir/sinfo.txt" sinfo -o "%P|%a|%l|%D|%t|%c|%m|%f|%G" + run_capture "$outdir/sinfo-target-partition.txt" sinfo -p "$partition" -o "%P|%a|%l|%D|%t|%c|%m|%f|%G" + fi + + if have squeue; then + run_capture "$outdir/squeue-user.txt" squeue -u "${USER:-}" -o "%.18i %.9P %.30j %.8u %.2t %.10M %.10l %.6D %R" + fi + + if have sacctmgr; then + run_capture "$outdir/sacctmgr-user.txt" sacctmgr -P show assoc "user=${USER:-}" format=Cluster,Account,User,Partition,QOS + fi + + if have sbatch; then + run_capture "$outdir/sbatch-version.txt" sbatch --version + fi + + if have sacct; then + run_capture "$outdir/sacct-version.txt" sacct --version + fi + + if have singularity; then + run_capture "$outdir/container-runtime.txt" singularity --version + elif have apptainer; then + run_capture "$outdir/container-runtime.txt" apptainer --version + fi + + if type module >/dev/null 2>&1; then + module -t list > "$outdir/modules-loaded.txt" 2>&1 || true + module -t avail nextflow > "$outdir/modules-nextflow.txt" 2>&1 || true + module -t avail singularity > "$outdir/modules-singularity.txt" 2>&1 || true + module -t avail apptainer > "$outdir/modules-apptainer.txt" 2>&1 || true + fi + + echo "Preflight written to $outdir" +} + +collect_slurm_accounting() { + local outdir="$1" + local trace="$outdir/trace.tsv" + local slurm="$outdir/slurm.tsv" + + if ! have sacct || [[ ! -f "$trace" ]]; then + return 0 + fi + + python3 - "$trace" "$slurm.jobs" <<'PY' +import csv +import sys + +trace_path, jobs_path = sys.argv[1:3] +jobs = [] +with open(trace_path, newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + if not reader.fieldnames or "native_id" not in reader.fieldnames: + raise SystemExit(0) + for row in reader: + native_id = (row.get("native_id") or "").strip() + if native_id and native_id not in jobs: + jobs.append(native_id) + +with open(jobs_path, "w") as handle: + for job in jobs: + handle.write(job + "\n") +PY + + if [[ -s "$slurm.jobs" ]]; then + sacct -j "$(paste -sd, "$slurm.jobs")" --parsable2 \ + --format=JobID,JobName%40,Partition,Account,AllocCPUS,State,ExitCode,Submit,Start,End,Elapsed,ReqMem,MaxRSS,NTasks,NNodes \ + > "$slurm" || true + fi + rm -f "$slurm.jobs" +} + +doctor() { + echo "FormaScute doctor" + echo "repo: $ROOT_DIR" + + if have nextflow; then + if ! nextflow -version 2>&1 | sed -n '1,6p'; then + echo "nextflow: found but not runnable" + fi + else + echo "nextflow: missing" + fi + + if have sbatch; then + echo "sbatch: $(command -v sbatch)" + else + echo "sbatch: missing" + fi + + if have sacct; then + echo "sacct: $(command -v sacct)" + else + echo "sacct: missing" + fi + + if have singularity; then + singularity --version + elif have apptainer; then + apptainer --version + else + echo "singularity/apptainer: missing" + fi +} + +run_nextflow() { + local experiment="$1" + local profile="$2" + local id="$3" + local items="$4" + local batch_size="$5" + local account="$6" + local partition="$7" + local qos="$8" + local outdir="$ROOT_DIR/results/$id" + local experiment_config="$ROOT_DIR/conf/experiments/${experiment}.config" + + validate_name "run-id" "$id" + validate_name "experiment" "$experiment" + validate_name "profile" "$profile" + validate_name "account" "$account" + validate_name "partition" "$partition" + validate_name "qos" "$qos" + validate_positive_int "items" "$items" 256 + validate_positive_int "batch-size" "$batch_size" 64 + + if [[ ! -f "$experiment_config" ]]; then + echo "Unknown experiment: $experiment" >&2 + exit 2 + fi + + if ! have nextflow; then + echo "nextflow is required for runs" >&2 + exit 127 + fi + + write_experiment_file "$outdir" "$experiment" "$profile" "$id" "$items" "$batch_size" "$account" "$partition" "$qos" + preflight "$id" "$account" "$partition" "$qos" + + { + echo "FORMASCUTE_ACCOUNT=\"$account\" FORMASCUTE_PARTITION=\"$partition\" FORMASCUTE_QOS=\"$qos\" nextflow run workflows/synthetic.nf -profile $profile --run_id $id --outdir $outdir --items $items --batch_size $batch_size" + } > "$outdir/run-command.txt" + + set +e + FORMASCUTE_ACCOUNT="$account" FORMASCUTE_PARTITION="$partition" FORMASCUTE_QOS="$qos" \ + nextflow run "$ROOT_DIR/workflows/synthetic.nf" \ + -profile "$profile" \ + -c "$ROOT_DIR/nextflow.config" \ + -c "$ROOT_DIR/conf/characterization.config" \ + -c "$experiment_config" \ + --run_id "$id" \ + --outdir "$outdir" \ + --items "$items" \ + --batch_size "$batch_size" \ + 2>&1 | tee "$outdir/nextflow.log" + local nextflow_status="${PIPESTATUS[0]}" + set -e + + collect_slurm_accounting "$outdir" + echo "nextflow_exit_status: $nextflow_status" > "$outdir/completion_status.txt" + + if [[ "$nextflow_status" -ne 0 ]]; then + echo "Run failed; artifacts written to $outdir" >&2 + exit "$nextflow_status" + fi + + echo "Run written to $outdir" +} + +submit_slurm() { + local experiment="$1" + local id="$2" + local items="$3" + local batch_size="$4" + local account="$5" + local partition="$6" + local qos="$7" + local dry_run="$8" + local outdir="$ROOT_DIR/results/$id" + local script="$outdir/coordinator.sbatch" + local experiment_config="$ROOT_DIR/conf/experiments/${experiment}.config" + local account_directive="" + local qos_directive="" + + validate_name "run-id" "$id" + validate_name "experiment" "$experiment" + validate_name "account" "$account" + validate_name "partition" "$partition" + validate_name "qos" "$qos" + validate_positive_int "items" "$items" 256 + validate_positive_int "batch-size" "$batch_size" 64 + + if [[ ! -f "$experiment_config" ]]; then + echo "Unknown experiment: $experiment" >&2 + exit 2 + fi + + if [[ "$dry_run" != "true" ]] && ! have sbatch; then + echo "sbatch is required for submit" >&2 + exit 127 + fi + + if [[ -n "$account" ]]; then + account_directive="#SBATCH --account=$account" + fi + + if [[ -n "$qos" ]]; then + qos_directive="#SBATCH --qos=$qos" + fi + + mkdir -p "$outdir" + preflight "$id" "$account" "$partition" "$qos" + cat > "$script" </dev/null 2>&1; then + module purge + module load "\${FORMASCUTE_NEXTFLOW_MODULE:-nextflow}" + module load "\${FORMASCUTE_CONTAINER_MODULE:-singularity}" +else + echo "Environment modules are not available; using commands already on PATH." +fi + +FORMASCUTE_ACCOUNT="$account" FORMASCUTE_PARTITION="$partition" FORMASCUTE_QOS="$qos" \\ + bash bin/formascute run "$experiment" --profile alpine --run-id "$id" --items "$items" --batch-size "$batch_size" +EOF + + if [[ "$dry_run" == "true" ]]; then + echo "Dry run: not submitting to Slurm" | tee "$outdir/slurm-submit.txt" + else + sbatch "$script" | tee "$outdir/slurm-submit.txt" + fi + echo "Coordinator script written to $script" +} + +cmd="${1:-}" +shift || true + +case "$cmd" in + doctor) + doctor + ;; + check) + check_repo + ;; + preflight) + id="$(run_id)" + account="${FORMASCUTE_ACCOUNT:-}" + partition="${FORMASCUTE_PARTITION:-amilan}" + qos="${FORMASCUTE_QOS:-normal}" + while [[ $# -gt 0 ]]; do + case "$1" in + --run-id) id="$2"; shift 2 ;; + --account) account="$2"; shift 2 ;; + --partition) partition="$2"; shift 2 ;; + --qos) qos="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac + done + preflight "$id" "$account" "$partition" "$qos" + ;; + run-local-smoke) + id="$(run_id)" + while [[ $# -gt 0 ]]; do + case "$1" in + --run-id) id="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac + done + run_nextflow "independent_jobs" "local_smoke" "$id" 4 1 "" "local" "" + ;; + run) + experiment="${1:-}" + if [[ -z "$experiment" ]]; then + usage + exit 2 + fi + shift + + profile="alpine" + id="$(run_id)" + items=16 + account="${FORMASCUTE_ACCOUNT:-}" + partition="${FORMASCUTE_PARTITION:-amilan}" + qos="${FORMASCUTE_QOS:-normal}" + case "$experiment" in + batched_jobs) batch_size=8 ;; + nf1_featurization_batched) batch_size=4 ;; + *) batch_size=1 ;; + esac + + while [[ $# -gt 0 ]]; do + case "$1" in + --profile) profile="$2"; shift 2 ;; + --run-id) id="$2"; shift 2 ;; + --items) items="$2"; shift 2 ;; + --batch-size) batch_size="$2"; shift 2 ;; + --account) account="$2"; shift 2 ;; + --partition) partition="$2"; shift 2 ;; + --qos) qos="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac + done + + run_nextflow "$experiment" "$profile" "$id" "$items" "$batch_size" "$account" "$partition" "$qos" + ;; + submit) + experiment="${1:-}" + if [[ -z "$experiment" ]]; then + usage + exit 2 + fi + shift + + id="$(run_id)" + items=16 + account="${FORMASCUTE_ACCOUNT:-}" + partition="${FORMASCUTE_PARTITION:-amilan}" + qos="${FORMASCUTE_QOS:-normal}" + dry_run="false" + case "$experiment" in + batched_jobs) batch_size=8 ;; + nf1_featurization_batched) batch_size=4 ;; + *) batch_size=1 ;; + esac + + while [[ $# -gt 0 ]]; do + case "$1" in + --run-id) id="$2"; shift 2 ;; + --items) items="$2"; shift 2 ;; + --batch-size) batch_size="$2"; shift 2 ;; + --account) account="$2"; shift 2 ;; + --partition) partition="$2"; shift 2 ;; + --qos) qos="$2"; shift 2 ;; + --dry-run) dry_run="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 2 ;; + esac + done + + submit_slurm "$experiment" "$id" "$items" "$batch_size" "$account" "$partition" "$qos" "$dry_run" + ;; + -h|--help|"") + usage + ;; + *) + echo "Unknown command: $cmd" >&2 + usage + exit 2 + ;; +esac diff --git a/conf/alpine.reference.config b/conf/alpine.reference.config new file mode 100644 index 0000000..c950905 --- /dev/null +++ b/conf/alpine.reference.config @@ -0,0 +1,73 @@ +/* + * Reference Slurm and Singularity settings for the CURC Alpine baseline. + * Treat this as the control configuration. Experiments should override one + * setting at a time in a separate config file. + */ + +def formascute_user = System.getenv('USER') ?: 'unknown' +def formascute_scratch = System.getenv('FORMASCUTE_SCRATCH') ?: "/scratch/alpine/${formascute_user}" +def formascute_project = System.getenv('FORMASCUTE_PROJECT') ?: "/projects/${formascute_user}" + +params { + config_profile_description = 'CURC Alpine cluster profile' + config_profile_contact = 'Thomas Danhorn' + slurm_account = System.getenv('FORMASCUTE_ACCOUNT') ?: '' + slurm_partition = System.getenv('FORMASCUTE_PARTITION') ?: 'amilan' + slurm_qos = System.getenv('FORMASCUTE_QOS') ?: 'normal' + scratch_root = formascute_scratch + project_root = formascute_project + max_memory = 240.GB + max_cpus = 48 + max_time = 24.h +} + +singularity { + enabled = true + cacheDir = "${params.project_root}/container_cache" + autoMounts = true + runOptions = "-B ${params.scratch_root}:/tmp" +} + +process { + executor = 'slurm' + queue = params.slurm_partition + errorStrategy = { task.attempt <= 3 ? 'retry' : 'finish' } + maxRetries = 3 + clusterOptions = { + def opts = [] + if (params.slurm_account) { + opts << "--account=${params.slurm_account}" + } + if (params.slurm_qos) { + opts << "--qos=${params.slurm_qos}" + } + opts.join(' ') + } + + withLabel:process_long { + clusterOptions = { + def opts = [] + if (params.slurm_account) { + opts << "--account=${params.slurm_account}" + } + if (params.slurm_qos) { + opts << "--qos=${params.slurm_qos}" + } + opts.join(' ') + } + time = 24.h + } +} + +executor { + name = 'slurm' + queueSize = 200 + perJobMemLimit = true + submitRateLimit = '30 / 1 min' + killBatchSize = 50 +} + +env { + SINGULARITY_TMPDIR = "${params.scratch_root}/singularity/tmp" + SINGULARITY_CACHEDIR = "${params.scratch_root}/singularity/cache" +} diff --git a/conf/characterization.config b/conf/characterization.config new file mode 100644 index 0000000..64e6701 --- /dev/null +++ b/conf/characterization.config @@ -0,0 +1,18 @@ +/* + * Shared safety limits for small characterization runs. + */ + +params { + max_items = 256 + max_batch_size = 64 + max_sleep_seconds = 300 + max_cpu_seconds = 300 + max_mem_mb = 4096 + max_io_mb = 256 +} + +process { + cpus = 1 + memory = '1 GB' + time = '15 min' +} diff --git a/conf/experiments/batched_jobs.config b/conf/experiments/batched_jobs.config new file mode 100644 index 0000000..658535f --- /dev/null +++ b/conf/experiments/batched_jobs.config @@ -0,0 +1,4 @@ +params { + experiment = 'batched_jobs' + batch_size = 8 +} diff --git a/conf/experiments/independent_jobs.config b/conf/experiments/independent_jobs.config new file mode 100644 index 0000000..24c7c40 --- /dev/null +++ b/conf/experiments/independent_jobs.config @@ -0,0 +1,4 @@ +params { + experiment = 'independent_jobs' + batch_size = 1 +} diff --git a/conf/experiments/nf1_featurization_batched.config b/conf/experiments/nf1_featurization_batched.config new file mode 100644 index 0000000..b37f771 --- /dev/null +++ b/conf/experiments/nf1_featurization_batched.config @@ -0,0 +1,7 @@ +params { + experiment = 'nf1_featurization_batched' + items = 16 + batch_size = 4 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/nf1_featurization_independent.config b/conf/experiments/nf1_featurization_independent.config new file mode 100644 index 0000000..e2151a1 --- /dev/null +++ b/conf/experiments/nf1_featurization_independent.config @@ -0,0 +1,7 @@ +params { + experiment = 'nf1_featurization_independent' + items = 16 + batch_size = 1 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/nf1_queue_size_20.config b/conf/experiments/nf1_queue_size_20.config new file mode 100644 index 0000000..2c75616 --- /dev/null +++ b/conf/experiments/nf1_queue_size_20.config @@ -0,0 +1,12 @@ +executor { + queueSize = 20 + submitRateLimit = '20 / 1 min' +} + +params { + experiment = 'nf1_queue_size_20' + items = 16 + batch_size = 1 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/queue_size_20.config b/conf/experiments/queue_size_20.config new file mode 100644 index 0000000..39d9f8c --- /dev/null +++ b/conf/experiments/queue_size_20.config @@ -0,0 +1,7 @@ +executor { + queueSize = 20 +} + +params { + experiment = 'queue_size_20' +} diff --git a/docs/clone-ready.md b/docs/clone-ready.md new file mode 100644 index 0000000..11f4d58 --- /dev/null +++ b/docs/clone-ready.md @@ -0,0 +1,39 @@ +# Clone-Ready Alpine Test + +Everything needed for the first Alpine characterization runs is in this +repository. The first phase uses only synthetic data generated by the workflow. +No NF1 image data are required. + +From an Alpine login node: + +```bash +git clone formascute +cd formascute +module load nextflow +module load singularity +make check +make doctor +make preflight ACCOUNT= +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +``` + +After the job finishes, inspect: + +```bash +cat results//completion_status.txt +cat results//validation.json +head results//trace.tsv +head results//slurm.tsv +cat results//coordinator_*.err +``` + +Then run the two comparison experiments: + +```bash +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= +make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= +``` + +Keep these runs small until the outputs show stable scheduler submission, +accounting, retry, and validation behavior. diff --git a/docs/hpc-quickstart.md b/docs/hpc-quickstart.md new file mode 100644 index 0000000..ac2fd79 --- /dev/null +++ b/docs/hpc-quickstart.md @@ -0,0 +1,95 @@ +# HPC Quickstart + +Clone the repository on the HPC login node, load the site modules, and run a +small characterization experiment from the repository root. + +```bash +git clone formascute +cd formascute +module load nextflow +module load singularity +make check +make doctor +make preflight ACCOUNT=amc-general +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general +make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general +``` + +Each run writes a timestamped directory under `results/` with: + +- `experiment.yaml` +- `trace.tsv` +- `timeline.html` +- `report.html` +- `dag.html` +- `validation.json` +- `summary.json` +- `questions.md` +- `preflight/` + +Use `make doctor` and `make submit` from a login node. Avoid `make smoke` and +`make run` on login nodes unless CURC support specifically tells you the run is +acceptable there. + +No installation step is required. If `make` is not available, use the equivalent +commands directly: + +```bash +bash bin/formascute doctor +bash bin/formascute preflight --account amc-general +bash bin/formascute submit nf1_featurization_independent --items 16 --account amc-general --dry-run +bash bin/formascute submit independent_jobs --items 16 --account amc-general +``` + +## Baseline And Experiments + +The Alpine baseline lives in `conf/alpine.reference.config`. Treat it as the +control. Experiment configs in `conf/experiments/` change one setting at a time. + +Initial experiments: + +- `independent_jobs`: one work item per task. +- `batched_jobs`: multiple work items per task. +- `queue_size_20`: Slurm executor queue size set to 20. +- `nf1_featurization_independent`: small approximation of the NF1 Stage 3 + parent/child fan-out pattern. +- `nf1_featurization_batched`: same synthetic NF1-like work grouped into + batches. +- `nf1_queue_size_20`: NF1-like independent work with a conservative queue size. + +Run examples: + +```bash +make submit EXPERIMENT=independent_jobs ITEMS=32 ACCOUNT=amc-general +make submit EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4 ACCOUNT=amc-general +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT=amc-general +make run EXPERIMENT=independent_jobs ITEMS=32 +make run EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4 +make run EXPERIMENT=queue_size_20 ITEMS=32 +``` + +## Arguments + +- `EXPERIMENT`: experiment config name from `conf/experiments/`, without + `.config`. +- `ITEMS`: number of synthetic work items to run. +- `BATCH_SIZE`: number of work items per Nextflow task. Defaults to `1`, + `8` for `batched_jobs`, and `4` for `nf1_featurization_batched`. +- `RUN_ID`: output directory name under `results/`. Defaults to a UTC timestamp. +- `PROFILE`: Nextflow profile for `make run`. Defaults to `alpine`. +- `ACCOUNT`: Slurm account/allocation for `make submit`. Defaults to no explicit + account. +- `PARTITION`: Slurm partition for `make submit`. Defaults to `amilan`. +- `QOS`: Slurm QoS for `make submit`. Defaults to `normal`. + +Useful environment overrides: + +- `FORMASCUTE_NEXTFLOW_MODULE`: module loaded by generated Slurm scripts. + Defaults to `nextflow`. +- `FORMASCUTE_CONTAINER_MODULE`: container module loaded by generated Slurm + scripts. Defaults to `singularity`. +- `FORMASCUTE_SCRATCH`: shared scratch root. Defaults to `/scratch/alpine/$USER`. +- `FORMASCUTE_PROJECT`: project/cache root. Defaults to `/projects/$USER`. + +When behavior is unclear, share the complete run directory with HPC support. diff --git a/docs/nf1-pipeline-target.md b/docs/nf1-pipeline-target.md new file mode 100644 index 0000000..016bce3 --- /dev/null +++ b/docs/nf1-pipeline-target.md @@ -0,0 +1,67 @@ +# NF1 3D Organoid Pipeline Target + +FormaScute is aimed at improving the Alpine execution model for +`WayScience/NF1_3D_organoid_profiling_pipeline`. + +The current NF1 pipeline uses Slurm directly from shell scripts. The largest +scheduler pressure appears in Stage 3 feature extraction and Stage 4 profile +merging, where parent scripts read work lists, throttle with `squeue`, and submit +many child jobs with `sbatch`. + +Relevant current behavior: + +- Stage 3 featurization submits one child job per patient, well-FOV, feature, + compartment, channel, and processor combination. +- Stage 3 throttles based on the user's queued/running Slurm job count. +- Stage 4 profile merging submits one child job per patient and well-FOV. +- Alpine CPU jobs use `amilan`; GPU jobs use `aa100`. +- The README describes a maximum of 990 concurrent jobs per user. +- The pipeline uses both CPU and GPU jobs, with larger GPU work for SAMMed3D. +- Per well-FOV storage can be around 1-2 GB across raw, z-stack, mask, feature, + and profile outputs. + +## Characterization Questions + +Start with scheduler behavior before touching real image data. These experiments +must stay synthetic, small, and reversible until Alpine behavior is understood. + +1. Can Nextflow submit the same kind of fan-out workload more safely than the + current parent/child shell scripts? +2. What `queueSize` and `submitRateLimit` keep Alpine responsive below the + practical user job limit? +3. Does batching multiple well-FOV-like work items into one Slurm task reduce + scheduler load without hiding failures or making retries too expensive? +4. Which work should stay as independent jobs because runtime or memory varies + too much? +5. How should CPU and GPU work be separated so CPU queue pressure does not block + SAMMed3D-style GPU tasks? + +## Initial Experiments + +Use only synthetic work in this phase: + +```bash +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= +make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= +``` + +Interpretation: + +- `nf1_featurization_independent` approximates the current one-child-job-per-work + item pattern. +- `nf1_featurization_batched` tests whether grouping well-FOV-like work reduces + queue pressure. +- `nf1_queue_size_20` tests a conservative Nextflow queue size well below the + pipeline's documented 990-user-job scale. + +## What This Does Not Answer Yet + +These experiments do not measure image-processing runtime, GPU memory, CellPose, +SAMMed3D, DuckDB, or parquet performance. They only characterize scheduler, +submission, retry, validation, and artifact behavior with a small controlled +workload. + +Do not use NF1 image data in this phase. Data-based testing belongs in a later +phase, after synthetic runs have shown stable submission, accounting, +validation, and retry behavior on Alpine. diff --git a/docs/production-checklist.md b/docs/production-checklist.md new file mode 100644 index 0000000..1aeb019 --- /dev/null +++ b/docs/production-checklist.md @@ -0,0 +1,62 @@ +# Production Checklist + +Use this checklist before increasing run size on an HPC system. + +## First Clone + +```bash +git clone formascute +cd formascute +module load nextflow +module load singularity +make check +make doctor +make preflight ACCOUNT= +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +``` + +Review `results//preflight/` before submitting work. + +Confirm: + +- `nextflow-version.txt` shows a runnable Nextflow installation. +- `sinfo-target-partition.txt` includes the requested partition. +- `sacctmgr-user.txt` shows the expected allocation and QoS when available. +- `container-runtime.txt` shows Singularity or Apptainer. +- `summary.txt` records the expected account, partition, and QoS. +- `results//coordinator.sbatch` from `submit-dry-run` has the expected + Slurm directives. + +## First Scheduler Run + +Submit the smallest Slurm-backed experiment: + +```bash +make submit EXPERIMENT=independent_jobs ITEMS=4 ACCOUNT= +``` + +Review the run directory after completion. + +Confirm: + +- `completion_status.txt` reports `nextflow_exit_status: 0`. +- `validation.json` reports `"valid": true`. +- `trace.tsv` exists and includes Slurm native job IDs. +- `slurm.tsv` exists when accounting is available. +- `coordinator_.out` and `coordinator_.err` do not contain module, + filesystem, scheduler, or container errors. + +## Scale-Up Rule + +Increase only one setting at a time. Prefer this sequence: + +1. `ITEMS=4` +2. `ITEMS=16` +3. `ITEMS=32` +4. `EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4` +5. `EXPERIMENT=queue_size_20 ITEMS=16` +6. `EXPERIMENT=nf1_featurization_independent ITEMS=16` +7. `EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4` + +Stop and review artifacts whenever validation fails, retry behavior appears, or +queue wait differs from expectations. diff --git a/examples/curc_alpine.config b/examples/curc_alpine.config new file mode 100644 index 0000000..2fa2fb4 --- /dev/null +++ b/examples/curc_alpine.config @@ -0,0 +1,51 @@ +/* + * ------------------------------------------------------------- + * Nextflow nf-core config file for the University of Colorado + * Research Computing alpine HPC cluster + * ------------------------------------------------------------- + * Defines SLURM process executor and singularity settings. + * + */ +params { + config_profile_description = 'CURC alpine cluster profile' + config_profile_contact = 'Thomas Danhorn' +} + +singularity { + enabled = true + cacheDir = '/projects/$USER/container_cache' + autoMounts = true + // /tmp is often too small -> redirect to scratch: + runOptions = "-B /scratch/alpine/$USER:/tmp" +} + +process{ + executor = 'slurm' + queue = 'acpu' + errorStrategy = { task.attempt <= 3 ? "retry" : "finish" } + process.maxRetries = 3 + clusterOptions = '--account=amc-general --qos=cpu-normal' + withLabel:process_long { + clusterOptions = '--qos=cpu-normal' + max_time = 24.h + } +} + +executor{ + name = 'slurm' + queueSize = 200 + perJobMemLimit = true + submitRateLimit = '30 / 1 min' + killBatchSize = 50 +} + +params { + max_memory = 240.GB + max_cpus = 48 + max_time = 24.h +} + +env { + SINGULARITY_TMPDIR = "/scratch/alpine/$USER/singularity/tmp" + SINGULARITY_CACHEDIR = "/scratch/alpine/$USER/singularity/cache" +} diff --git a/examples/run_sarek.sh b/examples/run_sarek.sh new file mode 100644 index 0000000..c446003 --- /dev/null +++ b/examples/run_sarek.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#SBATCH -J sarek +#SBATCH --partition acpu +#SBATCH --account=amc-general +#SBATCH -o log/sarek_%j.out # Output file with the job ID +#SBATCH -e log/sarek_%j.err # Error file with the job ID +#SBATCH -t 24:00:00 # Set the wall time: D-HH:MM:SS +#SBATCH --qos=cpu-normal +#SBATCH -n 1 -c 2 # ask for number of nodes/cores +#SBATCH --mem=18GB # Specify memory allocation + +set -euo pipefail + +# make software accessible: +module load nextflow/23.04 +module load singularity/3.7.4 + +echo ____________________________________ +nextflow -version +echo ____________________________________ +singularity --version +echo ____________________________________ + + +## Override defaults set by nextflow module: +export NXF_WORK="$scrproj/work" +export NXF_TEMP="$scrproj/tmp" +export NXF_HOME="/projects/$USER/software/nextflow_config" +export _JAVA_OPTIONS='-Xmx16G' # increase heap to avoid java.lang.OutOfMemoryError + + + +nextflow run "$pipelinepath" -ansi-log false \ + --input "$samplefile" --trim_fastq \ + --save_trimmed --save_mapped --save_output_as_bam --outdir "$pipeoutdir" \ + --tools mutect2,strelka,tiddit,freebayes,vep,snpeff -resume \ + -c curc_alpine.config,fastp.config + + diff --git a/nextflow.config b/nextflow.config new file mode 100644 index 0000000..f2659bd --- /dev/null +++ b/nextflow.config @@ -0,0 +1,57 @@ +nextflow.enable.dsl = 2 + +params.run_id = null +params.outdir = null +params.experiment = 'independent_jobs' +params.items = 8 +params.batch_size = 1 +params.sleep_seconds = 1 +params.cpu_seconds = 0 +params.mem_mb = 0 +params.io_mb = 1 +params.fail_items = '' +params.max_items = 256 +params.max_batch_size = 64 +params.max_sleep_seconds = 300 +params.max_cpu_seconds = 300 +params.max_mem_mb = 4096 +params.max_io_mb = 256 + +profiles { + local_smoke { + process.executor = 'local' + executor.queueSize = 2 + params.items = 4 + params.batch_size = 1 + params.sleep_seconds = 1 + params.io_mb = 1 + } + + alpine { + includeConfig 'conf/alpine.reference.config' + } +} + +trace { + enabled = true + file = "${params.outdir ?: 'results/' + (params.run_id ?: 'manual')}/trace.tsv" + overwrite = true +} + +timeline { + enabled = true + file = "${params.outdir ?: 'results/' + (params.run_id ?: 'manual')}/timeline.html" + overwrite = true +} + +report { + enabled = true + file = "${params.outdir ?: 'results/' + (params.run_id ?: 'manual')}/report.html" + overwrite = true +} + +dag { + enabled = true + file = "${params.outdir ?: 'results/' + (params.run_id ?: 'manual')}/dag.html" + overwrite = true +} diff --git a/workflows/synthetic.nf b/workflows/synthetic.nf new file mode 100644 index 0000000..edf41da --- /dev/null +++ b/workflows/synthetic.nf @@ -0,0 +1,184 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +def outdir = params.outdir ?: "results/${params.run_id ?: 'manual'}" +def items = params.items as int +def batchSize = params.batch_size as int +def failItems = (params.fail_items ?: '') + .split(',') + .collect { it.trim() } + .findAll { it } + .collect { it as int } + .toSet() + +if (items < 1 || items > (params.max_items as int)) { + error "params.items must be between 1 and ${params.max_items}" +} + +if (batchSize < 1 || batchSize > (params.max_batch_size as int)) { + error "params.batch_size must be between 1 and ${params.max_batch_size}" +} + +if ((params.sleep_seconds as int) > (params.max_sleep_seconds as int)) { + error "params.sleep_seconds exceeds ${params.max_sleep_seconds}" +} + +if ((params.cpu_seconds as int) > (params.max_cpu_seconds as int)) { + error "params.cpu_seconds exceeds ${params.max_cpu_seconds}" +} + +if ((params.mem_mb as int) > (params.max_mem_mb as int)) { + error "params.mem_mb exceeds ${params.max_mem_mb}" +} + +if ((params.io_mb as int) > (params.max_io_mb as int)) { + error "params.io_mb exceeds ${params.max_io_mb}" +} + +process CHARACTERIZE_ITEM { + tag { "batch_${batch_id}" } + publishDir "${outdir}/work-items", mode: 'copy' + + input: + tuple val(batch_id), val(item_ids), val(should_fail) + + output: + path "batch_${batch_id}.jsonl" + + script: + def itemCsv = item_ids.join(',') + """ + set -euo pipefail + + python3 - <<'PY' +import hashlib +import json +import time +from pathlib import Path + +batch_id = ${batch_id} +item_ids = [int(value) for value in "${itemCsv}".split(",") if value] +should_fail = "${should_fail}" == "true" +sleep_seconds = ${params.sleep_seconds} +cpu_seconds = ${params.cpu_seconds} +mem_mb = ${params.mem_mb} +io_mb = ${params.io_mb} + +if should_fail: + raise SystemExit(f"Intentional failure for batch {batch_id}") + +scratch = Path(f"scratch_{batch_id}") +scratch.mkdir(exist_ok=True) + +with Path(f"batch_{batch_id}.jsonl").open("w") as handle: + for item_id in item_ids: + start_epoch = int(time.time()) + time.sleep(sleep_seconds) + + if cpu_seconds > 0: + end = time.time() + cpu_seconds + value = 0 + while time.time() < end: + value = (value + 1) % 1000003 + + if mem_mb > 0: + buf = bytearray(mem_mb * 1024 * 1024) + buf[0] = 1 + buf[-1] = 1 + + checksum = "" + if io_mb > 0: + path = scratch / f"item_{item_id}.bin" + block = b"0" * 1024 * 1024 + digest = hashlib.sha256() + with path.open("wb") as output: + for _ in range(io_mb): + output.write(block) + digest.update(block) + checksum = digest.hexdigest() + + end_epoch = int(time.time()) + handle.write(json.dumps({ + "item_id": item_id, + "batch_id": batch_id, + "start_epoch": start_epoch, + "end_epoch": end_epoch, + "checksum": checksum, + }) + "\\n") +PY + """ +} + +process VALIDATE_RESULTS { + publishDir "${outdir}", mode: 'copy' + + input: + path batch_files + + output: + path 'validation.json' + path 'summary.json' + path 'questions.md' + + script: + """ + set -euo pipefail + + python3 - <<'PY' +import json +from pathlib import Path + +expected_items = ${items} +records = [] +for path in sorted(Path('.').glob('batch_*.jsonl')): + for line in path.read_text().splitlines(): + if line.strip(): + records.append(json.loads(line)) + +observed = sorted(record['item_id'] for record in records) +expected = list(range(1, expected_items + 1)) +valid = observed == expected + +validation = { + 'valid': valid, + 'expected_items': expected_items, + 'observed_items': len(records), + 'missing_items': sorted(set(expected) - set(observed)), + 'unexpected_items': sorted(set(observed) - set(expected)), +} + +durations = [record['end_epoch'] - record['start_epoch'] for record in records] +summary = { + 'experiment': '${params.experiment}', + 'run_id': '${params.run_id ?: 'manual'}', + 'items': expected_items, + 'batch_size': ${batchSize}, + 'valid': valid, + 'min_item_seconds': min(durations) if durations else None, + 'max_item_seconds': max(durations) if durations else None, + 'total_item_seconds': sum(durations), +} + +Path('validation.json').write_text(json.dumps(validation, indent=2) + '\\n') +Path('summary.json').write_text(json.dumps(summary, indent=2) + '\\n') +Path('questions.md').write_text( + '# Questions\\n\\n' + '- Review trace.tsv and Slurm accounting for queue wait, retry behavior, and scheduler limits.\\n' + '- If observed queue wait or retry behavior is unexpected, share this run directory with HPC support.\\n' +) + +if not valid: + raise SystemExit('Validation failed') +PY + """ +} + +workflow { + batches = Channel + .fromList((1..items).collate(batchSize)) + .map { batch -> tuple(batch.first(), batch, batch.any { failItems.contains(it) }) } + + CHARACTERIZE_ITEM(batches) + VALIDATE_RESULTS(CHARACTERIZE_ITEM.out.collect()) +} From a7c2a96bc90f406fdadc77cf3f38deabcb9c29e6 Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 4 Aug 2026 11:45:42 -0600 Subject: [PATCH 2/4] linting and consistent title case --- .github/workflows/run-checks.yml | 23 +++++++++++++++++++++++ .pre-commit-config.yaml | 8 ++++++++ Makefile | 14 +++++++++++++- PLAN.md | 10 +++++----- README.md | 10 ++++++---- bin/formascute | 6 +++--- docs/clone-ready.md | 1 + docs/nf1-pipeline-target.md | 2 +- docs/production-checklist.md | 1 + 9 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/run-checks.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml new file mode 100644 index 0000000..5be9931 --- /dev/null +++ b/.github/workflows/run-checks.yml @@ -0,0 +1,23 @@ +--- +name: checks + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +jobs: + prek: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Run prek + uses: j178/prek-action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3f1b60b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: + - repo: local + hooks: + - id: formascute-check + name: formascute repository check + entry: bash bin/formascute check + language: system + pass_filenames: false diff --git a/Makefile b/Makefile index 09190fc..778afad 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,13 @@ ifneq ($(strip $(BATCH_SIZE)),) ARGS += --batch-size $(BATCH_SIZE) endif -.PHONY: help check doctor preflight smoke run submit submit-dry-run list-experiments +.PHONY: help check prek doctor preflight smoke run submit submit-dry-run list-experiments help: @bash bin/formascute --help @printf '\nMake targets:\n' @printf ' make check\n' + @printf ' make prek\n' @printf ' make doctor\n' @printf ' make preflight ACCOUNT=amc-general\n' @printf ' make smoke\n' @@ -51,6 +52,17 @@ help: check: @bash bin/formascute check +prek: + @if command -v prek >/dev/null 2>&1; then \ + prek run --all-files; \ + elif command -v pre-commit >/dev/null 2>&1; then \ + pre-commit run --all-files; \ + else \ + printf 'Install prek or pre-commit, then rerun: make prek\n' >&2; \ + printf ' python -m pip install prek\n' >&2; \ + exit 127; \ + fi + doctor: @bash bin/formascute doctor diff --git a/PLAN.md b/PLAN.md index f4ce019..0c1210e 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,13 +1,13 @@ -# FormaScute +# formascute -**FormaScute** is a reproducible characterization framework for +**formascute** is a reproducible characterization framework for understanding how scientific workflows execute on HPC systems. > *Reveal the shape of workflow execution.* ## Philosophy -FormaScute is **not** a benchmarking suite. +formascute is **not** a benchmarking suite. It is a characterization framework that helps answer engineering questions about workflow execution by running small, reproducible @@ -164,13 +164,13 @@ Each run produces: ## Questions for HPC engineers -FormaScute should never guess. +formascute should never guess. When observations cannot be explained, generate concise engineering questions supported by reproducible evidence. ## Long-term vision -FormaScute becomes a reusable characterization framework that any +formascute becomes a reusable characterization framework that any institution can point at an HPC cluster to understand workflow behavior before running production-scale scientific analyses. diff --git a/README.md b/README.md index 30bb67b..67e354d 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# FormaScute +# formascute Reveal the shape of workflow execution. -FormaScute takes its name from shape and scute: the patterned plates on a turtle +formascute takes its name from shape and scute: the patterned plates on a turtle shell. The project is about seeing the shape of the shell before we need it, making each layer of workflow execution visible before production science depends on it. 🐢 -FormaScute runs small, reproducible characterization experiments for workflow +formascute runs small, reproducible characterization experiments for workflow engines on HPC systems. It starts from a known-working deployment, changes one execution setting at a time, validates correctness, and records enough provenance to make scheduler and filesystem behavior discussable with HPC engineers. -FormaScute is not a benchmarking suite. The goal is to understand execution +formascute is not a benchmarking suite. The goal is to understand execution behavior before running production science. ## Quick Start @@ -22,6 +22,7 @@ From the repository root: ```bash make check +make prek make doctor make smoke ``` @@ -45,6 +46,7 @@ Preflight output is written to `results//preflight/`. - `make check`: verify the cloned repository has the expected local files and a writable `results/` directory. +- `make prek`: run the same local check configuration used by pull request CI. - `make doctor`: show available runtime tools. - `make preflight ACCOUNT=amc-general`: record Slurm, module, filesystem, and runtime context without submitting work. diff --git a/bin/formascute b/bin/formascute index 8c04d10..0427772 100755 --- a/bin/formascute +++ b/bin/formascute @@ -5,7 +5,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" usage() { cat <<'USAGE' -FormaScute +formascute Usage: bin/formascute doctor @@ -105,7 +105,7 @@ check_repo() { "docs/nf1-pipeline-target.md" ) - echo "FormaScute repository check" + echo "formascute repository check" for path in "${required[@]}"; do if [[ -e "$ROOT_DIR/$path" ]]; then echo "ok: $path" @@ -272,7 +272,7 @@ PY } doctor() { - echo "FormaScute doctor" + echo "formascute doctor" echo "repo: $ROOT_DIR" if have nextflow; then diff --git a/docs/clone-ready.md b/docs/clone-ready.md index 11f4d58..7bb007a 100644 --- a/docs/clone-ready.md +++ b/docs/clone-ready.md @@ -12,6 +12,7 @@ cd formascute module load nextflow module load singularity make check +make prek make doctor make preflight ACCOUNT= make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= diff --git a/docs/nf1-pipeline-target.md b/docs/nf1-pipeline-target.md index 016bce3..42f63f1 100644 --- a/docs/nf1-pipeline-target.md +++ b/docs/nf1-pipeline-target.md @@ -1,6 +1,6 @@ # NF1 3D Organoid Pipeline Target -FormaScute is aimed at improving the Alpine execution model for +formascute is aimed at improving the Alpine execution model for `WayScience/NF1_3D_organoid_profiling_pipeline`. The current NF1 pipeline uses Slurm directly from shell scripts. The largest diff --git a/docs/production-checklist.md b/docs/production-checklist.md index 1aeb019..e9025b6 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -10,6 +10,7 @@ cd formascute module load nextflow module load singularity make check +make prek make doctor make preflight ACCOUNT= make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= From 7f537a36c3c01a6216b948e71a0f8dcfee529a3a Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 4 Aug 2026 13:33:43 -0600 Subject: [PATCH 3/4] more specific linting stuff; reality checks --- .github/workflows/run-checks.yml | 8 ++ .markdownlint-cli2.yaml | 7 ++ .pre-commit-config.yaml | 4 +- Makefile | 8 +- PLAN.md | 142 ++++++++++++++++--------------- README.md | 4 +- bin/formascute | 3 + docs/clone-ready.md | 1 - docs/production-checklist.md | 1 - examples/run_sarek.sh | 6 +- scripts/check | 11 +++ scripts/lint | 51 +++++++++++ 12 files changed, 168 insertions(+), 78 deletions(-) create mode 100644 .markdownlint-cli2.yaml create mode 100755 scripts/check create mode 100755 scripts/lint diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index 5be9931..44af3d7 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -19,5 +19,13 @@ jobs: with: persist-credentials: false + - name: Install ShellCheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + + - name: Setup Node.js + uses: actions/setup-node@v6.0.0 + with: + node-version: "24" + - name: Run prek uses: j178/prek-action@v3.0.0 diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..e3aa107 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,7 @@ +config: + MD013: false + MD033: false +globs: + - "*.md" + - "docs/**/*.md" + - ".github/**/*.md" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f1b60b..2314707 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: - repo: local hooks: - id: formascute-check - name: formascute repository check - entry: bash bin/formascute check + name: formascute lint + entry: bash scripts/lint language: system pass_filenames: false diff --git a/Makefile b/Makefile index 778afad..e846611 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,13 @@ ifneq ($(strip $(BATCH_SIZE)),) ARGS += --batch-size $(BATCH_SIZE) endif -.PHONY: help check prek doctor preflight smoke run submit submit-dry-run list-experiments +.PHONY: help check lint prek doctor preflight smoke run submit submit-dry-run list-experiments help: @bash bin/formascute --help @printf '\nMake targets:\n' @printf ' make check\n' + @printf ' make lint\n' @printf ' make prek\n' @printf ' make doctor\n' @printf ' make preflight ACCOUNT=amc-general\n' @@ -50,7 +51,10 @@ help: @printf ' FORMASCUTE_PROJECT Project/cache root; default /projects/$$USER\n' check: - @bash bin/formascute check + @bash scripts/check + +lint: + @bash scripts/lint prek: @if command -v prek >/dev/null 2>&1; then \ diff --git a/PLAN.md b/PLAN.md index 0c1210e..c1b173d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -23,22 +23,22 @@ everything else untouched. ## Goals -- Understand execution behavior rather than maximize benchmark scores. -- Reach scientific results sooner by understanding scheduler behavior. -- Produce evidence that HPC engineers can interpret. -- Generate reproducible observations and targeted questions. -- Keep every experiment small, deterministic, and open source. +- Understand execution behavior rather than maximize benchmark scores. +- Reach scientific results sooner by understanding scheduler behavior. +- Produce evidence that HPC engineers can interpret. +- Generate reproducible observations and targeted questions. +- Keep every experiment small, deterministic, and open source. ## Baseline Start from the existing Alpine deployment: -- Coordinator launched with `sbatch` -- Nextflow loads as a module -- Singularity/Apptainer environment -- Existing `NXF_WORK`, `NXF_TEMP`, and `NXF_HOME` -- Existing Slurm executor configuration -- Existing queue, QoS, account, retry, and submission settings +- Coordinator launched with `sbatch` +- Nextflow loads as a module +- Singularity/Apptainer environment +- Existing `NXF_WORK`, `NXF_TEMP`, and `NXF_HOME` +- Existing Slurm executor configuration +- Existing queue, QoS, account, retry, and submission settings Treat this configuration as the **control**. @@ -66,66 +66,68 @@ formascute/ Each experiment: -1. Starts from the Alpine reference configuration. -2. Applies one configuration mutation. -3. Executes the same synthetic workflow. -4. Validates output correctness. -5. Collects Nextflow + Slurm metrics. -6. Produces observations. -7. Generates questions for HPC engineers if behavior is unexplained. +1. Starts from the Alpine reference configuration. +2. Applies one configuration mutation. +3. Executes the same synthetic workflow. +4. Validates output correctness. +5. Collects Nextflow + Slurm metrics. +6. Produces observations. +7. Generates questions for HPC engineers if behavior is unexplained. ## Canonical synthetic workflow - Manifest - │ - Fan out N work items - │ - Aggregate - │ - Validate +```text +Manifest + │ +Fan out N work items + │ +Aggregate + │ +Validate +``` Synthetic work items support: -- CPU work -- Memory allocation -- I/O -- Sleep -- Controlled failures +- CPU work +- Memory allocation +- I/O +- Sleep +- Controlled failures No biological data are required. ## Initial execution strategies -- One Slurm job per Nextflow process -- Batched work items -- Slurm job arrays -- Persistent Nextflow coordinator +- One Slurm job per Nextflow process +- Batched work items +- Slurm job arrays +- Persistent Nextflow coordinator ## Initial characterization dimensions Only modify one at a time. -- queueSize -- submitRateLimit -- batching -- concurrency -- QoS -- partition -- retries -- work directory placement -- scratch usage +- queueSize +- submitRateLimit +- batching +- concurrency +- QoS +- partition +- retries +- work directory placement +- scratch usage ## Safety Every experiment must define: -- maximum jobs -- maximum concurrent jobs -- maximum CPUs -- maximum memory -- maximum walltime -- maximum output size -- maximum retries +- maximum jobs +- maximum concurrent jobs +- maximum CPUs +- maximum memory +- maximum walltime +- maximum output size +- maximum retries Experiments terminate immediately if limits are exceeded. @@ -133,15 +135,15 @@ Experiments terminate immediately if limits are exceeded. Capture: -- Nextflow trace -- Timeline -- Report -- DAG -- Slurm accounting -- Queue wait -- Runtime -- Retry behavior -- Output correctness +- Nextflow trace +- Timeline +- Report +- DAG +- Slurm accounting +- Queue wait +- Runtime +- Retry behavior +- Output correctness Primary metric: @@ -151,16 +153,18 @@ Primary metric: Each run produces: - results/ - └── run-id/ - ├── experiment.yaml - ├── trace.tsv - ├── report.html - ├── timeline.html - ├── slurm.tsv - ├── validation.json - ├── summary.json - └── questions.md +```text +results/ +└── run-id/ + ├── experiment.yaml + ├── trace.tsv + ├── report.html + ├── timeline.html + ├── slurm.tsv + ├── validation.json + ├── summary.json + └── questions.md +``` ## Questions for HPC engineers diff --git a/README.md b/README.md index 67e354d..1f155a7 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ From the repository root: ```bash make check -make prek make doctor make smoke ``` @@ -46,7 +45,8 @@ Preflight output is written to `results//preflight/`. - `make check`: verify the cloned repository has the expected local files and a writable `results/` directory. -- `make prek`: run the same local check configuration used by pull request CI. +- `make lint`: run shell, Markdown, and best-effort Nextflow lint checks. +- `make prek`: run the same lint configuration used by pull request CI. - `make doctor`: show available runtime tools. - `make preflight ACCOUNT=amc-general`: record Slurm, module, filesystem, and runtime context without submitting work. diff --git a/bin/formascute b/bin/formascute index 0427772..5f37c41 100755 --- a/bin/formascute +++ b/bin/formascute @@ -96,9 +96,12 @@ check_repo() { local failed=0 local required=( "Makefile" + ".markdownlint-cli2.yaml" "nextflow.config" "conf/alpine.reference.config" "conf/characterization.config" + "scripts/check" + "scripts/lint" "workflows/synthetic.nf" "docs/hpc-quickstart.md" "docs/production-checklist.md" diff --git a/docs/clone-ready.md b/docs/clone-ready.md index 7bb007a..11f4d58 100644 --- a/docs/clone-ready.md +++ b/docs/clone-ready.md @@ -12,7 +12,6 @@ cd formascute module load nextflow module load singularity make check -make prek make doctor make preflight ACCOUNT= make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= diff --git a/docs/production-checklist.md b/docs/production-checklist.md index e9025b6..1aeb019 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -10,7 +10,6 @@ cd formascute module load nextflow module load singularity make check -make prek make doctor make preflight ACCOUNT= make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= diff --git a/examples/run_sarek.sh b/examples/run_sarek.sh index c446003..dc4afaa 100644 --- a/examples/run_sarek.sh +++ b/examples/run_sarek.sh @@ -11,6 +11,11 @@ set -euo pipefail +: "${scrproj:?Set scrproj to the scratch project directory}" +: "${pipelinepath:?Set pipelinepath to the Nextflow pipeline path}" +: "${samplefile:?Set samplefile to the Sarek sample sheet}" +: "${pipeoutdir:?Set pipeoutdir to the pipeline output directory}" + # make software accessible: module load nextflow/23.04 module load singularity/3.7.4 @@ -36,4 +41,3 @@ nextflow run "$pipelinepath" -ansi-log false \ --tools mutect2,strelka,tiddit,freebayes,vep,snpeff -resume \ -c curc_alpine.config,fastp.config - diff --git a/scripts/check b/scripts/check new file mode 100755 index 0000000..ff5465f --- /dev/null +++ b/scripts/check @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +have() { + command -v "$1" >/dev/null 2>&1 +} + +echo "repository" +bash "$ROOT_DIR/bin/formascute" check diff --git a/scripts/lint b/scripts/lint new file mode 100755 index 0000000..9dce92f --- /dev/null +++ b/scripts/lint @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +have() { + command -v "$1" >/dev/null 2>&1 +} + +echo "shell syntax" +while IFS= read -r path; do + bash -n "$path" + echo "ok: ${path#"$ROOT_DIR"/}" +done < <(find "$ROOT_DIR" -path "$ROOT_DIR/.git" -prune -o -type f -name '*.sh' -print; printf '%s\n' "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint") + +echo +echo "shellcheck" +if have shellcheck; then + shellcheck "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint" + while IFS= read -r path; do + shellcheck "$path" + done < <(find "$ROOT_DIR" -path "$ROOT_DIR/.git" -prune -o -type f -name '*.sh' -print) +else + echo "skip: shellcheck is not installed" +fi + +echo +echo "markdownlint" +if have markdownlint-cli2; then + markdownlint-cli2 +elif have markdownlint; then + markdownlint README.md PLAN.md docs/**/*.md +elif have npx; then + npx --yes markdownlint-cli2 +else + echo "skip: markdownlint is not installed" +fi + +echo +echo "nextflow" +if have nextflow && nextflow -version >/dev/null 2>&1; then + nextflow config "$ROOT_DIR/workflows/synthetic.nf" \ + -profile local_smoke \ + -c "$ROOT_DIR/nextflow.config" \ + -c "$ROOT_DIR/conf/characterization.config" \ + -c "$ROOT_DIR/conf/experiments/independent_jobs.config" \ + >/dev/null + echo "ok: nextflow config local_smoke" +else + echo "skip: nextflow is not installed or not runnable" +fi From 746ea1824c50f755496c97193012da283f163cf9 Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 4 Aug 2026 15:14:08 -0600 Subject: [PATCH 4/4] findings; skill; etc. --- .agents/skills/alpine.md | 162 +++++++++++++++ .gitignore | 1 + .markdownlint-cli2.yaml | 1 + AGENTS.md | 5 + CITATION.cff | 16 ++ CODE_OF_CONDUCT.md | 31 +++ CONTRIBUTING.md | 60 ++++++ Makefile | 32 +-- README.md | 66 +++++-- bin/formascute | 81 ++++++-- conf/alpine.reference.config | 7 +- .../nf1_featurization_batched_2.config | 7 + conf/experiments/nf1_minimal_overhead.config | 7 + conf/experiments/nf1_queue_size_4.config | 11 ++ conf/experiments/nf1_submit_rate_4.config | 11 ++ conf/experiments/zp_synthetic_features.config | 13 ++ docs/alpine-findings.md | 154 +++++++++++++++ docs/clone-ready.md | 14 +- docs/hpc-quickstart.md | 47 +++-- docs/nf1-pipeline-target.md | 62 +++++- docs/north-star.md | 184 ++++++++++++++++++ docs/production-checklist.md | 26 ++- examples/curc_alpine.config | 2 +- examples/run_sarek.sh | 3 +- nextflow.config | 7 + scripts/bootstrap | 26 +++ scripts/lint | 4 +- workflows/synthetic.nf | 124 ++++++++++-- 28 files changed, 1063 insertions(+), 101 deletions(-) create mode 100644 .agents/skills/alpine.md create mode 100644 AGENTS.md create mode 100644 CITATION.cff create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 conf/experiments/nf1_featurization_batched_2.config create mode 100644 conf/experiments/nf1_minimal_overhead.config create mode 100644 conf/experiments/nf1_queue_size_4.config create mode 100644 conf/experiments/nf1_submit_rate_4.config create mode 100644 conf/experiments/zp_synthetic_features.config create mode 100644 docs/alpine-findings.md create mode 100644 docs/north-star.md create mode 100644 scripts/bootstrap diff --git a/.agents/skills/alpine.md b/.agents/skills/alpine.md new file mode 100644 index 0000000..ab7ace9 --- /dev/null +++ b/.agents/skills/alpine.md @@ -0,0 +1,162 @@ +# Alpine Skill + +Use this note before testing formascute on CURC Alpine. + +## Connection + +From the local machine, use the `ssh-alpine` zsh alias: + +```bash +zsh -lic 'ssh-alpine "hostname; pwd"' +``` + +The alias should be defined in the user's shell configuration as an SSH command +to `login.rc.colorado.edu` using that user's Alpine SSH key. + +`Persistence1` is not directly resolvable from the local machine. It is reachable +from the Alpine login node: + +```bash +zsh -lic 'ssh-alpine "ssh Persistence1 hostname"' +``` + +## Module Behavior + +The regular Alpine login node did not expose a usable Nextflow module during +validation. `Persistence1` did expose Nextflow modules: + +- `nextflow/22.10.6` +- `nextflow/23.04` +- `nextflow/24.04.4` +- `nextflow/25.10.2` + +The validated module is: + +```bash +module load nextflow/25.10.2 +nextflow -version +``` + +On `Persistence1`, `/usr/bin/apptainer` and `/usr/bin/singularity` were both +available and reported Apptainer 1.4.5. + +## Slurm Defaults + +Use these defaults for small synthetic CPU tests: + +- account: project allocation supplied by the user +- partition: `acpu` +- QoS: `cpu-normal` +- submit host: `Persistence1` + +Slurm accepted `acpu` and `cpu-normal` and mapped them to the current CPU +partition backing the older `amilan`/`normal` names. + +`Persistence1` is the selected submission location for this project. Treat it as +known, not as an open question. + +## Validated Submission Path + +From a clone on Alpine shared scratch: + +```bash +make check +make preflight ACCOUNT= +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_independent ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 +``` + +The successful validation run used: + +```bash +make submit EXPERIMENT=nf1_featurization_independent ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 RUN_ID=remote-smoke-persistence +``` + +Observed result: + +- coordinator Slurm job completed +- `completion_status.txt` reported `nextflow_exit_status: 0` +- `validation.json` reported `"valid": true` +- `trace.tsv` contained native Slurm job IDs +- `slurm.tsv` was collected + +## Synthetic Experiment Findings + +Small synthetic runs completed successfully through `Persistence1`. + +Useful observed shape: + +- one-item minimal overhead: about 21 seconds +- independent 16-item fan-out: about 51-52 seconds, 17 native Slurm jobs +- batch size 2 with 16 items: about 42 seconds, 9 native Slurm jobs +- batch size 4 with 16 items: about 41 seconds, 5 native Slurm jobs +- `queueSize=4` with 16 independent items: about 1 minute +- `submitRateLimit='4 / 1 min'`: about 4 minutes and should not be the first + tuning knob +- queue wait was negligible in tiny tests, usually 0-1 seconds + +Interpretation: + +- Nextflow can submit, track, validate, and collect Slurm accounting on Alpine + when submitted through `Persistence1`. +- Batching reduced Slurm job count and modestly improved elapsed time in tiny + synthetic tests. +- Batch sizes 2-4 are the first batching range to consider. +- Use `queueSize` before `submitRateLimit` for scheduler pressure control. +- Keep CPU and GPU work separated into different profiles or runs. + +For NF1 orchestration, prefer a thin Nextflow layer around the existing Stage 3 +work list and feature scripts before rewriting processing code. + +## ZedProfiler Notes + +The `zp_synthetic_features` experiment completed successfully on Alpine through +`Persistence1`: + +```bash +make submit EXPERIMENT=zp_synthetic_features ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 RUN_ID=zp-sim-01 +``` + +Observed result: + +- coordinator job completed with Nextflow exit status `0` +- validation passed for 4 expected items +- 4 feature tasks plus 1 validation task were submitted +- 16 simulated feature rows were emitted +- feature tasks took about 1-3 seconds each +- peak RSS was about 9-32 MB + +This confirms the orchestration shape for tiny CPU-first 3D feature work, but it +does not validate real ZedProfiler imports or image I/O. + +The tested base Python environment on `Persistence1` is not suitable for real +ZedProfiler: + +- Python 3.9.13 +- NumPy 1.21.5 +- Pandas 1.4.4 +- scikit-image 0.19.2 +- missing `mahotas` +- missing `pyarrow` +- missing `zedprofiler` + +Real ZedProfiler work needs a project-owned Python 3.11+ environment or an +Apptainer image. For NF1, prefer grouping compatible ZedProfiler feature families +by image-set when possible so each task loads image data once and writes validated +outputs. + +## Important Failure Mode + +Submitting directly from the regular Alpine login node failed because the batch +job could not load `nextflow/25.10.2`. Use `SUBMIT_HOST=Persistence1` unless the +module environment changes. + +## CURC Questions To Preserve + +Ask CURC before scaling: + +- What `queueSize` range is recommended for Nextflow on Alpine? +- Should small NF1-style jobs use moderate independent submission or batching? +- Are Slurm job arrays recommended, or is Nextflow-managed submission preferred + for retries and accounting? +- Should SAMMed3D GPU work be a separate Nextflow profile and submission path? diff --git a/.gitignore b/.gitignore index f4698b9..3dee7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -184,6 +184,7 @@ cython_debug/ # Nextflow and FormaScute run outputs .nextflow/ .nextflow.log* +.formascute/ work/ results/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index e3aa107..9b02558 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -3,5 +3,6 @@ config: MD033: false globs: - "*.md" + - ".agents/**/*.md" - "docs/**/*.md" - ".github/**/*.md" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c5b4921 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# Agent Notes + +Use `.agents/skills/alpine.md` before working on Alpine validation or Slurm +submission behavior. It records the connection and module-loading details already +validated for this project. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..3919c24 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,16 @@ +cff-version: 1.2.0 +message: "If you use formascute, please cite this repository." +title: "formascute" +abstract: >- + Small, reproducible characterization experiments for understanding workflow + execution on HPC systems before running production science. +authors: + - name: "formascute contributors" +license: "BSD-3-Clause" +type: software +keywords: + - high-performance computing + - workflow orchestration + - nextflow + - slurm + - reproducible research diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..b54fd82 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,31 @@ +# Code Of Conduct + +This project follows a practical standard for respectful, research-focused +collaboration. + +## Expected Behavior + +- Be direct, professional, and constructive. +- Focus technical discussion on evidence, reproducibility, and clear tradeoffs. +- Assume collaborators are working in good faith. +- Make uncertainty explicit when interpreting HPC behavior. +- Respect privacy and operational constraints around shared computing systems. + +## Unacceptable Behavior + +- Harassment, intimidation, discriminatory language, or personal attacks. +- Sharing private credentials, SSH keys, tokens, or user-specific access details. +- Publishing sensitive HPC account, path, allocation, or dataset information. +- Misrepresenting benchmark-like results as production performance guarantees. +- Pressuring contributors to run unsafe or inappropriate workloads on shared + systems. + +## Reporting + +Report concerns through the maintainers or the repository issue tracker. If the +concern involves private access details, credentials, or sensitive system +information, do not post those details publicly. + +Maintainers should respond by clarifying the concern, preserving privacy, and +taking proportionate action to keep the project useful and safe for +collaboration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2390dbf --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# Contributing + +formascute is intentionally small. Contributions should preserve that shape: +simple entry points, reproducible experiments, explicit validation, and clear +provenance. + +## Development Checks + +Run these before opening a pull request: + +```bash +make check +make lint +``` + +If `prek` or `pre-commit` is installed, also run: + +```bash +make prek +``` + +`make check` verifies that the repository is ready to clone and use on Alpine. +`make lint` covers shell syntax, ShellCheck when available, Markdown, and +best-effort Nextflow config checks. + +## Contribution Guidelines + +- Keep experiments small and bounded. +- Do not commit real HPC usernames, private paths, allocation names, SSH key + paths, production data paths, or Slurm job identifiers. +- Use placeholders such as ``, ``, and `` in + documentation. +- Do not use production NF1 data in this repository. +- Add or update documentation when a command, assumption, or Alpine observation + changes. +- Prefer Make targets and `bin/formascute` entry points over one-off scripts. +- Keep CPU and GPU assumptions separate. + +## Experiment Criteria + +Every experiment should: + +- change one thing relative to the baseline +- run quickly at small scale +- validate correctness +- write outputs under `results//` +- preserve enough provenance to interpret the run later +- document remaining uncertainty + +## Pull Requests + +Pull requests should describe: + +- what changed +- how it was validated +- whether Alpine behavior or assumptions changed +- any follow-up questions for HPC support + +Do not include generated `results/` outputs unless there is a specific reason to +review a tiny artifact. diff --git a/Makefile b/Makefile index e846611..24fe20b 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,9 @@ BATCH_SIZE ?= RUN_ID ?= PROFILE ?= alpine ACCOUNT ?= -PARTITION ?= amilan -QOS ?= normal +PARTITION ?= acpu +QOS ?= cpu-normal +SUBMIT_HOST ?= ARGS := ifneq ($(strip $(RUN_ID)),) @@ -20,20 +21,21 @@ ifneq ($(strip $(BATCH_SIZE)),) ARGS += --batch-size $(BATCH_SIZE) endif -.PHONY: help check lint prek doctor preflight smoke run submit submit-dry-run list-experiments +.PHONY: help bootstrap check lint prek doctor preflight smoke run submit submit-dry-run list-experiments help: @bash bin/formascute --help @printf '\nMake targets:\n' + @printf ' make bootstrap\n' @printf ' make check\n' @printf ' make lint\n' @printf ' make prek\n' @printf ' make doctor\n' - @printf ' make preflight ACCOUNT=amc-general\n' + @printf ' make preflight ACCOUNT=\n' @printf ' make smoke\n' @printf ' make run EXPERIMENT=independent_jobs ITEMS=16\n' - @printf ' make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general\n' - @printf ' make submit-dry-run EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general\n' + @printf ' make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=\n' + @printf ' make submit-dry-run EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=\n' @printf ' make list-experiments\n' @printf '\nMake variables:\n' @printf ' EXPERIMENT Experiment name from conf/experiments. Default: independent_jobs\n' @@ -42,14 +44,20 @@ help: @printf ' RUN_ID Output directory name under results/. Default: UTC timestamp\n' @printf ' PROFILE Nextflow profile for make run. Default: alpine\n' @printf ' ACCOUNT Slurm account/allocation for make submit. Default: none\n' - @printf ' PARTITION Slurm partition for make submit. Default: amilan\n' - @printf ' QOS Slurm QoS for make submit. Default: normal\n' + @printf ' PARTITION Slurm partition for make submit. Default: acpu\n' + @printf ' QOS Slurm QoS for make submit. Default: cpu-normal\n' + @printf ' SUBMIT_HOST Optional SSH host used to run sbatch. Alpine: Persistence1\n' @printf '\nEnvironment overrides:\n' - @printf ' FORMASCUTE_NEXTFLOW_MODULE Module loaded in generated Slurm scripts\n' - @printf ' FORMASCUTE_CONTAINER_MODULE Container module loaded in generated Slurm scripts\n' + @printf ' FORMASCUTE_NEXTFLOW_MODULE Module loaded in generated Slurm scripts; default nextflow/25.10.2\n' + @printf ' FORMASCUTE_CONTAINER_MODULE Optional container module loaded in generated Slurm scripts\n' + @printf ' FORMASCUTE_ENABLE_CONTAINER Enable Nextflow Singularity support; default false\n' + @printf ' FORMASCUTE_ENV_DIR Bootstrap env directory; default .formascute/conda\n' @printf ' FORMASCUTE_SCRATCH Shared scratch root; default /scratch/alpine/$$USER\n' @printf ' FORMASCUTE_PROJECT Project/cache root; default /projects/$$USER\n' +bootstrap: + @bash scripts/bootstrap + check: @bash scripts/check @@ -81,10 +89,10 @@ run: bash bin/formascute run $(EXPERIMENT) --profile $(PROFILE) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" submit: - @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" + @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" $(if $(SUBMIT_HOST),--submit-host "$(SUBMIT_HOST)") submit-dry-run: - @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" --dry-run + @bash bin/formascute submit $(EXPERIMENT) $(ARGS) --account "$(ACCOUNT)" --partition "$(PARTITION)" --qos "$(QOS)" $(if $(SUBMIT_HOST),--submit-host "$(SUBMIT_HOST)") --dry-run list-experiments: @find conf/experiments -maxdepth 1 -type f -name '*.config' | sed 's#conf/experiments/##; s#\.config##' | sort diff --git a/README.md b/README.md index 1f155a7..d63b2f5 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,11 @@ make smoke On an HPC system with Nextflow and Slurm: ```bash -module load nextflow -module load singularity make check make doctor -make preflight ACCOUNT=amc-general -make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general -make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general +make preflight ACCOUNT= +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 ``` Run outputs are written to `results//`. @@ -43,12 +41,14 @@ Preflight output is written to `results//preflight/`. ## Entry Points +- `make bootstrap`: optional fallback to create a repo-local conda environment + with Nextflow and Java 17 when site modules do not provide them. - `make check`: verify the cloned repository has the expected local files and a writable `results/` directory. - `make lint`: run shell, Markdown, and best-effort Nextflow lint checks. - `make prek`: run the same lint configuration used by pull request CI. - `make doctor`: show available runtime tools. -- `make preflight ACCOUNT=amc-general`: record Slurm, module, filesystem, and +- `make preflight ACCOUNT=`: record Slurm, module, filesystem, and runtime context without submitting work. - `make smoke`: run a tiny local Nextflow check. - `make run EXPERIMENT=independent_jobs ITEMS=16`: run directly. @@ -67,14 +67,42 @@ No installation step is required. Clone the repository on the HPC system and run the Make targets from the repository root. If `make` is unavailable, use the matching `bash bin/formascute` commands directly. +## Alpine SSH Access + +Agent-driven Alpine work assumes the local machine has an `ssh-alpine` shell +alias or equivalent SSH config entry. It should connect to the Alpine login node +using the user's CURC/ACCESS identity and SSH key, for example: + +```bash +alias ssh-alpine='ssh -i ~/.ssh/ @xsede.org@login.rc.colorado.edu' +``` + +The exact key path and username are user-specific. Configure SSH access following +CURC's AMC SSH key authentication guidance: +[SSH Key-Based Authentication for Anschutz Medical Campus](https://curc.readthedocs.io/en/latest/additional-resources/amc_ssh_auth.html). + +The alias lets local automation run commands like: + +```bash +zsh -lic 'ssh-alpine "hostname; pwd"' +``` + +Within Alpine, formascute submits workflow coordinator jobs through +`Persistence1` with `SUBMIT_HOST=Persistence1`. + Available initial experiments: - `independent_jobs` - `batched_jobs` - `queue_size_20` +- `nf1_minimal_overhead` - `nf1_featurization_independent` +- `nf1_featurization_batched_2` - `nf1_featurization_batched` +- `nf1_queue_size_4` - `nf1_queue_size_20` +- `nf1_submit_rate_4` +- `zp_synthetic_features` ## Arguments @@ -87,20 +115,32 @@ Available initial experiments: - `PROFILE`: Nextflow profile for `make run`. Defaults to `alpine`. - `ACCOUNT`: Slurm account/allocation for `make submit`. Defaults to no explicit account. -- `PARTITION`: Slurm partition for `make submit`. Defaults to `amilan`. -- `QOS`: Slurm QoS for `make submit`. Defaults to `normal`. +- `PARTITION`: Slurm partition for `make submit`. Defaults to `acpu`. +- `QOS`: Slurm QoS for `make submit`. Defaults to `cpu-normal`. +- `SUBMIT_HOST`: optional SSH host used to run `sbatch`. On Alpine, use + `Persistence1` so the Nextflow module tree is available at submission time. Useful environment overrides: - `FORMASCUTE_NEXTFLOW_MODULE`: module loaded by generated Slurm scripts. - Defaults to `nextflow`. -- `FORMASCUTE_CONTAINER_MODULE`: container module loaded by generated Slurm - scripts. Defaults to `singularity`. + Defaults to `nextflow/25.10.2`. +- `FORMASCUTE_CONTAINER_MODULE`: optional container module loaded by generated + Slurm scripts. +- `FORMASCUTE_ENABLE_CONTAINER`: enable Nextflow Singularity support. Defaults + to `false` for the synthetic experiments. +- `FORMASCUTE_ENV_DIR`: bootstrap environment directory. Defaults to + `.formascute/conda`. +- `FORMASCUTE_SUBMIT_HOST`: same behavior as `SUBMIT_HOST`. - `FORMASCUTE_SCRATCH`: shared scratch root. Defaults to `/scratch/alpine/$USER`. - `FORMASCUTE_PROJECT`: project/cache root. Defaults to `/projects/$USER`. See `docs/clone-ready.md` for the first clone-and-run sequence, `docs/hpc-quickstart.md` for HPC usage notes, `docs/production-checklist.md` for first-run validation, -`docs/nf1-pipeline-target.md` for the target pipeline, and `PLAN.md` for the -project direction. +`docs/alpine-findings.md` for validated Alpine observations, +`docs/nf1-pipeline-target.md` for the target pipeline, +`docs/north-star.md` for the current NF1 gold-standard direction, and `PLAN.md` +for the project direction. + +See `CONTRIBUTING.md` for development expectations, `CODE_OF_CONDUCT.md` for +collaboration standards, and `CITATION.cff` for citation metadata. diff --git a/bin/formascute b/bin/formascute index 5f37c41..739d69c 100755 --- a/bin/formascute +++ b/bin/formascute @@ -13,24 +13,29 @@ Usage: bin/formascute preflight [--run-id ID] [--account NAME] [--partition NAME] [--qos NAME] bin/formascute run-local-smoke [--run-id ID] bin/formascute run EXPERIMENT [--profile PROFILE] [--run-id ID] [--items N] [--batch-size N] [--account NAME] [--partition NAME] [--qos NAME] - bin/formascute submit EXPERIMENT [--run-id ID] [--items N] [--batch-size N] [--account NAME] [--partition NAME] [--qos NAME] [--dry-run] + bin/formascute submit EXPERIMENT [--run-id ID] [--items N] [--batch-size N] [--account NAME] [--partition NAME] [--qos NAME] [--submit-host HOST] [--dry-run] Examples: bin/formascute doctor bin/formascute check - bin/formascute preflight --account amc-general + bin/formascute preflight --account bin/formascute run-local-smoke bin/formascute run independent_jobs --profile alpine --items 16 bin/formascute run batched_jobs --profile alpine --items 16 --batch-size 4 - bin/formascute submit independent_jobs --items 16 --account amc-general + bin/formascute submit independent_jobs --items 16 --account Experiments: independent_jobs batched_jobs queue_size_20 + nf1_minimal_overhead nf1_featurization_independent + nf1_featurization_batched_2 nf1_featurization_batched + nf1_queue_size_4 nf1_queue_size_20 + nf1_submit_rate_4 + zp_synthetic_features Options: EXPERIMENT Experiment config name from conf/experiments without .config. @@ -39,12 +44,15 @@ Options: --items Number of synthetic work items. Default: 16. --batch-size Work items per Nextflow task. Default: 1, 8 for batched_jobs, or 4 for nf1_featurization_batched. --account Slurm account/allocation. Default: FORMASCUTE_ACCOUNT or none. - --partition Slurm partition. Default: FORMASCUTE_PARTITION or amilan. - --qos Slurm QoS. Default: FORMASCUTE_QOS or normal. + --partition Slurm partition. Default: FORMASCUTE_PARTITION or acpu. + --qos Slurm QoS. Default: FORMASCUTE_QOS or cpu-normal. + --submit-host Optional SSH host used to run sbatch. Example: Persistence1. Environment: - FORMASCUTE_NEXTFLOW_MODULE Module loaded by generated Slurm scripts. Default: nextflow. - FORMASCUTE_CONTAINER_MODULE Module loaded by generated Slurm scripts. Default: singularity. + FORMASCUTE_NEXTFLOW_MODULE Module loaded by generated Slurm scripts. Default: nextflow/25.10.2. + FORMASCUTE_CONTAINER_MODULE Optional container module loaded by generated Slurm scripts. + FORMASCUTE_ENABLE_CONTAINER Enable Nextflow Singularity support. Default: false. + FORMASCUTE_ENV_DIR Repo-local bootstrap environment. Default: .formascute/conda. FORMASCUTE_SCRATCH Shared scratch root. Default: /scratch/alpine/$USER. FORMASCUTE_PROJECT Project root for caches. Default: /projects/$USER. USAGE @@ -96,16 +104,24 @@ check_repo() { local failed=0 local required=( "Makefile" + "CITATION.cff" + "CODE_OF_CONDUCT.md" + "CONTRIBUTING.md" + "AGENTS.md" + ".agents/skills/alpine.md" ".markdownlint-cli2.yaml" "nextflow.config" "conf/alpine.reference.config" "conf/characterization.config" + "scripts/bootstrap" "scripts/check" "scripts/lint" "workflows/synthetic.nf" "docs/hpc-quickstart.md" "docs/production-checklist.md" + "docs/alpine-findings.md" "docs/nf1-pipeline-target.md" + "docs/north-star.md" ) echo "formascute repository check" @@ -286,6 +302,13 @@ doctor() { echo "nextflow: missing" fi + local env_dir="${FORMASCUTE_ENV_DIR:-$ROOT_DIR/.formascute/conda}" + if [[ -x "$env_dir/bin/nextflow" ]]; then + echo "bootstrap_nextflow: $env_dir/bin/nextflow" + else + echo "bootstrap_nextflow: missing" + fi + if have sbatch; then echo "sbatch: $(command -v sbatch)" else @@ -380,6 +403,7 @@ submit_slurm() { local partition="$6" local qos="$7" local dry_run="$8" + local submit_host="$9" local outdir="$ROOT_DIR/results/$id" local script="$outdir/coordinator.sbatch" local experiment_config="$ROOT_DIR/conf/experiments/${experiment}.config" @@ -391,6 +415,7 @@ submit_slurm() { validate_name "account" "$account" validate_name "partition" "$partition" validate_name "qos" "$qos" + validate_name "submit-host" "$submit_host" validate_positive_int "items" "$items" 256 validate_positive_int "batch-size" "$batch_size" 64 @@ -399,11 +424,16 @@ submit_slurm() { exit 2 fi - if [[ "$dry_run" != "true" ]] && ! have sbatch; then + if [[ "$dry_run" != "true" && -z "$submit_host" ]] && ! have sbatch; then echo "sbatch is required for submit" >&2 exit 127 fi + if [[ "$dry_run" != "true" && -n "$submit_host" ]] && ! have ssh; then + echo "ssh is required when --submit-host is used" >&2 + exit 127 + fi + if [[ -n "$account" ]]; then account_directive="#SBATCH --account=$account" fi @@ -432,10 +462,18 @@ set -euo pipefail cd "$ROOT_DIR" -if type module >/dev/null 2>&1; then +FORMASCUTE_ENV_DIR="\${FORMASCUTE_ENV_DIR:-$ROOT_DIR/.formascute/conda}" +if [[ -f "\$FORMASCUTE_ENV_DIR/bin/activate" ]]; then + # shellcheck disable=SC1091 + source "\$FORMASCUTE_ENV_DIR/bin/activate" +elif type module >/dev/null 2>&1; then module purge - module load "\${FORMASCUTE_NEXTFLOW_MODULE:-nextflow}" - module load "\${FORMASCUTE_CONTAINER_MODULE:-singularity}" + if [[ -n "\${FORMASCUTE_NEXTFLOW_MODULE:-nextflow/25.10.2}" && "\${FORMASCUTE_NEXTFLOW_MODULE:-nextflow/25.10.2}" != "none" ]]; then + module load "\${FORMASCUTE_NEXTFLOW_MODULE:-nextflow/25.10.2}" + fi + if [[ -n "\${FORMASCUTE_CONTAINER_MODULE:-}" && "\${FORMASCUTE_CONTAINER_MODULE:-}" != "none" ]]; then + module load "\${FORMASCUTE_CONTAINER_MODULE}" + fi else echo "Environment modules are not available; using commands already on PATH." fi @@ -446,6 +484,9 @@ EOF if [[ "$dry_run" == "true" ]]; then echo "Dry run: not submitting to Slurm" | tee "$outdir/slurm-submit.txt" + elif [[ -n "$submit_host" ]]; then + # shellcheck disable=SC2029 + ssh "$submit_host" "cd '$ROOT_DIR' && sbatch '$script'" | tee "$outdir/slurm-submit.txt" else sbatch "$script" | tee "$outdir/slurm-submit.txt" fi @@ -465,8 +506,8 @@ case "$cmd" in preflight) id="$(run_id)" account="${FORMASCUTE_ACCOUNT:-}" - partition="${FORMASCUTE_PARTITION:-amilan}" - qos="${FORMASCUTE_QOS:-normal}" + partition="${FORMASCUTE_PARTITION:-acpu}" + qos="${FORMASCUTE_QOS:-cpu-normal}" while [[ $# -gt 0 ]]; do case "$1" in --run-id) id="$2"; shift 2 ;; @@ -502,10 +543,11 @@ case "$cmd" in id="$(run_id)" items=16 account="${FORMASCUTE_ACCOUNT:-}" - partition="${FORMASCUTE_PARTITION:-amilan}" - qos="${FORMASCUTE_QOS:-normal}" + partition="${FORMASCUTE_PARTITION:-acpu}" + qos="${FORMASCUTE_QOS:-cpu-normal}" case "$experiment" in batched_jobs) batch_size=8 ;; + nf1_featurization_batched_2) batch_size=2 ;; nf1_featurization_batched) batch_size=4 ;; *) batch_size=1 ;; esac @@ -537,11 +579,13 @@ case "$cmd" in id="$(run_id)" items=16 account="${FORMASCUTE_ACCOUNT:-}" - partition="${FORMASCUTE_PARTITION:-amilan}" - qos="${FORMASCUTE_QOS:-normal}" + partition="${FORMASCUTE_PARTITION:-acpu}" + qos="${FORMASCUTE_QOS:-cpu-normal}" dry_run="false" + submit_host="${FORMASCUTE_SUBMIT_HOST:-}" case "$experiment" in batched_jobs) batch_size=8 ;; + nf1_featurization_batched_2) batch_size=2 ;; nf1_featurization_batched) batch_size=4 ;; *) batch_size=1 ;; esac @@ -554,13 +598,14 @@ case "$cmd" in --account) account="$2"; shift 2 ;; --partition) partition="$2"; shift 2 ;; --qos) qos="$2"; shift 2 ;; + --submit-host) submit_host="$2"; shift 2 ;; --dry-run) dry_run="true"; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; exit 2 ;; esac done - submit_slurm "$experiment" "$id" "$items" "$batch_size" "$account" "$partition" "$qos" "$dry_run" + submit_slurm "$experiment" "$id" "$items" "$batch_size" "$account" "$partition" "$qos" "$dry_run" "$submit_host" ;; -h|--help|"") usage diff --git a/conf/alpine.reference.config b/conf/alpine.reference.config index c950905..83ae3ff 100644 --- a/conf/alpine.reference.config +++ b/conf/alpine.reference.config @@ -7,13 +7,14 @@ def formascute_user = System.getenv('USER') ?: 'unknown' def formascute_scratch = System.getenv('FORMASCUTE_SCRATCH') ?: "/scratch/alpine/${formascute_user}" def formascute_project = System.getenv('FORMASCUTE_PROJECT') ?: "/projects/${formascute_user}" +def formascute_container_enabled = (System.getenv('FORMASCUTE_ENABLE_CONTAINER') ?: 'false').toBoolean() params { config_profile_description = 'CURC Alpine cluster profile' config_profile_contact = 'Thomas Danhorn' slurm_account = System.getenv('FORMASCUTE_ACCOUNT') ?: '' - slurm_partition = System.getenv('FORMASCUTE_PARTITION') ?: 'amilan' - slurm_qos = System.getenv('FORMASCUTE_QOS') ?: 'normal' + slurm_partition = System.getenv('FORMASCUTE_PARTITION') ?: 'acpu' + slurm_qos = System.getenv('FORMASCUTE_QOS') ?: 'cpu-normal' scratch_root = formascute_scratch project_root = formascute_project max_memory = 240.GB @@ -22,7 +23,7 @@ params { } singularity { - enabled = true + enabled = formascute_container_enabled cacheDir = "${params.project_root}/container_cache" autoMounts = true runOptions = "-B ${params.scratch_root}:/tmp" diff --git a/conf/experiments/nf1_featurization_batched_2.config b/conf/experiments/nf1_featurization_batched_2.config new file mode 100644 index 0000000..71c7b4a --- /dev/null +++ b/conf/experiments/nf1_featurization_batched_2.config @@ -0,0 +1,7 @@ +params { + experiment = 'nf1_featurization_batched_2' + items = 16 + batch_size = 2 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/nf1_minimal_overhead.config b/conf/experiments/nf1_minimal_overhead.config new file mode 100644 index 0000000..fe9a435 --- /dev/null +++ b/conf/experiments/nf1_minimal_overhead.config @@ -0,0 +1,7 @@ +params { + experiment = 'nf1_minimal_overhead' + items = 1 + batch_size = 1 + sleep_seconds = 1 + io_mb = 0 +} diff --git a/conf/experiments/nf1_queue_size_4.config b/conf/experiments/nf1_queue_size_4.config new file mode 100644 index 0000000..2160145 --- /dev/null +++ b/conf/experiments/nf1_queue_size_4.config @@ -0,0 +1,11 @@ +executor { + queueSize = 4 +} + +params { + experiment = 'nf1_queue_size_4' + items = 16 + batch_size = 1 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/nf1_submit_rate_4.config b/conf/experiments/nf1_submit_rate_4.config new file mode 100644 index 0000000..ddfec7b --- /dev/null +++ b/conf/experiments/nf1_submit_rate_4.config @@ -0,0 +1,11 @@ +executor { + submitRateLimit = '4 / 1 min' +} + +params { + experiment = 'nf1_submit_rate_4' + items = 16 + batch_size = 1 + sleep_seconds = 3 + io_mb = 2 +} diff --git a/conf/experiments/zp_synthetic_features.config b/conf/experiments/zp_synthetic_features.config new file mode 100644 index 0000000..3d803e2 --- /dev/null +++ b/conf/experiments/zp_synthetic_features.config @@ -0,0 +1,13 @@ +params { + experiment = 'zp_synthetic_features' + workload = 'zedprofiler_sim' + items = 8 + batch_size = 1 + sleep_seconds = 0 + cpu_seconds = 1 + io_mb = 0 + zp_z = 20 + zp_y = 32 + zp_x = 32 + zp_objects = 4 +} diff --git a/docs/alpine-findings.md b/docs/alpine-findings.md new file mode 100644 index 0000000..3284090 --- /dev/null +++ b/docs/alpine-findings.md @@ -0,0 +1,154 @@ +# Alpine Findings + +These findings come from small synthetic formascute runs on CURC Alpine. No NF1 +image data were used. + +## Validated Setup + +- Submit from the Alpine login node with `SUBMIT_HOST=Persistence1`. +- `Persistence1` is the selected submission location for this project. +- Use the project allocation supplied by the user. +- Use partition `acpu` and QoS `cpu-normal`. +- The coordinator job loads `nextflow/25.10.2`. +- `Persistence1` exposes the Nextflow module tree; the regular login node did + not expose that module during testing. +- `/usr/bin/apptainer` and `/usr/bin/singularity` are available on the tested + compute environment. + +The validated smoke command was: + +```bash +make submit EXPERIMENT=nf1_featurization_independent ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 RUN_ID=remote-smoke-persistence +``` + +Result: + +- coordinator job completed +- Nextflow exit status was `0` +- validation passed +- `trace.tsv` contained native Slurm job IDs +- `slurm.tsv` was collected + +## Small Experiment Results + +All runs validated successfully. + +| mode | experiment | items | batch | coordinator elapsed | native jobs | median wait | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| burst | `nf1_minimal_overhead` | 1 | 1 | 00:00:21 | 2 | 1 s | +| burst | `nf1_featurization_independent` | 4 | 1 | 00:00:27 | 5 | 1 s | +| burst | `nf1_featurization_independent` | 16 | 1 | 00:00:52 | 17 | 1 s | +| burst | `nf1_featurization_batched_2` | 16 | 2 | 00:00:41 | 9 | 1 s | +| burst | `nf1_featurization_batched` | 16 | 4 | 00:00:37 | 5 | 0 s | +| burst | `nf1_queue_size_4` | 16 | 1 | 00:00:57 | 17 | 0 s | +| burst | `nf1_queue_size_20` | 16 | 1 | 00:01:06 | 17 | 0 s | +| burst | `nf1_submit_rate_4` | 16 | 1 | 00:04:16 | 17 | 1 s | +| sequential | `nf1_featurization_independent` | 16 | 1 | 00:00:51 | 17 | 1 s | +| sequential | `nf1_featurization_batched_2` | 16 | 2 | 00:00:42 | 9 | 1 s | +| sequential | `nf1_featurization_batched` | 16 | 4 | 00:00:41 | 5 | 1 s | +| sequential | `nf1_queue_size_4` | 16 | 1 | 00:01:02 | 17 | 1 s | +| burst | `zp_synthetic_features` | 4 | 1 | about 00:00:42 | 5 | 0-1 s | + +## ZedProfiler-Shaped Probe + +`zp_synthetic_features` is a tiny generated-volume workload shaped after +ZedProfiler's CPU-first 3D feature extraction. It creates small object-like +volumes, computes intensity and volume-style summaries, records deterministic +feature checksums, and validates expected item completion. It does not import +ZedProfiler or use NF1 image data. + +The validated command was: + +```bash +make submit EXPERIMENT=zp_synthetic_features ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 RUN_ID=zp-sim-01 +``` + +Result: + +- coordinator job completed +- Nextflow exit status was `0` +- validation passed for 4 expected items +- the run emitted 16 feature rows +- `trace.tsv` contained 4 feature tasks and 1 validation task +- Slurm accounting showed feature tasks completed in about 1-3 seconds each +- peak RSS for feature tasks was about 9-32 MB + +The base Python environment on `Persistence1` is not sufficient for real +ZedProfiler execution: + +- Python: 3.9.13 +- NumPy: 1.21.5 +- Pandas: 1.4.4 +- scikit-image: 0.19.2 +- missing: `mahotas`, `pyarrow`, `zedprofiler` + +ZedProfiler requires Python `>=3.11,<3.15` and depends on packages that are not +present in the tested base environment. A production NF1/ZedProfiler run should +therefore use a project-owned environment or container instead of the default +`Persistence1` Python. + +## Interpretation + +- Nextflow can successfully submit and track Slurm jobs on Alpine when submitted + through `Persistence1`. +- Queue wait was negligible for these tiny runs, usually around 0-1 seconds. +- The fixed overhead for a one-item run was about 21 seconds. +- Independent 16-item fan-out completed in about 51-52 seconds. +- Batching reduced the number of Slurm jobs and slightly reduced elapsed time. +- Batch size 4 was fastest in the burst run, but only slightly faster than batch + size 2 in the sequential run. +- `queueSize=4` was slower than the default independent run at this scale. +- `submitRateLimit='4 / 1 min'` was much slower and should not be the first + control knob for NF1-style work. +- The ZedProfiler-shaped probe behaved like the earlier tiny fan-out tests: + scheduler wait was negligible, and task overhead dominated actual work. + +## Recommended NF1 Orchestration Direction + +Start with a Nextflow wrapper around the existing Stage 3 CPU featurization work +list rather than rewriting feature code. + +Recommended first implementation: + +- Generate a manifest equivalent to `3.cellprofiling/load_data/load_combinations.txt`. +- Use one Nextflow process for CPU featurization work items. +- Keep the existing Python feature scripts as the task payload. +- Use Slurm executor with `acpu`, `cpu-normal`, and account configuration. +- Build an explicit ZedProfiler runtime environment before real data tests. +- Start with independent jobs for real feature tasks when each task loads a + single image-set and runs enough feature work to amortize Slurm overhead. +- Add optional batching when individual feature calls are very short or repeatedly + reload the same image data. +- Prefer batch sizes of 2-4 when batching is needed. +- Do not start with aggressive submit-rate throttling. +- Use `queueSize` as the first scheduler pressure control, then tune only after + observing real queue behavior. +- Consider changing the NF1/ZedProfiler integration so one task can load an + image-set once and compute multiple compatible feature families before writing + outputs. That direction should reduce repeated image I/O compared with one + Slurm job per feature family, channel, and compartment. + +Avoid for the first production-facing iteration: + +- Rewriting the image-processing code. +- Moving directly to Slurm job arrays. +- Mixing CPU and GPU work in one executor profile. +- Running real NF1 image data before a tiny real-work probe is designed. +- Relying on the base `Persistence1` Python environment for ZedProfiler. + +## Questions For CURC + +Ask CURC these before scaling: + +- Should production runs use `acpu`/`cpu-normal` now, or keep compatibility with + `amilan`/`normal` during the transition? +- What `queueSize` range is recommended for Nextflow on Alpine under the + documented user job limit? +- Is a moderate burst of independent jobs preferred over submit-rate throttling + for small jobs? +- Are Slurm job arrays recommended for this workload, or is Nextflow-managed + independent submission easier for accounting and retries? +- Should CPU featurization and SAMMed3D GPU work be split into separate Nextflow + profiles and runs? +- Should ZedProfiler be installed as a project conda/uv environment or as an + Apptainer image for production NF1 runs? diff --git a/docs/clone-ready.md b/docs/clone-ready.md index 11f4d58..8de8a46 100644 --- a/docs/clone-ready.md +++ b/docs/clone-ready.md @@ -9,15 +9,19 @@ From an Alpine login node: ```bash git clone formascute cd formascute -module load nextflow -module load singularity make check make doctor make preflight ACCOUNT= make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= -make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 ``` +On Alpine, formascute defaults to `acpu` and `cpu-normal`. Slurm currently maps +those names to the CPU partition backing the former `amilan`/`normal` names. +The generated coordinator job loads `nextflow/25.10.2` inside the batch job. +Submit through `Persistence1`; it is the selected submission location for this +project. + After the job finishes, inspect: ```bash @@ -31,8 +35,8 @@ cat results//coordinator_*.err Then run the two comparison experiments: ```bash -make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= -make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 ``` Keep these runs small until the outputs show stable scheduler submission, diff --git a/docs/hpc-quickstart.md b/docs/hpc-quickstart.md index ac2fd79..29b7c08 100644 --- a/docs/hpc-quickstart.md +++ b/docs/hpc-quickstart.md @@ -6,13 +6,11 @@ small characterization experiment from the repository root. ```bash git clone formascute cd formascute -module load nextflow -module load singularity make check make doctor -make preflight ACCOUNT=amc-general -make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general -make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT=amc-general +make preflight ACCOUNT= +make submit-dry-run EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= +make submit EXPERIMENT=independent_jobs ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 ``` Each run writes a timestamped directory under `results/` with: @@ -36,9 +34,9 @@ commands directly: ```bash bash bin/formascute doctor -bash bin/formascute preflight --account amc-general -bash bin/formascute submit nf1_featurization_independent --items 16 --account amc-general --dry-run -bash bin/formascute submit independent_jobs --items 16 --account amc-general +bash bin/formascute preflight --account +bash bin/formascute submit nf1_featurization_independent --items 16 --account --dry-run +bash bin/formascute submit independent_jobs --items 16 --account --submit-host Persistence1 ``` ## Baseline And Experiments @@ -51,19 +49,27 @@ Initial experiments: - `independent_jobs`: one work item per task. - `batched_jobs`: multiple work items per task. - `queue_size_20`: Slurm executor queue size set to 20. +- `nf1_minimal_overhead`: one tiny synthetic item to measure coordinator overhead. - `nf1_featurization_independent`: small approximation of the NF1 Stage 3 parent/child fan-out pattern. +- `nf1_featurization_batched_2`: same synthetic NF1-like work grouped in pairs. - `nf1_featurization_batched`: same synthetic NF1-like work grouped into batches. +- `nf1_queue_size_4`: NF1-like independent work with a very small queue size. - `nf1_queue_size_20`: NF1-like independent work with a conservative queue size. +- `nf1_submit_rate_4`: NF1-like independent work with slow submission rate. Run examples: ```bash -make submit EXPERIMENT=independent_jobs ITEMS=32 ACCOUNT=amc-general -make submit EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4 ACCOUNT=amc-general -make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT=amc-general -make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT=amc-general +make submit EXPERIMENT=independent_jobs ITEMS=32 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_minimal_overhead ITEMS=1 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_batched_2 ITEMS=16 BATCH_SIZE=2 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_queue_size_4 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_submit_rate_4 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 make run EXPERIMENT=independent_jobs ITEMS=32 make run EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4 make run EXPERIMENT=queue_size_20 ITEMS=32 @@ -80,15 +86,22 @@ make run EXPERIMENT=queue_size_20 ITEMS=32 - `PROFILE`: Nextflow profile for `make run`. Defaults to `alpine`. - `ACCOUNT`: Slurm account/allocation for `make submit`. Defaults to no explicit account. -- `PARTITION`: Slurm partition for `make submit`. Defaults to `amilan`. -- `QOS`: Slurm QoS for `make submit`. Defaults to `normal`. +- `PARTITION`: Slurm partition for `make submit`. Defaults to `acpu`. +- `QOS`: Slurm QoS for `make submit`. Defaults to `cpu-normal`. +- `SUBMIT_HOST`: optional SSH host used to run `sbatch`. On Alpine, use + `Persistence1`; it is the selected submission location for this project. Useful environment overrides: - `FORMASCUTE_NEXTFLOW_MODULE`: module loaded by generated Slurm scripts. - Defaults to `nextflow`. -- `FORMASCUTE_CONTAINER_MODULE`: container module loaded by generated Slurm - scripts. Defaults to `singularity`. + Defaults to `nextflow/25.10.2`. +- `FORMASCUTE_CONTAINER_MODULE`: optional container module loaded by generated + Slurm scripts. +- `FORMASCUTE_ENABLE_CONTAINER`: enable Nextflow Singularity support. Defaults + to `false` for the synthetic experiments. +- `FORMASCUTE_ENV_DIR`: bootstrap environment directory. Defaults to + `.formascute/conda`. +- `FORMASCUTE_SUBMIT_HOST`: same behavior as `SUBMIT_HOST`. - `FORMASCUTE_SCRATCH`: shared scratch root. Defaults to `/scratch/alpine/$USER`. - `FORMASCUTE_PROJECT`: project/cache root. Defaults to `/projects/$USER`. diff --git a/docs/nf1-pipeline-target.md b/docs/nf1-pipeline-target.md index 42f63f1..27dcaa9 100644 --- a/docs/nf1-pipeline-target.md +++ b/docs/nf1-pipeline-target.md @@ -3,6 +3,9 @@ formascute is aimed at improving the Alpine execution model for `WayScience/NF1_3D_organoid_profiling_pipeline`. +See `docs/north-star.md` for the current gold-standard direction for applying +these findings to the NF1 pipeline. + The current NF1 pipeline uses Slurm directly from shell scripts. The largest scheduler pressure appears in Stage 3 feature extraction and Stage 4 profile merging, where parent scripts read work lists, throttle with `squeue`, and submit @@ -14,7 +17,7 @@ Relevant current behavior: compartment, channel, and processor combination. - Stage 3 throttles based on the user's queued/running Slurm job count. - Stage 4 profile merging submits one child job per patient and well-FOV. -- Alpine CPU jobs use `amilan`; GPU jobs use `aa100`. +- Alpine CPU jobs use `acpu` with `cpu-normal`; GPU jobs use `aa100`. - The README describes a maximum of 990 concurrent jobs per user. - The pipeline uses both CPU and GPU jobs, with larger GPU work for SAMMed3D. - Per well-FOV storage can be around 1-2 GB across raw, z-stack, mask, feature, @@ -35,32 +38,77 @@ must stay synthetic, small, and reversible until Alpine behavior is understood. too much? 5. How should CPU and GPU work be separated so CPU queue pressure does not block SAMMed3D-style GPU tasks? +6. Should ZedProfiler work be grouped by image-set so one task loads data once + and computes multiple compatible feature families? ## Initial Experiments Use only synthetic work in this phase: ```bash -make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= -make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= -make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= +make submit EXPERIMENT=nf1_minimal_overhead ITEMS=1 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_independent ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_batched_2 ITEMS=16 BATCH_SIZE=2 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_queue_size_4 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_queue_size_20 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=nf1_submit_rate_4 ITEMS=16 ACCOUNT= SUBMIT_HOST=Persistence1 +make submit EXPERIMENT=zp_synthetic_features ITEMS=8 ACCOUNT= SUBMIT_HOST=Persistence1 ``` Interpretation: +- `nf1_minimal_overhead` estimates Nextflow + Slurm coordinator overhead before + testing fan-out behavior. - `nf1_featurization_independent` approximates the current one-child-job-per-work item pattern. +- `nf1_featurization_batched_2` tests a small batch size that may reduce job + count without hiding much per-item variability. - `nf1_featurization_batched` tests whether grouping well-FOV-like work reduces queue pressure. +- `nf1_queue_size_4` tests a very conservative active submission window. - `nf1_queue_size_20` tests a conservative Nextflow queue size well below the pipeline's documented 990-user-job scale. +- `nf1_submit_rate_4` tests whether explicit submit-rate throttling changes queue + wait or scheduler behavior separately from queue size. +- `zp_synthetic_features` approximates CPU-first 3D feature extraction using + generated tiny volumes, object-like labels, intensity summaries, and feature + checksums. It does not import ZedProfiler yet; it tests the orchestration + shape before adding that dependency stack. + +Use the resulting `trace.tsv` and `slurm.tsv` files to ask focused questions: + +- Does Alpine prefer fewer larger batches, or many short independent jobs? +- Is queue wait dominated by scheduler load, submit rate, or job size? +- Are small jobs starting quickly enough to justify independent work items? +- Does `queueSize` or `submitRateLimit` better control scheduler pressure? +- Which limits should be set in a future Nextflow implementation before running + real NF1 data? +- Does grouping by image-set reduce repeated image loading enough to justify + larger, more vertical tasks? + +## ZedProfiler Direction + +ZedProfiler is CPU-first and documented as scalable across independent profiling +tasks. On Alpine, the base `Persistence1` Python environment is not ready for +real ZedProfiler work because it provides Python 3.9 and lacks required packages +such as `mahotas`, `pyarrow`, and `zedprofiler`. + +For the NF1 refactor, prefer: + +- a project-owned Python 3.11+ environment or Apptainer image for ZedProfiler +- a Nextflow process that consumes a manifest of image-set work items +- task payloads that can compute multiple compatible feature families after one + image load +- independent Nextflow tasks first, with batch size 2-4 only when per-task + runtime is too short or repeated image loading dominates ## What This Does Not Answer Yet These experiments do not measure image-processing runtime, GPU memory, CellPose, -SAMMed3D, DuckDB, or parquet performance. They only characterize scheduler, -submission, retry, validation, and artifact behavior with a small controlled -workload. +SAMMed3D, DuckDB, parquet performance, or real ZedProfiler dependency +installation. They only characterize scheduler, submission, retry, validation, +and artifact behavior with a small controlled workload. Do not use NF1 image data in this phase. Data-based testing belongs in a later phase, after synthetic runs have shown stable submission, accounting, diff --git a/docs/north-star.md b/docs/north-star.md new file mode 100644 index 0000000..fe739d4 --- /dev/null +++ b/docs/north-star.md @@ -0,0 +1,184 @@ +# North Star + +This is the current gold-standard target for applying formascute's Alpine +findings to `WayScience/NF1_3D_organoid_profiling_pipeline`. + +The goal is not to make the NF1 pipeline more abstract. The goal is to make its +execution model easier to reason about, resume, validate, and discuss with CURC +before production-scale image profiling. + +## Position + +Use Nextflow as the orchestration layer for NF1 execution on Alpine. Keep the +domain processing code in Python, R, and existing scripts where that code is +already correct. Replace nested `sbatch` parent/child submission with explicit +Nextflow processes, manifests, resource labels, and validated outputs. + +This should be a thin workflow layer around real work, not a broad rewrite. + +## Baseline Facts + +- Alpine submission should run through `SUBMIT_HOST=Persistence1`. +- Use `acpu` and `cpu-normal` for small CPU characterization and initial CPU + feature extraction work. +- Load `nextflow/25.10.2` in the coordinator job. +- Keep CPU feature extraction and GPU segmentation or SAMMed3D-style work in + separate profiles or separate runs. +- Do not rely on the base `Persistence1` Python for ZedProfiler. It is Python + 3.9 and lacks required packages. +- Use a project-owned Python 3.11+ environment or an Apptainer image for + ZedProfiler. + +## Target Shape + +The preferred production shape is: + +1. Generate a manifest of work items. +2. Submit one Nextflow coordinator job. +3. Let Nextflow submit Slurm tasks with bounded concurrency. +4. Validate every task output. +5. Collect `trace.tsv`, `slurm.tsv`, logs, reports, and manifest provenance. +6. Resume from the same work directory when safe. +7. Scale only after a small real-work probe succeeds. + +The manifest should be the contract between NF1 data discovery and workflow +execution. It should include enough fields to reproduce a task without rerunning +discovery: + +- patient +- plate or batch +- well +- field of view +- channel paths +- compartment mask paths +- feature families to compute +- output path +- expected output schema or validator +- CPU, memory, time, and profile label + +## ZedProfiler Strategy + +For ZedProfiler-based feature extraction, prefer one task per image-set when +possible. An image-set means one well/FOV with the channels and masks needed for +compatible feature families. + +Within one task: + +- load the image-set once +- compute compatible ZedProfiler feature families +- write one or more validated parquet outputs +- emit a compact task summary + +This is preferred over one Slurm job per feature family, compartment, and channel +when those jobs repeatedly load the same image data. The formascute findings show +that tiny jobs are dominated by orchestration overhead; ZedProfiler's own +scalability guidance also supports choosing horizontal or vertical task grouping +based on data-loading cost and available compute. + +Start independent, then batch only when evidence says to batch: + +- Use independent image-set tasks when each task does enough CPU work to amortize + Slurm overhead. +- Use batch sizes of 2-4 only when tasks are very short or repeatedly reload the + same data. +- Use `queueSize` as the first scheduler pressure control. +- Avoid aggressive `submitRateLimit` unless CURC asks for it or scheduler + behavior requires it. + +## Nextflow Profiles + +Use separate profiles or labels for distinct resource classes: + +- `alpine_cpu`: `acpu`, `cpu-normal`, ZedProfiler and CPU feature extraction +- `alpine_gpu`: GPU partition and QoS for segmentation or SAMMed3D-style work +- `local_smoke`: tiny local syntax and validation check + +Each process should declare realistic starting resources and retry escalation. +Initial CPU feature extraction defaults should be conservative: + +- `cpus`: 1-4 +- `memory`: 4-16 GB +- `time`: 1-4 hours +- `queueSize`: conservative and below user job limits +- `errorStrategy`: retry transient failures, fail validation failures + +Do not mix CPU and GPU work in one executor profile. + +## Validation Standard + +Every production-facing task must produce a small validation artifact alongside +the science output. A task is not complete just because the command exited zero. + +Validation should check: + +- expected output files exist +- outputs are non-empty +- row counts are plausible +- required columns are present +- object or image-set identifiers match the manifest +- parquet or tabular outputs can be opened +- task summary records runtime inputs, output paths, and package versions + +The workflow run directory should preserve: + +- input manifest +- resolved config +- command line +- Nextflow log +- `trace.tsv` +- `timeline.html` +- `report.html` +- `dag.html` +- Slurm accounting when available +- per-task validation summaries + +## First NF1 Implementation Slice + +The first production-facing NF1 branch should implement only Stage 3 CPU +feature extraction with a tiny controlled manifest. + +Recommended slice: + +1. Add a manifest builder for a few known-safe image-sets. +2. Add a ZedProfiler runtime environment definition. +3. Add one Nextflow process for CPU feature extraction. +4. Add per-task validation and summary output. +5. Run one image-set with real ZedProfiler on Alpine. +6. Run 4-8 image-sets independently. +7. Compare independent execution to batch size 2 only if tasks are short. + +Do not include full dataset scale, GPU segmentation, SAMMed3D, profile merging, +or production data movement in the first slice. + +## Scaling Gates + +Scale only after these gates pass: + +- formascute synthetic Alpine checks pass from a fresh clone +- ZedProfiler environment builds reproducibly on Alpine +- one real image-set completes and validates +- 4-8 real image-sets complete and validate +- `trace.tsv` and `slurm.tsv` show acceptable queue wait and memory use +- failed-task behavior is understood +- CURC has reviewed the intended `queueSize`, partition, QoS, and job volume + +## Not The North Star + +Avoid these as first moves: + +- replacing scientific feature code before orchestration is stable +- using the base Alpine Python environment for ZedProfiler +- keeping nested parent/child `sbatch` submission for new work +- moving directly to Slurm job arrays +- submitting hundreds of tiny jobs before real task duration is known +- combining CPU and GPU work in one Nextflow profile +- using production NF1 data as the first validation target + +## Current Best Bet + +The best direction today is a thin Nextflow wrapper around NF1 Stage 3 that +consumes a manifest of image-set work items and runs ZedProfiler in a controlled +Python 3.11+ environment. Start with independent CPU tasks on `acpu` / +`cpu-normal` through `Persistence1`, validate every output, collect trace and +Slurm accounting, and tune batching only after real image-set runtimes show +whether repeated image loading or scheduler overhead dominates. diff --git a/docs/production-checklist.md b/docs/production-checklist.md index 1aeb019..7ea42e5 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -7,8 +7,6 @@ Use this checklist before increasing run size on an HPC system. ```bash git clone formascute cd formascute -module load nextflow -module load singularity make check make doctor make preflight ACCOUNT= @@ -27,12 +25,19 @@ Confirm: - `results//coordinator.sbatch` from `submit-dry-run` has the expected Slurm directives. +Expected Alpine defaults: + +- partition: `acpu` +- QoS: `cpu-normal` +- Nextflow module: `nextflow/25.10.2` +- submit host: `Persistence1` + ## First Scheduler Run Submit the smallest Slurm-backed experiment: ```bash -make submit EXPERIMENT=independent_jobs ITEMS=4 ACCOUNT= +make submit EXPERIMENT=independent_jobs ITEMS=4 ACCOUNT= SUBMIT_HOST=Persistence1 ``` Review the run directory after completion. @@ -50,13 +55,14 @@ Confirm: Increase only one setting at a time. Prefer this sequence: -1. `ITEMS=4` -2. `ITEMS=16` -3. `ITEMS=32` -4. `EXPERIMENT=batched_jobs ITEMS=16 BATCH_SIZE=4` -5. `EXPERIMENT=queue_size_20 ITEMS=16` -6. `EXPERIMENT=nf1_featurization_independent ITEMS=16` -7. `EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4` +1. `EXPERIMENT=nf1_minimal_overhead ITEMS=1` +2. `EXPERIMENT=nf1_featurization_independent ITEMS=4` +3. `EXPERIMENT=nf1_featurization_independent ITEMS=16` +4. `EXPERIMENT=nf1_featurization_batched_2 ITEMS=16 BATCH_SIZE=2` +5. `EXPERIMENT=nf1_featurization_batched ITEMS=16 BATCH_SIZE=4` +6. `EXPERIMENT=nf1_queue_size_4 ITEMS=16` +7. `EXPERIMENT=nf1_queue_size_20 ITEMS=16` +8. `EXPERIMENT=nf1_submit_rate_4 ITEMS=16` Stop and review artifacts whenever validation fails, retry behavior appears, or queue wait differs from expectations. diff --git a/examples/curc_alpine.config b/examples/curc_alpine.config index 2fa2fb4..d7ee9cc 100644 --- a/examples/curc_alpine.config +++ b/examples/curc_alpine.config @@ -24,7 +24,7 @@ process{ queue = 'acpu' errorStrategy = { task.attempt <= 3 ? "retry" : "finish" } process.maxRetries = 3 - clusterOptions = '--account=amc-general --qos=cpu-normal' + clusterOptions = '--account= --qos=cpu-normal' withLabel:process_long { clusterOptions = '--qos=cpu-normal' max_time = 24.h diff --git a/examples/run_sarek.sh b/examples/run_sarek.sh index dc4afaa..20cb3d2 100644 --- a/examples/run_sarek.sh +++ b/examples/run_sarek.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -J sarek #SBATCH --partition acpu -#SBATCH --account=amc-general +#SBATCH --account= #SBATCH -o log/sarek_%j.out # Output file with the job ID #SBATCH -e log/sarek_%j.err # Error file with the job ID #SBATCH -t 24:00:00 # Set the wall time: D-HH:MM:SS @@ -40,4 +40,3 @@ nextflow run "$pipelinepath" -ansi-log false \ --save_trimmed --save_mapped --save_output_as_bam --outdir "$pipeoutdir" \ --tools mutect2,strelka,tiddit,freebayes,vep,snpeff -resume \ -c curc_alpine.config,fastp.config - diff --git a/nextflow.config b/nextflow.config index f2659bd..d0f74e1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -9,6 +9,11 @@ params.sleep_seconds = 1 params.cpu_seconds = 0 params.mem_mb = 0 params.io_mb = 1 +params.workload = 'basic' +params.zp_z = 20 +params.zp_y = 32 +params.zp_x = 32 +params.zp_objects = 4 params.fail_items = '' params.max_items = 256 params.max_batch_size = 64 @@ -16,6 +21,8 @@ params.max_sleep_seconds = 300 params.max_cpu_seconds = 300 params.max_mem_mb = 4096 params.max_io_mb = 256 +params.max_zp_voxels = 1048576 +params.max_zp_objects = 128 profiles { local_smoke { diff --git a/scripts/bootstrap b/scripts/bootstrap new file mode 100644 index 0000000..cbe0a97 --- /dev/null +++ b/scripts/bootstrap @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV_DIR="${FORMASCUTE_ENV_DIR:-$ROOT_DIR/.formascute/conda}" + +have() { + command -v "$1" >/dev/null 2>&1 +} + +if [[ -x "$ENV_DIR/bin/nextflow" && -x "$ENV_DIR/bin/java" ]]; then + echo "ok: existing formascute environment at $ENV_DIR" + "$ENV_DIR/bin/nextflow" -version | sed -n '1,6p' + exit 0 +fi + +if have conda; then + conda create -y -p "$ENV_DIR" -c conda-forge -c bioconda nextflow openjdk=17 +elif have mamba; then + mamba create -y -p "$ENV_DIR" -c conda-forge -c bioconda nextflow openjdk=17 +else + echo "conda or mamba is required for bootstrap" >&2 + exit 127 +fi + +"$ENV_DIR/bin/nextflow" -version | sed -n '1,6p' diff --git a/scripts/lint b/scripts/lint index 9dce92f..41a0016 100755 --- a/scripts/lint +++ b/scripts/lint @@ -11,12 +11,12 @@ echo "shell syntax" while IFS= read -r path; do bash -n "$path" echo "ok: ${path#"$ROOT_DIR"/}" -done < <(find "$ROOT_DIR" -path "$ROOT_DIR/.git" -prune -o -type f -name '*.sh' -print; printf '%s\n' "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint") +done < <(find "$ROOT_DIR" -path "$ROOT_DIR/.git" -prune -o -type f -name '*.sh' -print; printf '%s\n' "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/bootstrap" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint") echo echo "shellcheck" if have shellcheck; then - shellcheck "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint" + shellcheck "$ROOT_DIR/bin/formascute" "$ROOT_DIR/scripts/bootstrap" "$ROOT_DIR/scripts/check" "$ROOT_DIR/scripts/lint" while IFS= read -r path; do shellcheck "$path" done < <(find "$ROOT_DIR" -path "$ROOT_DIR/.git" -prune -o -type f -name '*.sh' -print) diff --git a/workflows/synthetic.nf b/workflows/synthetic.nf index edf41da..4e3cdbf 100644 --- a/workflows/synthetic.nf +++ b/workflows/synthetic.nf @@ -36,6 +36,15 @@ if ((params.io_mb as int) > (params.max_io_mb as int)) { error "params.io_mb exceeds ${params.max_io_mb}" } +def zpVoxels = (params.zp_z as int) * (params.zp_y as int) * (params.zp_x as int) +if (zpVoxels > (params.max_zp_voxels as int)) { + error "ZedProfiler synthetic volume exceeds ${params.max_zp_voxels} voxels" +} + +if ((params.zp_objects as int) > (params.max_zp_objects as int)) { + error "params.zp_objects exceeds ${params.max_zp_objects}" +} + process CHARACTERIZE_ITEM { tag { "batch_${batch_id}" } publishDir "${outdir}/work-items", mode: 'copy' @@ -54,6 +63,8 @@ process CHARACTERIZE_ITEM { python3 - <<'PY' import hashlib import json +import math +import statistics import time from pathlib import Path @@ -64,6 +75,9 @@ sleep_seconds = ${params.sleep_seconds} cpu_seconds = ${params.cpu_seconds} mem_mb = ${params.mem_mb} io_mb = ${params.io_mb} +workload = "${params.workload}" +zp_shape = (${params.zp_z}, ${params.zp_y}, ${params.zp_x}) +zp_objects = ${params.zp_objects} if should_fail: raise SystemExit(f"Intentional failure for batch {batch_id}") @@ -71,6 +85,87 @@ if should_fail: scratch = Path(f"scratch_{batch_id}") scratch.mkdir(exist_ok=True) +def run_basic_work(item_id): + checksum = "" + if io_mb > 0: + path = scratch / f"item_{item_id}.bin" + block = b"0" * 1024 * 1024 + digest = hashlib.sha256() + with path.open("wb") as output: + for _ in range(io_mb): + output.write(block) + digest.update(block) + checksum = digest.hexdigest() + return { + "checksum": checksum, + "feature_rows": 0, + "feature_checksum": "", + "shape": "", + } + +def run_zedprofiler_sim(item_id): + z_size, y_size, x_size = zp_shape + cube = 3 + records = [] + digest = hashlib.sha256() + + for object_id in range(1, zp_objects + 1): + z0 = 1 + ((item_id + object_id) % max(1, z_size - cube - 1)) + y0 = 1 + ((item_id * 3 + object_id * 5) % max(1, y_size - cube - 1)) + x0 = 1 + ((item_id * 7 + object_id * 11) % max(1, x_size - cube - 1)) + values = [] + weighted_z = weighted_y = weighted_x = 0.0 + + for z in range(z0, z0 + cube): + for y in range(y0, y0 + cube): + for x in range(x0, x0 + cube): + value = ((item_id * 17 + object_id * 31 + z * 3 + y * 5 + x * 7) % 251) + 1 + values.append(value) + weighted_z += z * value + weighted_y += y * value + weighted_x += x * value + + integrated = sum(values) + volume = len(values) + mean_value = integrated / volume + center_z = sum(range(z0, z0 + cube)) / cube + center_y = sum(range(y0, y0 + cube)) / cube + center_x = sum(range(x0, x0 + cube)) / cube + intensity_z = weighted_z / integrated + intensity_y = weighted_y / integrated + intensity_x = weighted_x / integrated + displacement = math.sqrt( + (center_z - intensity_z) ** 2 + + (center_y - intensity_y) ** 2 + + (center_x - intensity_x) ** 2 + ) + row = { + "item_id": item_id, + "object_id": object_id, + "volume_voxels": volume, + "integrated_intensity": integrated, + "mean_intensity": round(mean_value, 6), + "std_intensity": round(statistics.pstdev(values), 6), + "min_intensity": min(values), + "max_intensity": max(values), + "mass_displacement": round(displacement, 6), + } + encoded = json.dumps(row, sort_keys=True).encode() + digest.update(encoded) + digest.update(b"\\n") + records.append(row) + + feature_path = scratch / f"item_{item_id}_zp_features.jsonl" + feature_path.write_text( + "".join(json.dumps(record, sort_keys=True) + "\\n" for record in records) + ) + return { + "checksum": "", + "feature_rows": len(records), + "feature_checksum": digest.hexdigest(), + "shape": f"{z_size}x{y_size}x{x_size}", + } + with Path(f"batch_{batch_id}.jsonl").open("w") as handle: for item_id in item_ids: start_epoch = int(time.time()) @@ -87,16 +182,12 @@ with Path(f"batch_{batch_id}.jsonl").open("w") as handle: buf[0] = 1 buf[-1] = 1 - checksum = "" - if io_mb > 0: - path = scratch / f"item_{item_id}.bin" - block = b"0" * 1024 * 1024 - digest = hashlib.sha256() - with path.open("wb") as output: - for _ in range(io_mb): - output.write(block) - digest.update(block) - checksum = digest.hexdigest() + if workload == "basic": + payload = run_basic_work(item_id) + elif workload == "zedprofiler_sim": + payload = run_zedprofiler_sim(item_id) + else: + raise SystemExit(f"Unsupported workload: {workload}") end_epoch = int(time.time()) handle.write(json.dumps({ @@ -104,7 +195,11 @@ with Path(f"batch_{batch_id}.jsonl").open("w") as handle: "batch_id": batch_id, "start_epoch": start_epoch, "end_epoch": end_epoch, - "checksum": checksum, + "workload": workload, + "checksum": payload["checksum"], + "feature_rows": payload["feature_rows"], + "feature_checksum": payload["feature_checksum"], + "shape": payload["shape"], }) + "\\n") PY """ @@ -152,12 +247,19 @@ durations = [record['end_epoch'] - record['start_epoch'] for record in records] summary = { 'experiment': '${params.experiment}', 'run_id': '${params.run_id ?: 'manual'}', + 'workload': '${params.workload}', 'items': expected_items, 'batch_size': ${batchSize}, 'valid': valid, 'min_item_seconds': min(durations) if durations else None, 'max_item_seconds': max(durations) if durations else None, 'total_item_seconds': sum(durations), + 'feature_rows': sum(int(record.get('feature_rows') or 0) for record in records), + 'feature_checksums': sorted( + record.get('feature_checksum') + for record in records + if record.get('feature_checksum') + ), } Path('validation.json').write_text(json.dumps(validation, indent=2) + '\\n')