From 573e13dcd9a237b125a06c03b7756395ff8c0e0a Mon Sep 17 00:00:00 2001 From: Christian Paquin Date: Tue, 29 Apr 2025 15:46:24 -0400 Subject: [PATCH 1/7] Add CI to build sample Docker image --- .github/workflows/docker_ci.yml | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/docker_ci.yml diff --git a/.github/workflows/docker_ci.yml b/.github/workflows/docker_ci.yml new file mode 100644 index 00000000..860b86b1 --- /dev/null +++ b/.github/workflows/docker_ci.yml @@ -0,0 +1,70 @@ +name: Weekly build of Sample Docker CI + +on: + # 1) Weekly schedule + schedule: + - cron: '0 0 * * 0' # every Sunday at 00:00 UTC + + # 2) Trigger when the Dockerfile is changed on main + push: + branches: [ main ] + paths: + - 'sample/Dockerfile' + pull_request: + branches: [ main ] + paths: + - 'sample/Dockerfile' + +jobs: + build-and-smoke-test: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build crescent-sample image + uses: docker/build-push-action@v4 + with: + context: . + file: sample/Dockerfile + load: true + tags: crescent-sample:weekly + + - name: Smoke-test extension export + run: | + OUTDIR="${GITHUB_WORKSPACE}/crescent-extension" + rm -rf "$OUTDIR" + mkdir -p "$OUTDIR" + + # Directly copy the built extension & mDL into the host volume + docker run --rm \ + -v "$OUTDIR:/extension" \ + crescent-sample:weekly \ + /bin/bash -c "\ + cp -r /crescent-credentials/sample/client/dist/* /extension/ && \ + cp /crescent-credentials/sample/client/mdl.cbor.hex /extension/ \ + " + + # Fail if nothing was exported + if [ -z "$(ls -A "$OUTDIR")" ]; then + echo "❌ No extension files found in $OUTDIR" + exit 1 + else + echo "✅ Extension export succeeded; files in $OUTDIR:" + ls -1 "$OUTDIR" + fi From fc39872119a5a62aa1c2823fb4c8c7e2842ce236 Mon Sep 17 00:00:00 2001 From: Christian Paquin Date: Tue, 29 Apr 2025 15:48:51 -0400 Subject: [PATCH 2/7] touch Dockerfile to trigger CI --- sample/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample/Dockerfile b/sample/Dockerfile index c552fb0b..eb1abe66 100644 --- a/sample/Dockerfile +++ b/sample/Dockerfile @@ -1,4 +1,4 @@ -# +# # # This Dockerfile is used to build a Docker image for the Crescent credentials project and sample. # You can build and run the sample in the container without installing any dependencies on your local machine @@ -32,7 +32,7 @@ # # Linux # docker run -v "$(pwd)/crescent-extension:/extension" -p 8001:8001 -p 8003:8003 -p 8004:8004 crescent-sample # -# +# FROM rust:slim From dda0bd771b479a021a343bb943c155b7e65a49d8 Mon Sep 17 00:00:00 2001 From: Larry Joy Date: Wed, 18 Jun 2025 17:14:55 -0700 Subject: [PATCH 3/7] Update Docker to copy less to image and run in chunks --- .dockerignore | 23 +++++++++ sample/Dockerfile => Dockerfile | 89 ++++++++++++++++++++++++--------- 2 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 .dockerignore rename sample/Dockerfile => Dockerfile (61%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..782d0104 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.github +.vscode +target +Cargo.lock +node_modules + +circuit_setup/scripts/__pycache__ +circuit_setup/inputs/* +!circuit_setup/inputs/**/*.json +circuit_setup/generated_files/* +circuit_setup/circuits-mdl/circomlib +circuit_setup/circuits/circomlib + +creds/test-vectors/* +creds/pkg + +sample/client/dist +sample/client/mdl.json +sample/client_helper/data +sample/issuer/data +sample/issuer/keys +sample/issuer/.well-known +sample/verifier/data \ No newline at end of file diff --git a/sample/Dockerfile b/Dockerfile similarity index 61% rename from sample/Dockerfile rename to Dockerfile index eb1abe66..a537775f 100644 --- a/sample/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# +# # # This Dockerfile is used to build a Docker image for the Crescent credentials project and sample. # You can build and run the sample in the container without installing any dependencies on your local machine @@ -15,7 +15,7 @@ # cd crescent-credentials # # # Build the Docker image -# docker build -f sample/Dockerfile -t crescent-sample . +# docker build -t crescent-sample . # # # Run the Docker container: @@ -32,9 +32,9 @@ # # Linux # docker run -v "$(pwd)/crescent-extension:/extension" -p 8001:8001 -p 8003:8003 -p 8004:8004 crescent-sample # -# +# -FROM rust:slim +FROM rust:slim AS base SHELL ["/bin/bash", "-c"] @@ -42,7 +42,7 @@ ENV DEBIAN_FRONTEND=noninteractive ENV ROCKET_ADDRESS=0.0.0.0 RUN apt-get update && apt-get upgrade -y && apt-get clean -RUN apt-get install python3.11-venv python3-pip curl git dos2unix m4 cmake libclang-dev bsdextrautils -y +RUN apt-get install python3.11-venv python3-pip curl git dos2unix m4 cmake libclang-dev bsdextrautils tree -y RUN curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - && apt-get install -y nodejs && node -v && npm -v # pip requires a virtual environment to not pollute the system python installation @@ -54,42 +54,82 @@ RUN pip install python_jwt git+https://github.com/peppelinux/pyMDOC-CBOR.git # Install circom RUN git clone https://github.com/iden3/circom.git && cd circom && git checkout v2.1.6 && cargo build --release && cargo install --path circom; +FROM base AS crescent + # Copy local Crescent source code to the container +# The .gitignore file will exclude build artifacts and other unnecessary files. COPY . /crescent-credentials -# Fix line endings for all shell scripts as we may be copying from Windows -RUN find /crescent-credentials -type f -name "*.sh" -exec dos2unix {} + + WORKDIR /crescent-credentials + RUN git submodule update --init --recursive; +RUN rm -rf .git + +RUN tree -d +RUN find . -type f -size +100M -exec stat --format="%s %n" {} + + + + + +# From the repo, there is a symlink here that points to /crescent-credentials/circuit_setup/circuits/circomlib +# On Windows, a script will replace the symlink with a Junction. +# Symlinks do work on Windows but require admin privileges when not in developer mode. +# COPY does not work with the Junction, so we ignore this directory in .dockerignore and re-create the original symlink here instead. +# IMPORTANT: Keep this in sync with the symlink in the repo. +WORKDIR /crescent-credentials/circuit_setup/circuits-mdl +RUN ln -s ../circuits/circomlib circomlib + +WORKDIR /crescent-credentials + +# Fix line endings for all shell scripts as we may be copying from Windows +RUN find /crescent-credentials -type f -name "*.sh" -exec dos2unix {} + + +RUN ./clean_all.sh + +FROM crescent AS setup WORKDIR /crescent-credentials/circuit_setup/scripts RUN ./run_setup.sh rs256 RUN ./run_setup.sh rs256-sd + +RUN ls /crescent-credentials/creds/test-vectors + RUN ./run_setup.sh rs256-db RUN ./run_setup.sh mdl1 WORKDIR /crescent-credentials/creds -RUN cargo run --bin crescent --release --features print-trace zksetup --name rs256 -RUN cargo run --bin crescent --release --features print-trace prove --name rs256 -RUN cargo run --bin crescent --release --features print-trace show --name rs256 -RUN cargo run --bin crescent --release --features print-trace verify --name rs256 -RUN cargo run --bin crescent --release --features print-trace zksetup --name rs256-sd -RUN cargo run --bin crescent --release --features print-trace prove --name rs256-sd -RUN cargo run --bin crescent --release --features print-trace show --name rs256-sd -RUN cargo run --bin crescent --release --features print-trace verify --name rs256-sd -RUN cargo run --bin crescent --release --features print-trace zksetup --name rs256-db -RUN cargo run --bin crescent --release --features print-trace prove --name rs256-db -RUN cargo run --bin crescent --release --features print-trace show --name rs256-db -RUN cargo run --bin crescent --release --features print-trace verify --name rs256-db -RUN cargo run --bin crescent --release --features print-trace zksetup --name mdl1 -RUN cargo run --bin crescent --release --features print-trace prove --name mdl1 -RUN cargo run --bin crescent --release --features print-trace show --name mdl1 -RUN cargo run --bin crescent --release --features print-trace verify --name mdl1 +RUN cargo build --release --features print-trace --bin crescent + +FROM setup AS rs256 +RUN ./target/release/crescent zksetup --name rs256 +RUN ./target/release/crescent prove --name rs256 +RUN ./target/release/crescent show --name rs256 +RUN ./target/release/crescent verify --name rs256 + +FROM rs256 AS rs256-sd +RUN ./target/release/crescent zksetup --name rs256-sd +RUN ./target/release/crescent prove --name rs256-sd +RUN ./target/release/crescent show --name rs256-sd +RUN ./target/release/crescent verify --name rs256-sd + +FROM rs256-sd AS rs256-db +RUN ./target/release/crescent zksetup --name rs256-db +RUN ./target/release/crescent prove --name rs256-db +RUN ./target/release/crescent show --name rs256-db +RUN ./target/release/crescent verify --name rs256-db + +FROM rs256-db AS mdl1 +RUN ./target/release/crescent zksetup --name mdl1 +RUN ./target/release/crescent prove --name mdl1 +RUN ./target/release/crescent show --name mdl1 +RUN ./target/release/crescent verify --name mdl1 WORKDIR /crescent-credentials/ecdsa-pop RUN cargo build --release +FROM mdl1 AS sample WORKDIR /crescent-credentials/sample RUN ./setup-sample.sh @@ -99,11 +139,10 @@ RUN echo '#!/bin/bash' > start-all.sh && \ echo 'cd issuer && cargo run --release &' >> start-all.sh && \ echo 'cd verifier && cargo run --release &' >> start-all.sh && \ echo 'cp -r /crescent-credentials/sample/client/dist/* /extension/' >> start-all.sh && \ - echo 'cp -r /crescent-credentials/sample/client/mdl.cbor.hex /extension/' >> start-all.sh && \ + echo 'cp -r /crescent-credentials/sample/client/mdl.json /extension/' >> start-all.sh && \ echo 'wait -n' >> start-all.sh && \ chmod +x start-all.sh EXPOSE 8001 8003 8004 CMD ["./start-all.sh"] - From c7c47cddf9380c651c47cae6632b22a443b9076c Mon Sep 17 00:00:00 2001 From: Larry Joy Date: Wed, 18 Jun 2025 18:27:45 -0700 Subject: [PATCH 4/7] update dockerfile path in ci --- .github/workflows/docker_ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_ci.yml b/.github/workflows/docker_ci.yml index 860b86b1..81e069d7 100644 --- a/.github/workflows/docker_ci.yml +++ b/.github/workflows/docker_ci.yml @@ -9,11 +9,11 @@ on: push: branches: [ main ] paths: - - 'sample/Dockerfile' + - 'Dockerfile' pull_request: branches: [ main ] paths: - - 'sample/Dockerfile' + - 'Dockerfile' jobs: build-and-smoke-test: @@ -41,7 +41,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - file: sample/Dockerfile + file: Dockerfile load: true tags: crescent-sample:weekly From 5d1e9acf978b27721268cd447b44dd1ba74da877 Mon Sep 17 00:00:00 2001 From: Larry Joy Date: Wed, 18 Jun 2025 21:59:24 -0700 Subject: [PATCH 5/7] add manual start to docker ci --- .github/workflows/docker_ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker_ci.yml b/.github/workflows/docker_ci.yml index 81e069d7..881fe4d3 100644 --- a/.github/workflows/docker_ci.yml +++ b/.github/workflows/docker_ci.yml @@ -1,6 +1,9 @@ name: Weekly build of Sample Docker CI on: + + workflow_dispatch: + # 1) Weekly schedule schedule: - cron: '0 0 * * 0' # every Sunday at 00:00 UTC From ab0b6bb6e2e54a2b1324cef974f066674e4f0c57 Mon Sep 17 00:00:00 2001 From: Larry Joy Date: Thu, 19 Jun 2025 10:00:04 -0700 Subject: [PATCH 6/7] add disk usage output --- .github/workflows/docker_ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker_ci.yml b/.github/workflows/docker_ci.yml index 881fe4d3..d55c59a3 100644 --- a/.github/workflows/docker_ci.yml +++ b/.github/workflows/docker_ci.yml @@ -50,6 +50,12 @@ jobs: - name: Smoke-test extension export run: | + df -h + ls -la /usr/share/dotnet + ls -la /opt/ghc + ls -la /usr/local/lib/android + ls -la /opt/hostedtoolcache + OUTDIR="${GITHUB_WORKSPACE}/crescent-extension" rm -rf "$OUTDIR" mkdir -p "$OUTDIR" From 31c5b71a28a0bc93474b075e1ac52913cf6f7b0c Mon Sep 17 00:00:00 2001 From: Larry Joy Date: Thu, 19 Jun 2025 10:51:08 -0700 Subject: [PATCH 7/7] Temporarily reduce image work to investigate storage issues --- Dockerfile | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index a537775f..f92daea7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,36 +91,34 @@ RUN ./clean_all.sh FROM crescent AS setup WORKDIR /crescent-credentials/circuit_setup/scripts -RUN ./run_setup.sh rs256 -RUN ./run_setup.sh rs256-sd - -RUN ls /crescent-credentials/creds/test-vectors - -RUN ./run_setup.sh rs256-db +# RUN ./run_setup.sh rs256 +# RUN ./run_setup.sh rs256-sd +# RUN ./run_setup.sh rs256-db RUN ./run_setup.sh mdl1 WORKDIR /crescent-credentials/creds RUN cargo build --release --features print-trace --bin crescent -FROM setup AS rs256 -RUN ./target/release/crescent zksetup --name rs256 -RUN ./target/release/crescent prove --name rs256 -RUN ./target/release/crescent show --name rs256 -RUN ./target/release/crescent verify --name rs256 - -FROM rs256 AS rs256-sd -RUN ./target/release/crescent zksetup --name rs256-sd -RUN ./target/release/crescent prove --name rs256-sd -RUN ./target/release/crescent show --name rs256-sd -RUN ./target/release/crescent verify --name rs256-sd - -FROM rs256-sd AS rs256-db -RUN ./target/release/crescent zksetup --name rs256-db -RUN ./target/release/crescent prove --name rs256-db -RUN ./target/release/crescent show --name rs256-db -RUN ./target/release/crescent verify --name rs256-db - -FROM rs256-db AS mdl1 +# FROM setup AS rs256 +# RUN ./target/release/crescent zksetup --name rs256 +# RUN ./target/release/crescent prove --name rs256 +# RUN ./target/release/crescent show --name rs256 +# RUN ./target/release/crescent verify --name rs256 + +# FROM rs256 AS rs256-sd +# RUN ./target/release/crescent zksetup --name rs256-sd +# RUN ./target/release/crescent prove --name rs256-sd +# RUN ./target/release/crescent show --name rs256-sd +# RUN ./target/release/crescent verify --name rs256-sd + +# FROM rs256-sd AS rs256-db +# RUN ./target/release/crescent zksetup --name rs256-db +# RUN ./target/release/crescent prove --name rs256-db +# RUN ./target/release/crescent show --name rs256-db +# RUN ./target/release/crescent verify --name rs256-db + +# FROM rs256-db AS mdl1 +FROM setup AS mdl1 RUN ./target/release/crescent zksetup --name mdl1 RUN ./target/release/crescent prove --name mdl1 RUN ./target/release/crescent show --name mdl1