From 52cf15d09ae9f3f56addba77b1637620b52509c6 Mon Sep 17 00:00:00 2001 From: Charlie Savage Date: Mon, 26 Oct 2026 20:07:26 -0700 Subject: [PATCH] Upgrade CI and test infrastructure Move CI to Helm 4 and pin every tool so results don't change between runs. The unittest and chart-testing jobs used whatever Helm came with the runner image, and helm-unittest was installed unpinned from git. Since snapshot output depends on the helm-unittest version, a new plugin release broke the snapshot tests on master without any chart change. * Helm 3.19.0 -> 4.3.1, set up explicitly in all three jobs * helm-unittest pinned to 1.2.1 (installed with --verify=false, as Helm 4 requires for plugins installed from git) * kube-score 1.17.0 -> 1.20.0 * kind node images 1.32/1.33/1.34 -> 1.35.8/1.36.4/1.37.0, the images for kind 0.33 which kind-action 1.15.1 uses * actions/checkout v4 -> v7, actions/setup-python v5 -> v7 (Python 3.11 -> 3.13), azure/setup-helm 4.3.0 -> 5.0.1, chart-testing-action 2.7.0 -> 2.9.0, kind-action 1.5.0 -> 1.15.1 * Replace the deprecated ::set-output with $GITHUB_OUTPUT * Also run the workflow when .ci/ changes * Publish: set skip_existing so merges that don't bump the chart version (docs, tests, CI) don't fail re-releasing an existing version * Pin the chart-testing image used by local-ct-lint.sh * Regenerate snapshots with helm-unittest 1.2.1, which renders env entries without a value that the old snapshots were missing * TESTING.md: list the current jobs, fix the badge, and document running and updating the unit tests Co-Authored-By: Claude Opus 5.5 --- .ci/scripts/local-ct-lint.sh | 2 +- .github/workflows/on-push-lint-charts.yml | 51 +++++--- .../on-push-master-publish-chart.yml | 6 +- TESTING.md | 26 ++++- .../__snapshot__/deployment_test.yaml.snap | 110 ++++++++++++++++++ .../tests/__snapshot__/secret_test.yaml.snap | 110 ++++++++++++++++++ 6 files changed, 281 insertions(+), 24 deletions(-) diff --git a/.ci/scripts/local-ct-lint.sh b/.ci/scripts/local-ct-lint.sh index cdf9199b..af7b1144 100755 --- a/.ci/scripts/local-ct-lint.sh +++ b/.ci/scripts/local-ct-lint.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run --rm -it -w /repo -v $(pwd):/repo quay.io/helmpack/chart-testing ct lint --all --config .ci/ct-config.yaml +docker run --rm -it -w /repo -v $(pwd):/repo quay.io/helmpack/chart-testing:v3.15.0 ct lint --all --config .ci/ct-config.yaml diff --git a/.github/workflows/on-push-lint-charts.yml b/.github/workflows/on-push-lint-charts.yml index 42b304bf..1eec8f36 100644 --- a/.github/workflows/on-push-lint-charts.yml +++ b/.github/workflows/on-push-lint-charts.yml @@ -5,15 +5,21 @@ on: paths: - 'charts/**' - '.github/**' + - '.ci/**' pull_request: paths: - 'charts/**' - '.github/**' + - '.ci/**' workflow_dispatch: +# Pin every tool so that results don't change underneath us. In particular, test +# snapshots depend on the exact Helm and helm-unittest versions, so regenerate them +# with these versions (see TESTING.md). env: - KUBE_SCORE_VERSION: 1.17.0 - HELM_VERSION: v3.19.0 + HELM_VERSION: v4.3.1 + HELM_UNITTEST_VERSION: v1.2.1 + KUBE_SCORE_VERSION: 1.20.0 concurrency: group: ${{ github.ref }} @@ -24,12 +30,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Set up Helm - uses: azure/setup-helm@v4.3.0 + uses: azure/setup-helm@v5.0.1 with: version: ${{ env.HELM_VERSION }} @@ -48,13 +54,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 - - name: install helm unittest - run: | - helm plugin install https://github.com/helm-unittest/helm-unittest.git + - name: Set up Helm + uses: azure/setup-helm@v5.0.1 + with: + version: ${{ env.HELM_VERSION }} + + # Helm 4 verifies plugin signatures by default, which plugins installed from git don't have + - name: Install helm-unittest + run: helm plugin install https://github.com/helm-unittest/helm-unittest.git --version ${{ env.HELM_UNITTEST_VERSION }} --verify=false - name: Run helm unittest run: helm unittest charts/* @@ -63,40 +74,46 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Choose from https://hub.docker.com/r/kindest/node/tags - KubeVersion: [ 1.32.8, 1.33.4, 1.34.0] + # Node images for the kind version used by helm/kind-action, see + # https://github.com/kubernetes-sigs/kind/releases + KubeVersion: [ 1.35.8, 1.36.4, 1.37.0 ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 + - name: Set up Helm + uses: azure/setup-helm@v5.0.1 + with: + version: ${{ env.HELM_VERSION }} + # python is a requirement for the chart-testing action below (supports yamllint among other tests) - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: - python-version: 3.11 + python-version: '3.13' - name: Set up chart-testing - uses: helm/chart-testing-action@v2.7.0 + uses: helm/chart-testing-action@v2.9.0 - name: Run chart-testing (list-changed) id: list-changed run: | changed=$(ct list-changed --config .ci/ct-config.yaml) if [[ -n "$changed" ]]; then - echo "::set-output name=changed::true" + echo "changed=true" >> "$GITHUB_OUTPUT" fi - name: Run chart-testing (lint) run: ct lint --config .ci/ct-config.yaml - name: Create ${{matrix.KubeVersion}} kind cluster - uses: helm/kind-action@v1.5.0 + uses: helm/kind-action@v1.15.1 with: node_image: kindest/node:v${{matrix.KubeVersion}} cluster_name: kubernetes-${{matrix.KubeVersion}} if: steps.list-changed.outputs.changed == 'true' - name: Run chart-testing (install) against ${{matrix.KubeVersion}} - run: ct install --config .ci/ct-config.yaml \ No newline at end of file + run: ct install --config .ci/ct-config.yaml diff --git a/.github/workflows/on-push-master-publish-chart.yml b/.github/workflows/on-push-master-publish-chart.yml index 734d4eef..3de6aaa4 100644 --- a/.github/workflows/on-push-master-publish-chart.yml +++ b/.github/workflows/on-push-master-publish-chart.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -28,6 +28,10 @@ jobs: - name: Run chart-releaser uses: helm/chart-releaser-action@v1.7.0 + with: + # Changes that don't bump the chart version (docs, tests) would otherwise + # fail trying to re-release an existing version + skip_existing: true env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/TESTING.md b/TESTING.md index 08c61cdb..4fa63bb9 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,20 +2,36 @@ # Automated tests -Every pull request to the master branch trigger the following tests: +Every pull request to the master branch triggers the following tests: +* kube-score +* helm unittest * ct lint -* ct install +* ct install, against several Kubernetes versions using kind +[![Lint and Test Charts](https://github.com/docker-mailserver/docker-mailserver-helm/actions/workflows/on-push-lint-charts.yml/badge.svg)](.github/workflows/on-push-lint-charts.yml) -[![Linting](https://github.com/funkypenguin/helm-docker-mailserver/workflows/Linting/badge.svg)](.github/workflows/on-pr-lint-charts.yml) -[![Testing](https://github.com/funkypenguin/helm-docker-mailserver/workflows/Testing/badge.svg)](.github/workflows/on-pr-test-charts.yml) +The versions of Helm, helm-unittest and the other tools are pinned at the top of [the workflow](.github/workflows/on-push-lint-charts.yml). # Local testing If you're submitting a PR, and you want to ensure your changes will pass automated testing (above), here are your options: +## Unit tests + +Unit tests use [helm-unittest](https://github.com/helm-unittest/helm-unittest). Most of them compare the rendered manifests against snapshots in `charts/docker-mailserver/tests/__snapshot__`. If you change the templates or default values, regenerate the snapshots and check the diff. + +Snapshot output depends on the helm-unittest version, so install the same version as CI (`HELM_UNITTEST_VERSION` in the workflow): + +```console +helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.2.1 --verify=false +helm unittest charts/docker-mailserver # run the tests +helm unittest -u charts/docker-mailserver # update the snapshots +``` + +Leave out `--verify=false` on Helm 3. + ## Linting We use helm's [chart-testing](https://github.com/helm/chart-testing) tool to lint our charts. The tool can be installed locally, or it can be run in a Docker container. @@ -55,4 +71,4 @@ kubectl cluster-info --context kind-kind Have a nice day! 👋 ``` -Trigger a `ct install` test against the KinD cluster, by running `t install --config=.ci/ct-config.yaml`. **ct** will target your current context (be careful if you've got multiple contexts configured!), create a temporary namespace, and deploy the chart into that namespace, until `helm --wait` indicates success. After this, the helm release will be removed, the namespace deleted, and you can retire your KinD cluster by running `kind delete cluster`. +Trigger a `ct install` test against the KinD cluster, by running `ct install --config=.ci/ct-config.yaml`. **ct** will target your current context (be careful if you've got multiple contexts configured!), create a temporary namespace, and deploy the chart into that namespace, until `helm --wait` indicates success. After this, the helm release will be removed, the namespace deleted, and you can retire your KinD cluster by running `kind delete cluster`. diff --git a/charts/docker-mailserver/tests/__snapshot__/deployment_test.yaml.snap b/charts/docker-mailserver/tests/__snapshot__/deployment_test.yaml.snap index fe51cd23..64cc6739 100644 --- a/charts/docker-mailserver/tests/__snapshot__/deployment_test.yaml.snap +++ b/charts/docker-mailserver/tests/__snapshot__/deployment_test.yaml.snap @@ -25,12 +25,30 @@ manifest should match snapshot: spec: containers: - env: + - name: ACCOUNT_PROVISIONER + value: null - name: AMAVIS_LOGLEVEL value: "0" + - name: CLAMAV_MESSAGE_SIZE_LIMIT + value: null + - name: DEFAULT_RELAY_HOST + value: null + - name: DMS_VMAIL_GID + value: null + - name: DMS_VMAIL_UID + value: null + - name: DOVECOT_AUTH_BIND + value: null - name: DOVECOT_INET_PROTOCOLS value: all - name: DOVECOT_MAILBOX_FORMAT value: maildir + - name: DOVECOT_PASS_FILTER + value: null + - name: DOVECOT_TLS + value: null + - name: DOVECOT_USER_FILTER + value: null - name: ENABLE_AMAVIS value: "0" - name: ENABLE_CLAMAV @@ -45,12 +63,16 @@ manifest should match snapshot: value: "0" - name: ENABLE_IMAP value: "1" + - name: ENABLE_MANAGESIEVE + value: null - name: ENABLE_OPENDKIM value: "0" - name: ENABLE_OPENDMARC value: "0" - name: ENABLE_POLICYD_SPF value: "0" + - name: ENABLE_POP3 + value: null - name: ENABLE_POSTGREY value: "0" - name: ENABLE_QUOTAS @@ -77,22 +99,56 @@ manifest should match snapshot: value: "300" - name: GETMAIL_POLL value: "5" + - name: LDAP_QUERY_FILTER_ALIAS + value: null + - name: LDAP_QUERY_FILTER_DOMAIN + value: null + - name: LDAP_QUERY_FILTER_GROUP + value: null + - name: LDAP_QUERY_FILTER_USER + value: null + - name: LDAP_SEARCH_BASE + value: null + - name: LDAP_SERVER_HOST + value: null + - name: LDAP_START_TLS + value: null - name: LOGROTATE_COUNT value: "4" - name: LOGROTATE_INTERVAL value: weekly + - name: LOGWATCH_INTERVAL + value: null + - name: LOGWATCH_RECIPIENT + value: null + - name: LOGWATCH_SENDER + value: null - name: LOG_LEVEL value: info - name: MARK_SPAM_AS_READ value: "0" - name: MOVE_SPAM_TO_JUNK value: "1" + - name: NETWORK_INTERFACE + value: null - name: OVERRIDE_HOSTNAME value: mail.example.com - name: PERMIT_DOCKER value: none + - name: PFLOGSUMM_RECIPIENT + value: null + - name: PFLOGSUMM_SENDER + value: null + - name: PFLOGSUMM_TRIGGER + value: null + - name: POSTFIX_DAGENT + value: null - name: POSTFIX_INET_PROTOCOLS value: all + - name: POSTFIX_MAILBOX_SIZE_LIMIT + value: null + - name: POSTFIX_MESSAGE_SIZE_LIMIT + value: null - name: POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME value: "0" - name: POSTGREY_AUTO_WHITELIST_CLIENTS @@ -103,10 +159,20 @@ manifest should match snapshot: value: "35" - name: POSTGREY_TEXT value: Delayed by Postgrey + - name: POSTMASTER_ADDRESS + value: null - name: POSTSCREEN_ACTION value: enforce + - name: RELAY_HOST + value: null - name: RELAY_PORT value: "25" + - name: RELAY_USER + value: null + - name: REPORT_RECIPIENT + value: null + - name: REPORT_SENDER + value: null - name: RSPAMD_CHECK_AUTHENTICATED value: "0" - name: RSPAMD_GREYLISTING @@ -119,20 +185,64 @@ manifest should match snapshot: value: "0" - name: RSPAMD_NEURAL value: "0" + - name: SASLAUTHD_LDAP_AUTH_METHOD + value: null + - name: SASLAUTHD_LDAP_FILTER + value: null + - name: SASLAUTHD_LDAP_MECH + value: null + - name: SASLAUTHD_LDAP_PASSWORD_ATTR + value: null + - name: SASLAUTHD_LDAP_SEARCH_BASE + value: null + - name: SASLAUTHD_LDAP_SERVER + value: null + - name: SASLAUTHD_LDAP_START_TLS + value: null + - name: SASLAUTHD_LDAP_TLS_CACERT_DIR + value: null + - name: SASLAUTHD_LDAP_TLS_CACERT_FILE + value: null + - name: SASLAUTHD_LDAP_TLS_CHECK_PEER + value: null + - name: SASLAUTHD_MECHANISMS + value: null + - name: SASLAUTHD_MECH_OPTIONS + value: null - name: SA_KILL value: "10" - name: SA_TAG value: "2" - name: SA_TAG2 value: "6.31" + - name: SMTP_ONLY + value: null - name: SPAMASSASSIN_SPAM_TO_INBOX value: "1" - name: SPAM_SUBJECT value: '***SPAM*** ' + - name: SPOOF_PROTECTION + value: null + - name: SRS_EXCLUDE_DOMAINS + value: null + - name: SRS_SECRET + value: null - name: SRS_SENDER_CLASSES value: envelope_sender + - name: SSL_ALT_CERT_PATH + value: null + - name: SSL_ALT_KEY_PATH + value: null + - name: SUPERVISOR_LOGLEVEL + value: null + - name: TLS_LEVEL + value: null + - name: TZ + value: null - name: UPDATE_CHECK_INTERVAL value: 1d + - name: VIRUSMAILS_DELETE_DELAY + value: null image: mailserver/docker-mailserver:0.1.0 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/charts/docker-mailserver/tests/__snapshot__/secret_test.yaml.snap b/charts/docker-mailserver/tests/__snapshot__/secret_test.yaml.snap index 362acbb5..33c101b1 100644 --- a/charts/docker-mailserver/tests/__snapshot__/secret_test.yaml.snap +++ b/charts/docker-mailserver/tests/__snapshot__/secret_test.yaml.snap @@ -25,12 +25,30 @@ manifest should match snapshot: spec: containers: - env: + - name: ACCOUNT_PROVISIONER + value: null - name: AMAVIS_LOGLEVEL value: "0" + - name: CLAMAV_MESSAGE_SIZE_LIMIT + value: null + - name: DEFAULT_RELAY_HOST + value: null + - name: DMS_VMAIL_GID + value: null + - name: DMS_VMAIL_UID + value: null + - name: DOVECOT_AUTH_BIND + value: null - name: DOVECOT_INET_PROTOCOLS value: all - name: DOVECOT_MAILBOX_FORMAT value: maildir + - name: DOVECOT_PASS_FILTER + value: null + - name: DOVECOT_TLS + value: null + - name: DOVECOT_USER_FILTER + value: null - name: ENABLE_AMAVIS value: "0" - name: ENABLE_CLAMAV @@ -45,12 +63,16 @@ manifest should match snapshot: value: "0" - name: ENABLE_IMAP value: "1" + - name: ENABLE_MANAGESIEVE + value: null - name: ENABLE_OPENDKIM value: "0" - name: ENABLE_OPENDMARC value: "0" - name: ENABLE_POLICYD_SPF value: "0" + - name: ENABLE_POP3 + value: null - name: ENABLE_POSTGREY value: "0" - name: ENABLE_QUOTAS @@ -77,22 +99,56 @@ manifest should match snapshot: value: "300" - name: GETMAIL_POLL value: "5" + - name: LDAP_QUERY_FILTER_ALIAS + value: null + - name: LDAP_QUERY_FILTER_DOMAIN + value: null + - name: LDAP_QUERY_FILTER_GROUP + value: null + - name: LDAP_QUERY_FILTER_USER + value: null + - name: LDAP_SEARCH_BASE + value: null + - name: LDAP_SERVER_HOST + value: null + - name: LDAP_START_TLS + value: null - name: LOGROTATE_COUNT value: "4" - name: LOGROTATE_INTERVAL value: weekly + - name: LOGWATCH_INTERVAL + value: null + - name: LOGWATCH_RECIPIENT + value: null + - name: LOGWATCH_SENDER + value: null - name: LOG_LEVEL value: info - name: MARK_SPAM_AS_READ value: "0" - name: MOVE_SPAM_TO_JUNK value: "1" + - name: NETWORK_INTERFACE + value: null - name: OVERRIDE_HOSTNAME value: mail.example.com - name: PERMIT_DOCKER value: none + - name: PFLOGSUMM_RECIPIENT + value: null + - name: PFLOGSUMM_SENDER + value: null + - name: PFLOGSUMM_TRIGGER + value: null + - name: POSTFIX_DAGENT + value: null - name: POSTFIX_INET_PROTOCOLS value: all + - name: POSTFIX_MAILBOX_SIZE_LIMIT + value: null + - name: POSTFIX_MESSAGE_SIZE_LIMIT + value: null - name: POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME value: "0" - name: POSTGREY_AUTO_WHITELIST_CLIENTS @@ -103,10 +159,20 @@ manifest should match snapshot: value: "35" - name: POSTGREY_TEXT value: Delayed by Postgrey + - name: POSTMASTER_ADDRESS + value: null - name: POSTSCREEN_ACTION value: enforce + - name: RELAY_HOST + value: null - name: RELAY_PORT value: "25" + - name: RELAY_USER + value: null + - name: REPORT_RECIPIENT + value: null + - name: REPORT_SENDER + value: null - name: RSPAMD_CHECK_AUTHENTICATED value: "0" - name: RSPAMD_GREYLISTING @@ -119,20 +185,64 @@ manifest should match snapshot: value: "0" - name: RSPAMD_NEURAL value: "0" + - name: SASLAUTHD_LDAP_AUTH_METHOD + value: null + - name: SASLAUTHD_LDAP_FILTER + value: null + - name: SASLAUTHD_LDAP_MECH + value: null + - name: SASLAUTHD_LDAP_PASSWORD_ATTR + value: null + - name: SASLAUTHD_LDAP_SEARCH_BASE + value: null + - name: SASLAUTHD_LDAP_SERVER + value: null + - name: SASLAUTHD_LDAP_START_TLS + value: null + - name: SASLAUTHD_LDAP_TLS_CACERT_DIR + value: null + - name: SASLAUTHD_LDAP_TLS_CACERT_FILE + value: null + - name: SASLAUTHD_LDAP_TLS_CHECK_PEER + value: null + - name: SASLAUTHD_MECHANISMS + value: null + - name: SASLAUTHD_MECH_OPTIONS + value: null - name: SA_KILL value: "10" - name: SA_TAG value: "2" - name: SA_TAG2 value: "6.31" + - name: SMTP_ONLY + value: null - name: SPAMASSASSIN_SPAM_TO_INBOX value: "1" - name: SPAM_SUBJECT value: '***SPAM*** ' + - name: SPOOF_PROTECTION + value: null + - name: SRS_EXCLUDE_DOMAINS + value: null + - name: SRS_SECRET + value: null - name: SRS_SENDER_CLASSES value: envelope_sender + - name: SSL_ALT_CERT_PATH + value: null + - name: SSL_ALT_KEY_PATH + value: null + - name: SUPERVISOR_LOGLEVEL + value: null + - name: TLS_LEVEL + value: null + - name: TZ + value: null - name: UPDATE_CHECK_INTERVAL value: 1d + - name: VIRUSMAILS_DELETE_DELAY + value: null image: mailserver/docker-mailserver:0.1.0 imagePullPolicy: IfNotPresent livenessProbe: