From 5f1dfc7ea6889679c83a0e59521e2f1af3a84b1e Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 11:50:42 +0100 Subject: [PATCH 01/20] release: prepare 17.0.0 --- codacy/requirements.lock | 48 ++++++++++++++++++++++++++++++++++++++++ codacy/values.yaml | 8 +++---- 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 codacy/requirements.lock diff --git a/codacy/requirements.lock b/codacy/requirements.lock new file mode 100644 index 00000000..f93f5c76 --- /dev/null +++ b/codacy/requirements.lock @@ -0,0 +1,48 @@ +dependencies: +- name: minio + repository: https://charts.helm.sh/stable + version: 5.0.33 +- name: rabbitmq + repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + version: 7.5.7 +- name: postgresql + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami + version: 8.6.4 +- name: log-router + repository: https://charts.codacy.com/external + version: 0.6.4 +- name: codacy-ingress-sh + repository: https://charts.codacy.com/stable + version: 0.2.3 +- name: portal-sh + repository: https://charts.codacy.com/stable + version: 1.1.8 +- name: ragnaros-sh + repository: https://charts.codacy.com/stable + version: 2.0.7 +- name: remote-provider-service-sh + repository: https://charts.codacy.com/stable + version: 1.1.4 +- name: listener-sh + repository: https://charts.codacy.com/stable + version: 1.1.10 +- name: engine-sh + repository: https://charts.codacy.com/stable + version: 1.2.1 +- name: codacy-tools-sh + repository: https://charts.codacy.com/stable + version: 1.2.21 +- name: codacy-api-sh + repository: https://charts.codacy.com/stable + version: 1.2.9 +- name: worker-manager-sh + repository: https://charts.codacy.com/stable + version: 2.0.2 +- name: crow-sh + repository: https://charts.codacy.com/stable + version: 1.0.6 +- name: codacy-spa-sh + repository: https://charts.codacy.com/stable + version: 1.2.4 +digest: sha256:a2a2f97f57abc8101b00c4cb344dc30c05c79a44b2a64a9af601f4c3868ecc32 +generated: "2026-08-28T11:45:44.653253069+01:00" diff --git a/codacy/values.yaml b/codacy/values.yaml index 343286ff..6915eecc 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -28,9 +28,9 @@ global: # Trial license. Valid for 4 users until the end of 2030 license: "lV5QUFrZhlibdCP/MN8pZo76B3jhPeL/RAk4E2AL2WZlGI8Vno/LfvAwnSHm26uY817I3qvEjfhIZFCkDi8R6PngiB+UiT+h8W0Blica8Sp+d5ngnh9HuhTG6xAIf7gAQxyJpn3boV87CKTO+5nHYSeJccMd+s8esMxoS84Z2xI=" installation: - version: development + version: v17.0.0-RC-0 documentation: - version: development + version: v17.0 # github: # enabled: "false" @@ -101,7 +101,7 @@ global: workerManager: workers: config: - imageVersion: stable + imageVersion: sh-1.2.1 javaOpts: '-XX:MinRAMPercentage=50.0 -XX:MaxRAMPercentage=90.0' inactivityTimeout: 30 @@ -318,7 +318,7 @@ rabbitmq-ha: fullnameOverride: codacy-rabbitmq-ha global: imagePullSecrets: - - docker-credentials + - docker-credentials image: repository: bitnamilegacy/rabbitmq tag: "3.8.35-debian-11-r5" From 7b0ab9f2b225c0a200d02566c2f68d37669a1b16 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:04:36 +0100 Subject: [PATCH 02/20] changed rabbit mq values --- .circleci/config.yml | 3 +++ codacy/values-microk8s.yaml | 2 +- codacy/values-production.yaml | 3 +++ codacy/values.yaml | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5325488f..89544145 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,6 +83,9 @@ references: size: 10Gi rabbitmq-ha: replicaCount: 1 + image: + repository: bitnamilegacy/rabbitmq + tag: "3.8.35" ulimitNofiles: 32768 qa_automation_image: &qa_automation_image diff --git a/codacy/values-microk8s.yaml b/codacy/values-microk8s.yaml index 76505a1f..79309aa6 100644 --- a/codacy/values-microk8s.yaml +++ b/codacy/values-microk8s.yaml @@ -57,7 +57,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35-debian-11-r5" + tag: "3.8.35" resources: limits: cpu: 0.5 diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index bddaed44..3009b4e2 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -313,6 +313,9 @@ codacy-spa: rabbitmq-ha: replicaCount: 1 + image: + repository: bitnamilegacy/rabbitmq + tag: "3.8.35" rabbitmqErlangCookie: <--- erlang-cookie ---> # Generate one with `openssl rand -base64 32 | tr -dc 'a-zA-Z0-9'` persistentVolume: enabled: true diff --git a/codacy/values.yaml b/codacy/values.yaml index 6915eecc..dd8d2137 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -321,7 +321,7 @@ rabbitmq-ha: - docker-credentials image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35-debian-11-r5" + tag: "3.8.35" auth: username: rabbitmq-codacy password: CHANGEME From 340cf09a9a6f539bc7223e76378d3da0e73d8a18 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:12:03 +0100 Subject: [PATCH 03/20] Fixed requirements.lock --- codacy/requirements.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/requirements.lock b/codacy/requirements.lock index f93f5c76..1d1d5750 100644 --- a/codacy/requirements.lock +++ b/codacy/requirements.lock @@ -3,7 +3,7 @@ dependencies: repository: https://charts.helm.sh/stable version: 5.0.33 - name: rabbitmq - repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami version: 7.5.7 - name: postgresql repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami From e2fd2b3f01c9e7dfdb0c36df177182418dd65aa2 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:23:00 +0100 Subject: [PATCH 04/20] Dependencies update --- codacy/requirements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/requirements.yaml b/codacy/requirements.yaml index eb6daa18..4c61ebaf 100644 --- a/codacy/requirements.yaml +++ b/codacy/requirements.yaml @@ -7,7 +7,7 @@ dependencies: - name: rabbitmq version: 7.5.7 - repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + repository: https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami condition: global.rabbitmq.create alias: rabbitmq-ha From 297e074d75e6576fe06b3e41472fd08d153e4d3c Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:30:29 +0100 Subject: [PATCH 05/20] regenerated lock file due to digest sha --- codacy/requirements.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codacy/requirements.lock b/codacy/requirements.lock index 1d1d5750..3c802448 100644 --- a/codacy/requirements.lock +++ b/codacy/requirements.lock @@ -44,5 +44,5 @@ dependencies: - name: codacy-spa-sh repository: https://charts.codacy.com/stable version: 1.2.4 -digest: sha256:a2a2f97f57abc8101b00c4cb344dc30c05c79a44b2a64a9af601f4c3868ecc32 -generated: "2026-08-28T11:45:44.653253069+01:00" +digest: sha256:d19e971e5160470800251d2660f063c96c00de20180c13c2f7c7e90375c52862 +generated: "2026-08-28T14:29:21.804571894+01:00" From 02cad8a74336de4045c0bed6bc516a4ca20568ad Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:35:31 +0100 Subject: [PATCH 06/20] Rabbit MQ version pinned --- .circleci/config.yml | 2 +- codacy/values-microk8s.yaml | 2 +- codacy/values-production.yaml | 2 +- codacy/values.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89544145..f1aa5c7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ references: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" ulimitNofiles: 32768 qa_automation_image: &qa_automation_image diff --git a/codacy/values-microk8s.yaml b/codacy/values-microk8s.yaml index 79309aa6..1e6435e6 100644 --- a/codacy/values-microk8s.yaml +++ b/codacy/values-microk8s.yaml @@ -57,7 +57,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" resources: limits: cpu: 0.5 diff --git a/codacy/values-production.yaml b/codacy/values-production.yaml index 3009b4e2..5d65214b 100644 --- a/codacy/values-production.yaml +++ b/codacy/values-production.yaml @@ -315,7 +315,7 @@ rabbitmq-ha: replicaCount: 1 image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" rabbitmqErlangCookie: <--- erlang-cookie ---> # Generate one with `openssl rand -base64 32 | tr -dc 'a-zA-Z0-9'` persistentVolume: enabled: true diff --git a/codacy/values.yaml b/codacy/values.yaml index dd8d2137..cad01db2 100644 --- a/codacy/values.yaml +++ b/codacy/values.yaml @@ -321,7 +321,7 @@ rabbitmq-ha: - docker-credentials image: repository: bitnamilegacy/rabbitmq - tag: "3.8.35" + tag: "3.8.5" auth: username: rabbitmq-codacy password: CHANGEME From 7a974b45b80dd1266dffc87177a975a56f0be079 Mon Sep 17 00:00:00 2001 From: Ranga Ngwerume Date: Fri, 28 Aug 2026 14:39:10 +0100 Subject: [PATCH 07/20] Fixed makefile to inlcude snapshot repo --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e3f3dfe5..54ef0e9a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: setup_helm_repos setup_helm_repos: # Add legacy Bitnami snapshot repo used by requirements.yaml (apiVersion v1) + helm repo add bitnami-pre-2022 https://raw.githubusercontent.com/bitnami/charts/defb094c658024e4aa8245622dab202874880cbc/bitnami # Github tag before bitnami truncated the index. Old link didn't work anymore helm repo add stable https://charts.helm.sh/stable helm repo add codacy-stable https://charts.codacy.com/stable helm repo add codacy-unstable https://charts.codacy.com/unstable From c9485f2a483be942707cf330514acad8c68fe348 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 09:48:38 +0100 Subject: [PATCH 08/20] Increasing the make timeout --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a2454e5..c4e6eed8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,6 +333,7 @@ references: - codacy/run_skip_ssh: cmd_name: Install Codacy cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) + no_output_timeout: 15m helm_push: &helm_push steps: From bf2bcb1ed4a5a00a443df8063d86ad9e6acb0769 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 09:50:15 +0100 Subject: [PATCH 09/20] Increasing the make timeout --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4e6eed8..af454bea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -333,7 +333,7 @@ references: - codacy/run_skip_ssh: cmd_name: Install Codacy cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) - no_output_timeout: 15m + no_output_timeout: 15m helm_push: &helm_push steps: From c5d85711d0aedc47ee78e138e85a9a2e3dbb5aa6 Mon Sep 17 00:00:00 2001 From: Ricardo Bernardino Date: Mon, 31 Aug 2026 10:00:23 +0100 Subject: [PATCH 10/20] Increasing the make timeout --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af454bea..6a2d93e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,10 +330,10 @@ references: steps: - <<: *attach_workspace - <<: *doctl_authenticate - - codacy/run_skip_ssh: - cmd_name: Install Codacy - cmd: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) + - run: + name: Install Codacy no_output_timeout: 15m + command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) helm_push: &helm_push steps: From d8293ea80d41bdc0cbd027cdd43937310d6b54fe Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:14:34 +0100 Subject: [PATCH 11/20] Increase helm deploy timeout to fix release job timeout (#834) The deploy_to_doks_release job (job 30732) timed out because the CircleCI no_output_timeout (15m) matched the helm --timeout (900s). Helm produces no output while waiting for pods to roll out, so CircleCI killed the process just as helm was about to hit its own timeout. Fix: raise no_output_timeout to 30m and helm --timeout to 1500s (25m) so helm has room to complete or fail cleanly before CircleCI intervenes. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- .do/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a2d93e1..1446c015 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -332,7 +332,7 @@ references: - <<: *doctl_authenticate - run: name: Install Codacy - no_output_timeout: 15m + no_output_timeout: 30m command: make -e -C .do/ deploy_to_doks_from_chartmuseum VERSION=$(cat .version) helm_push: &helm_push diff --git a/.do/Makefile b/.do/Makefile index 6ecda9cf..0f2c7c07 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -13,7 +13,7 @@ define call_helm_install eval "extra_args=($$HELM_INSTALL_EXTRA_FLAGS)"; \ helm upgrade --install ${1} ${3} \ --atomic \ - --timeout=900s \ + --timeout=1500s \ -f ../codacy/values-production.yaml \ -f values.yaml \ --namespace ${2} \ From 2b42911e071b42703caec9eae9d4db8ae25eb8b9 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:43:19 +0100 Subject: [PATCH 12/20] Clean up stale hook jobs before helm upgrade to fix BackoffLimitExceeded (#836) * Clean up stale Kubernetes jobs before helm upgrade in release deployment Previous failed deployments leave hook jobs (e.g. DB migrations) behind in the codacy-release namespace. When helm tries to run the same post-upgrade hook on the next attempt it finds an already-failed job and immediately reports BackoffLimitExceeded, causing the deploy_to_doks_release CI job to fail (see job 30795). Add a clean_jobs Makefile target that deletes all jobs in the namespace before the helm upgrade runs. The codacy-release namespace is a dedicated Helm release namespace, so any lingering jobs are stale hook artifacts from prior failed runs and are safe to remove. AI-Generated: true * Apply suggestion from @codacy-production[bot] Co-authored-by: codacy-production[bot] <61871480+codacy-production[bot]@users.noreply.github.com> * Reduce the delete jobs scope --------- Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> Co-authored-by: Ricardo Bernardino Co-authored-by: codacy-production[bot] <61871480+codacy-production[bot]@users.noreply.github.com> --- .do/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.do/Makefile b/.do/Makefile index 0f2c7c07..0872a6b5 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -85,10 +85,14 @@ update_ingress_values: deploy_to_doks_infrastructure: set_cluster_context update_dependencies helm_install_infrastructure .PHONY: deploy_to_doks_from_local_dir -deploy_to_doks_from_local_dir: set_cluster_context update_dependencies helm_install_from_local_dir +deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_jobs helm_install_from_local_dir + +.PHONY: clean_jobs +clean_jobs: + -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum -deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos helm_install_from_chart_museum +deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum .PHONY: set_cluster_context set_cluster_context: From e4c8bdda0e2e986c0267049d9a163cd95f5a4bfb Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:05:36 +0100 Subject: [PATCH 13/20] Fix clean_jobs missing --namespace flag causing BackoffLimitExceeded (#837) The clean_jobs Makefile target ran kubectl delete jobs without specifying the --namespace flag, so it searched the default namespace instead of the release namespace (e.g. codacy-release). This meant stale hook jobs (DB migrations etc.) from prior failed deployments were never cleaned up, and helm's next attempt would find the existing BackoffLimitExceeded job and immediately fail again. Add --namespace ${NAMESPACE} to the kubectl delete command so it targets the correct namespace for every environment (dev, sandbox, release). AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 0872a6b5..641630bd 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -89,7 +89,7 @@ deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_job .PHONY: clean_jobs clean_jobs: - -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --ignore-not-found=true + -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum From b6dec040bb3247a105c602312ce0f5268dd31eed Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:16:54 +0100 Subject: [PATCH 14/20] Add retry logic to deploy_to_doks_from_chartmuseum for hook failures (#838) When a post-upgrade helm hook job fails with BackoffLimitExceeded, the failed job lingers in the namespace. The --atomic flag causes helm to roll back, leaving the hook job behind. On the next CI run, clean_jobs removes it before the upgrade, but the hook can fail again transiently. This change converts deploy_to_doks_from_chartmuseum from a prerequisite-chain target to a recipe that cleans up and retries the helm install once if the first attempt fails, covering transient hook failures (e.g. delete-rabbitmq-queues running before RabbitMQ is ready). AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.do/Makefile b/.do/Makefile index 641630bd..ac90f231 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -92,7 +92,9 @@ clean_jobs: -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum -deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos clean_jobs helm_install_from_chart_museum +deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos + -$(MAKE) clean_jobs + $(MAKE) helm_install_from_chart_museum || ($(MAKE) clean_jobs && $(MAKE) helm_install_from_chart_museum) .PHONY: set_cluster_context set_cluster_context: From 2632250e50d47f2e2dac5f89be57a0d51ad21afa Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:36:18 +0100 Subject: [PATCH 15/20] Fix fluentd lifecycle hook BackoffLimitExceeded by adding MinIO readiness wait (#839) The post-upgrade hook job was spawning 100 pods (backoffLimit: 100, restartPolicy: Never) each failing in ~9s because MinIO isn't ready immediately after a Helm upgrade, totalling ~15 minutes before BackoffLimitExceeded and causing helm --atomic to roll back. Changes: - Add a wait loop in the hook container that polls MinIO (aws s3 ls) every 10s for up to 10 minutes before attempting put-bucket-lifecycle-configuration - Reduce backoffLimit from 100 to 3 since retries are now handled within the container - Add before-hook-creation to hook-delete-policy so any stale failed job is always cleaned up before the next upgrade attempt creates a new one AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .../fluentd/lifecycle-police-job.yaml | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index 68e5167f..eb32590b 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -12,9 +12,9 @@ metadata: # Minio create bucket hook doesn't define any hook weight, # any positive value will ensure that the bucket is created before this "helm.sh/hook-weight": "10" - "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: - backoffLimit: 100 + backoffLimit: 3 template: metadata: name: "{{ .Release.Name }}" @@ -30,14 +30,26 @@ spec: value: "{{ .Values.global.minio.accessKey }}" - name: AWS_SECRET_ACCESS_KEY value: "{{ .Values.global.minio.secretKey }}" - command: ["aws"] + command: ["/bin/sh", "-c"] args: - - s3api - - put-bucket-lifecycle-configuration - - --bucket - - {{ .Values.fluentdoperator.bucketName }} - - --endpoint-url - - http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }} - - --lifecycle-configuration - - '{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + - | + ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" + BUCKET="{{ .Values.fluentdoperator.bucketName }}" + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + MAX_WAIT=600 + ELAPSED=0 + echo "Waiting for MinIO at ${ENDPOINT}..." + until aws s3 ls --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do + if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then + echo "Timed out waiting for MinIO after ${MAX_WAIT}s" + exit 1 + fi + sleep 10 + ELAPSED=$((ELAPSED + 10)) + done + echo "MinIO is ready. Setting lifecycle configuration on bucket ${BUCKET}..." + aws s3api put-bucket-lifecycle-configuration \ + --bucket "${BUCKET}" \ + --endpoint-url "${ENDPOINT}" \ + --lifecycle-configuration "${LIFECYCLE}" {{ end }} From 82b1310ede7140e548a19906d6fd4d9694e228a0 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:11:46 +0100 Subject: [PATCH 16/20] Fix fluentd lifecycle hook BackoffLimitExceeded: add Filter field and pre-push delete (#840) Two fixes for the persistent deploy_to_doks_release failure (job 31035): 1. codacy/templates/fluentd/lifecycle-police-job.yaml: Add "Filter":{} to the lifecycle configuration JSON. Newer MinIO versions strictly follow the AWS S3 spec which requires a Filter element in each lifecycle rule, even when empty. Without it, put-bucket-lifecycle-configuration returns a MalformedXML error immediately, causing 3 fast pod failures and BackoffLimitExceeded (the ~15 min total time is main-deployment rollout; the hook itself fails in seconds). 2. .circleci/config.yml: Delete the existing chart version from chart museum before pushing. Chart museum rejects re-uploads of the same version with 409, which failed the first pipeline run (job 31005). Adding a pre-push DELETE via the chart museum REST API (with || true to tolerate missing charts) ensures subsequent CI runs can always push the updated chart. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 ++ codacy/templates/fluentd/lifecycle-police-job.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1446c015..79ed6ae8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,6 +351,8 @@ references: helm dep build ./codacy echo "Adding 'https://charts.codacy.com/${CHANNEL}'" helm repo add --username "${CHARTS_REPO_USER}" --password "${CHARTS_REPO_PASS}" ${HELM_REPOSITORY} https://charts.codacy.com/${CHANNEL} + echo "Removing existing chart version from chart museum (if present)..." + curl -s -u "${CHARTS_REPO_USER}:${CHARTS_REPO_PASS}" -X DELETE "https://charts.codacy.com/${CHANNEL}/api/charts/codacy/$(cat .version)" || true helm push ./codacy ${HELM_REPOSITORY} - <<: *persist_to_workspace diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index eb32590b..a1fda355 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -35,7 +35,7 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' MAX_WAIT=600 ELAPSED=0 echo "Waiting for MinIO at ${ENDPOINT}..." From f5b218f2b162089382ef0a87b9d6ff97adfb342c Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:11:37 +0100 Subject: [PATCH 17/20] Fix fluentd lifecycle hook waiting for bucket existence, not just MinIO (#841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix (PR #839) waited for MinIO to respond to `aws s3 ls` before calling put-bucket-lifecycle-configuration. However, MinIO can be accessible while the specific bucket doesn't exist yet (bucket creation is a separate hook). This caused rapid failures of the lifecycle config command, hitting BackoffLimitExceeded (with backoffLimit:3) long before Helm's 1500s timeout. Changes: - Wait for the specific bucket (`aws s3api head-bucket`) instead of just MinIO (`aws s3 ls`), so the lifecycle config only runs once the bucket actually exists - Extend MAX_WAIT to 1200s (fits within Helm's 1500s timeout as a single container, no Kubernetes backoff overhead) - Set backoffLimit:0 so Kubernetes doesn't add exponential backoff delays between retries — all retry logic is in-shell - Add an in-shell retry loop (up to 5 attempts) for put-bucket-lifecycle-configuration to handle transient API errors after the bucket is confirmed ready AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> Co-authored-by: Ricardo Bernardino --- .../fluentd/lifecycle-police-job.yaml | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index a1fda355..f30463c8 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -14,7 +14,7 @@ metadata: "helm.sh/hook-weight": "10" "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: - backoffLimit: 3 + backoffLimit: 0 template: metadata: name: "{{ .Release.Name }}" @@ -35,21 +35,32 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' - MAX_WAIT=600 + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + MAX_WAIT=1200 ELAPSED=0 - echo "Waiting for MinIO at ${ENDPOINT}..." - until aws s3 ls --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do + echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." + until aws s3api head-bucket --bucket "${BUCKET}" --endpoint-url "${ENDPOINT}" > /dev/null 2>&1; do if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then - echo "Timed out waiting for MinIO after ${MAX_WAIT}s" + echo "Timed out waiting for MinIO bucket after ${MAX_WAIT}s" exit 1 fi + echo "Bucket not ready yet (${ELAPSED}s elapsed), retrying in 10s..." sleep 10 ELAPSED=$((ELAPSED + 10)) done - echo "MinIO is ready. Setting lifecycle configuration on bucket ${BUCKET}..." - aws s3api put-bucket-lifecycle-configuration \ + echo "MinIO bucket ${BUCKET} is ready. Setting lifecycle configuration..." + RETRY=0 + until aws s3api put-bucket-lifecycle-configuration \ --bucket "${BUCKET}" \ --endpoint-url "${ENDPOINT}" \ - --lifecycle-configuration "${LIFECYCLE}" + --lifecycle-configuration "${LIFECYCLE}"; do + RETRY=$((RETRY + 1)) + if [ "${RETRY}" -ge 5 ]; then + echo "Failed to set lifecycle configuration after ${RETRY} attempts" + exit 1 + fi + echo "Lifecycle config failed, retrying in 10s (attempt ${RETRY}/5)..." + sleep 10 + done + echo "Lifecycle configuration set successfully." {{ end }} From 1f73fa0e4761403224b4ee317cb51ecb74e9d49e Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:11:04 +0100 Subject: [PATCH 18/20] Fix minio-make-bucket-job lingering after failed upgrade (#842) The stable/minio chart (v5.0.33) uses old-style Helm labels (release: , app: minio-make-bucket-job) and has hook-delete-policy: hook-succeeded only. When the job fails, it is not deleted by Helm on rollback. clean_jobs was filtering by app.kubernetes.io/instance which doesn't exist on old-style chart resources, so the stuck job was never cleaned up. The second helm upgrade attempt would then fail with "failed to deploy codacy-minio-make-bucket-job" because the job already existed. Add a second kubectl delete with the legacy release= label selector so clean_jobs catches both new- and old-style Helm-labelled jobs. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .do/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/.do/Makefile b/.do/Makefile index ac90f231..e0148553 100644 --- a/.do/Makefile +++ b/.do/Makefile @@ -90,6 +90,7 @@ deploy_to_doks_from_local_dir: set_cluster_context update_dependencies clean_job .PHONY: clean_jobs clean_jobs: -kubectl delete jobs -l app.kubernetes.io/instance=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true + -kubectl delete jobs -l release=${RELEASE_NAME} --namespace ${NAMESPACE} --ignore-not-found=true .PHONY: deploy_to_doks_from_chartmuseum deploy_to_doks_from_chartmuseum: set_cluster_context setup_helm_repos From abee531836abdff562415086f7f0b452e36e0140 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:17:52 +0100 Subject: [PATCH 19/20] Restore Filter field in fluentd lifecycle configuration (#843) PR #841 accidentally removed the `"Filter":{}` field that PR #840 had added to the S3 lifecycle rule JSON. Newer MinIO versions strictly follow the AWS S3 spec and require a Filter element in each lifecycle rule (even when empty). Without it, put-bucket-lifecycle-configuration returns MalformedXML immediately, causing the in-shell retry loop to exhaust its 5 attempts and exit with failure, which Kubernetes surfaces as BackoffLimitExceeded on the post-upgrade hook. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- codacy/templates/fluentd/lifecycle-police-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codacy/templates/fluentd/lifecycle-police-job.yaml b/codacy/templates/fluentd/lifecycle-police-job.yaml index f30463c8..5f4621a4 100644 --- a/codacy/templates/fluentd/lifecycle-police-job.yaml +++ b/codacy/templates/fluentd/lifecycle-police-job.yaml @@ -35,7 +35,7 @@ spec: - | ENDPOINT="http://{{ .Values.global.minio.location }}:{{ .Values.global.minio.port }}" BUCKET="{{ .Values.fluentdoperator.bucketName }}" - LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"ID":"Delete old logs","Status":"Enabled"}]}' + LIFECYCLE='{"Rules":[{"Expiration":{"Days": {{ .Values.fluentdoperator.expirationDays }} },"Filter":{},"ID":"Delete old logs","Status":"Enabled"}]}' MAX_WAIT=1200 ELAPSED=0 echo "Waiting for MinIO bucket ${BUCKET} at ${ENDPOINT}..." From 4cc6583f1c4626b2eebe1a627540e9bbf2f04e26 Mon Sep 17 00:00:00 2001 From: "circleci-app[bot]" <127350680+circleci-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:27:47 +0100 Subject: [PATCH 20/20] Fix tag_version job: switch from HTTPS to SSH authentication (#844) The codacy/tag_version orb job was failing with: "Invalid username or token. Password authentication is not supported for Git operations." GitHub no longer accepts token-based HTTPS push for git operations. Replace both codacy/tag_version orb calls in the release_pipeline workflow with a custom tag_version_ssh job that uses SSH key authentication, bypassing the HTTPS credential issue entirely. AI-Generated: true Co-authored-by: circleci-app[bot] <127350680+circleci-app[bot]@users.noreply.github.com> --- .circleci/config.yml | 45 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79ed6ae8..8c710784 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -513,6 +513,47 @@ jobs: command: cat .version - <<: *persist_to_workspace + tag_version_ssh: + <<: *default_doks_image + parameters: + version: + type: string + default: "" + force: + type: boolean + default: false + steps: + - <<: *attach_workspace + - add_ssh_keys: + fingerprints: + - "df:83:d7:c7:d5:79:06:c2:3b:d1:fd:e2:a3:d1:12:c5" + - run: + name: Setup SSH + command: ssh-keyscan github.com >> ~/.ssh/known_hosts + - run: + name: Set Git Config + command: | + git config --global user.email "circleci@codacy.com" + git config --global user.name "CircleCI" + git remote set-url origin git@github.com:codacy/chart.git + - run: + name: Fetch Tags + command: git fetch --tags + - run: + name: Tag + command: | + TAG="<< parameters.version >>" + if [ -z "$TAG" ]; then + TAG=$(cat .version) + fi + if [ "<< parameters.force >>" = "true" ]; then + git tag -f "$TAG" + git push origin -f "refs/tags/$TAG" + else + git tag "$TAG" + git push origin "refs/tags/$TAG" + fi + build_docs: docker: - image: circleci/python:3.8 @@ -812,7 +853,7 @@ workflows: requires: - manual_qa_hold - manual_solutions_eng_hold - - codacy/tag_version: + - tag_version_ssh: name: tag_version context: CodacyDO requires: @@ -826,7 +867,7 @@ workflows: target_charts_repo_url: "https://charts.codacy.com/stable" requires: - tag_version - - codacy/tag_version: + - tag_version_ssh: name: tag_version_latest context: CodacyDO version: latest