From 29c464c61ae2ca67d12334026928e141f37d5429 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 17 Jul 2026 11:44:22 +0100 Subject: [PATCH 01/22] feat: add `prometheus` alerts for `etcd` Add alerts for `etcd` obtained from [1] [1]: https://samber.github.io/awesome-prometheus-alerts/rules/orchestrators/etcd/ Signed-off-by: Jack Hodgkiss --- etc/kayobe/kolla/config/prometheus/etcd.rules | 186 ++++++++++++++++++ ...cd-prometheus-alerts-f14710e37d97621f.yaml | 5 + 2 files changed, 191 insertions(+) create mode 100644 etc/kayobe/kolla/config/prometheus/etcd.rules create mode 100644 releasenotes/notes/add-etcd-prometheus-alerts-f14710e37d97621f.yaml diff --git a/etc/kayobe/kolla/config/prometheus/etcd.rules b/etc/kayobe/kolla/config/prometheus/etcd.rules new file mode 100644 index 0000000000..5946ae2fef --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/etcd.rules @@ -0,0 +1,186 @@ +# Source: https://github.com/samber/awesome-prometheus-alerts + +{% raw %} +groups: +- name: EmbeddedExporter + rules: + - alert: EtcdInsufficientMembers + expr: count(etcd_server_id) % 2 == 0 + for: 0m + labels: + severity: critical + annotations: + summary: Etcd insufficient Members (instance {{ $labels.instance }}) + description: "Etcd cluster should have an odd number of members\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdNoLeader + expr: etcd_server_has_leader == 0 + for: 0m + labels: + severity: critical + annotations: + summary: Etcd no Leader (instance {{ $labels.instance }}) + description: "Etcd cluster have no leader\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdHighNumberOfLeaderChanges + expr: increase(etcd_server_leader_changes_seen_total[10m]) > 2 + for: 0m + labels: + severity: warning + annotations: + summary: Etcd high number of leader changes (instance {{ $labels.instance }}) + description: "Etcd leader changed {{ $value }} times during 10 minutes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # Counts server-side failures only. NotFound, AlreadyExists and Cancelled are normal gRPC + # responses and are deliberately excluded from the error rate. + - alert: EtcdHighNumberOfFailedGRPCRequestsWarning + expr: sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.01 and sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd high number of failed GRPC requests warning (instance {{ $labels.instance }}) + description: "More than 1% GRPC request failure detected in Etcd\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # Counts server-side failures only. NotFound, AlreadyExists and Cancelled are normal gRPC + # responses and are deliberately excluded from the error rate. + - alert: EtcdHighNumberOfFailedGRPCRequestsCritical + expr: sum(rate(grpc_server_handled_total{grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.05 and sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: Etcd high number of failed GRPC requests critical (instance {{ $labels.instance }}) + description: "More than 5% GRPC request failure detected in Etcd\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # Excludes the Defragment method, which is inherently slow and would otherwise trigger + # false positives during a manual or automatic etcdctl defrag. + - alert: EtcdGRPCRequestsSlow + expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_method!="Defragment", grpc_type="unary"}[1m])) by (grpc_service, grpc_method, le)) > 0.15 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd GRPC requests slow (instance {{ $labels.instance }}) + description: "GRPC requests slowing down, 99th percentile is over 0.15s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # These etcd_http_* metrics are from the etcd v2 API and do not exist in etcd 3.x. Remove these rules if running etcd 3.x. + - alert: EtcdHighNumberOfFailedHTTPRequestsWarning + expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.01 and sum(rate(etcd_http_received_total[1m])) BY (method) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd high number of failed HTTP requests warning (instance {{ $labels.instance }}) + description: "More than 1% HTTP failure detected in Etcd\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # These etcd_http_* metrics are from the etcd v2 API and do not exist in etcd 3.x. Remove these rules if running etcd 3.x. + - alert: EtcdHighNumberOfFailedHTTPRequestsCritical + expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.05 and sum(rate(etcd_http_received_total[1m])) BY (method) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: Etcd high number of failed HTTP requests critical (instance {{ $labels.instance }}) + description: "More than 5% HTTP failure detected in Etcd\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + # This etcd_http_* metric is from the etcd v2 API and does not exist in etcd 3.x. Remove this rule if running etcd 3.x. + - alert: EtcdHTTPRequestsSlow + expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[1m])) > 0.15 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd HTTP requests slow (instance {{ $labels.instance }}) + description: "HTTP requests slowing down, 99th percentile is over 0.15s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdMemberCommunicationSlow + expr: histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) by (instance, le)) > 0.15 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd member communication slow (instance {{ $labels.instance }}) + description: "Etcd member communication slowing down, 99th percentile is over 0.15s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdHighNumberOfFailedProposals + expr: increase(etcd_server_proposals_failed_total[1h]) > 5 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd high number of failed proposals (instance {{ $labels.instance }}) + description: "Etcd server got {{ $value }} failed proposals in the past hour\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdHighFsyncDurations + expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)) > 0.5 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd high fsync durations (instance {{ $labels.instance }}) + description: "Etcd WAL fsync duration increasing, 99th percentile is over 0.5s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdHighCommitDurations + expr: histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)) > 0.25 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd high commit durations (instance {{ $labels.instance }}) + description: "Etcd commit duration increasing, 99th percentile is over 0.25s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdPeerCommunicationFailures + expr: sum(rate(etcd_network_peer_sent_failures_total[1m])) by (To) > 0.01 + for: 2m + labels: + severity: warning + annotations: + summary: Etcd peer communication failures (instance {{ $labels.instance }}) + description: "Etcd member is experiencing more than 0.01 failed peer sends per second to peer {{ $labels.To }}, which usually indicates the member is unreachable or network-partitioned.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdInsufficientMembersForQuorum + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: 3m + labels: + severity: critical + annotations: + summary: Etcd insufficient members for quorum (instance {{ $labels.instance }}) + description: "Only {{ $value }} etcd members are reachable, fewer than required for quorum; the cluster will reject writes until quorum is restored.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdDatabaseQuotaLowSpace + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m])) * 100 > 95 + for: 10m + labels: + severity: critical + annotations: + summary: Etcd database quota low space (instance {{ $labels.instance }}) + description: "Etcd database size is above 95% of the configured storage quota ({{ $value }}%); once the quota is reached, etcd stops accepting writes cluster-wide.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdExcessiveDatabaseGrowth + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes + for: 10m + labels: + severity: warning + annotations: + summary: Etcd excessive database growth (instance {{ $labels.instance }}) + description: "Based on the current write rate over the past 4 hours, the etcd database is predicted to exceed the configured storage quota within the next 4 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdDatabaseHighFragmentationRatio + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes > 104857600 + for: 10m + labels: + severity: warning + annotations: + summary: Etcd database high fragmentation ratio (instance {{ $labels.instance }}) + description: "Etcd database in-use size is below 50% of its allocated size ({{ $value | humanizePercentage }}), indicating fragmentation; run 'etcdctl defrag' to reclaim disk space.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: EtcdHighFsyncDurationsCritical + expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)) > 1 + for: 2m + labels: + severity: critical + annotations: + summary: Etcd high fsync durations critical (instance {{ $labels.instance }}) + description: "Etcd WAL fsync duration is critically high, 99th percentile is over 1s, which can cause request timeouts and trigger leader elections.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" +{% endraw %} diff --git a/releasenotes/notes/add-etcd-prometheus-alerts-f14710e37d97621f.yaml b/releasenotes/notes/add-etcd-prometheus-alerts-f14710e37d97621f.yaml new file mode 100644 index 0000000000..54c034ce43 --- /dev/null +++ b/releasenotes/notes/add-etcd-prometheus-alerts-f14710e37d97621f.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add alerts for etcd taken from `Awesome Prometheus + `__ From bb02fe397d4ff31b7fe2bdaaa77bab695ab38733 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Tue, 16 Jun 2026 15:43:21 +0100 Subject: [PATCH 02/22] Remove hostname fix reboot Previously there was a hostname-related OVN bug that required a reboot in the AIO CI instances to mitigate. It's been fixed for years[1] so this change reverts the workaround. It'll speed up CI and it's better to know if we regress. [1] https://bugs.launchpad.net/kolla-ansible/+bug/2080552 --- .github/workflows/stackhpc-all-in-one.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index fbfd8514d8..0d04ef1e7c 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -278,28 +278,6 @@ jobs: run: | docker image pull $KAYOBE_IMAGE - # Rocky 9 OVN deployments will fail when the hostname contains a '.' - - name: Fix hostname - run: | - docker run -t --rm \ - -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ - -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ - ${{ steps.kayobe_image.outputs.kayobe_image }} \ - /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible${{ inputs.upgrade && '/' || '/fixes/' }}fix-hostname.yml - env: - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} - - # Reboot to Apply hostname change - - name: Reboot - run: | - docker run -t --rm \ - -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ - -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ - ${{ steps.kayobe_image.outputs.kayobe_image }} \ - /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible${{ inputs.upgrade && '/' || '/maintenance/' }}reboot.yml -e reboot_with_bootstrap_user=true - env: - KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} - - name: Run growroot run: | docker run -t --rm \ From 0079870d1a19ffe8ce4e79783aa5e4902f1ee008 Mon Sep 17 00:00:00 2001 From: Owen Jones Date: Mon, 20 Jul 2026 12:40:14 +0100 Subject: [PATCH 03/22] Rocky 9.8 host image rebuild The previous Rocky 9.8 host image came with broken python 3.9 which caused issues starting deployments (`ASN1: NOT_ENOUGH_DATA`). An updated python package was released[0] and the host image has been rebuilt with this. OFED kernel module has been built for the latest kernel this image uses (`5.14.0.687.26.1.el9.8`). 0: https://errata.rockylinux.org/RLSA-2026:39798 --- etc/kayobe/ofed.yml | 2 +- etc/kayobe/pulp-host-image-versions.yml | 4 +-- etc/kayobe/pulp-repo-versions.yml | 26 +++++++++---------- ...cky-98-python-update-ca5e831d408a67ba.yaml | 10 +++++++ 4 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/rocky-98-python-update-ca5e831d408a67ba.yaml diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml index 28e1825470..6407446595 100644 --- a/etc/kayobe/ofed.yml +++ b/etc/kayobe/ofed.yml @@ -19,7 +19,7 @@ stackhpc_pulp_rocky_10_doca_version: "{{ stackhpc_pulp_doca_version_matrix[doca_ stackhpc_doca_kernel_version_matrix: "9.6": 5.14.0.570.21.1.el9.6 "9.7": 5.14.0.611.55.1.el9.7 - "9.8": 5.14.0.687.25.1.el9.8 + "9.8": 5.14.0.687.26.1.el9.8 "10.2": 6.12.0.211.33.1.el10.2 ############################################################################### diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 892da56abd..624fbea894 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,8 +1,8 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_rocky_9_overcloud_host_image_version: 2025.1-20260715T072311 -stackhpc_rocky_9_overcloud_host_image_version_aarch64: 2025.1-20260715T072311 +stackhpc_rocky_9_overcloud_host_image_version: 2025.1-20260720T114136 +stackhpc_rocky_9_overcloud_host_image_version_aarch64: 2025.1-20260720T114136 stackhpc_rocky_10_overcloud_host_image_version: 2025.1-20260715T072311 stackhpc_rocky_10_overcloud_host_image_version_aarch64: 2025.1-20260715T072311 stackhpc_ubuntu_noble_overcloud_host_image_version: 2025.1-20260706T191822 diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 0cccceb38b..d72b0f9c51 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -36,8 +36,8 @@ stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_aarch64_version: 20260715T185525 stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_version: 20260715T185525 stackhpc_pulp_repo_doca_3_2_3_rhel10_aarch64_version: 20260715T143437 stackhpc_pulp_repo_doca_3_2_3_rhel10_x86_64_version: 20260715T143437 -stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_aarch64_version: 20260715T185525 -stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_version: 20260715T185525 +stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_aarch64_version: 20260720T160316 +stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_version: 20260720T160316 stackhpc_pulp_repo_doca_3_2_3_rhel9_aarch64_version: 20260715T143437 stackhpc_pulp_repo_doca_3_2_3_rhel9_version: 20260715T143437 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20260704T203807 @@ -158,21 +158,21 @@ stackhpc_pulp_repo_rocky_9_7_highavailability_version: 20260514T220209 stackhpc_pulp_repo_rocky_9_7_security_aarch64_version: 20260519T222921 stackhpc_pulp_repo_rocky_9_7_security_source_version: 20260520T222157 stackhpc_pulp_repo_rocky_9_7_security_version: 20260520T215442 -stackhpc_pulp_repo_rocky_9_8_appstream_aarch64_version: 20260715T181215 -stackhpc_pulp_repo_rocky_9_8_appstream_source_version: 20260715T181215 -stackhpc_pulp_repo_rocky_9_8_appstream_version: 20260714T213458 -stackhpc_pulp_repo_rocky_9_8_baseos_aarch64_version: 20260714T221521 -stackhpc_pulp_repo_rocky_9_8_baseos_source_version: 20260713T053209 -stackhpc_pulp_repo_rocky_9_8_baseos_version: 20260714T220645 -stackhpc_pulp_repo_rocky_9_8_crb_aarch64_version: 20260713T223542 +stackhpc_pulp_repo_rocky_9_8_appstream_aarch64_version: 20260716T222443 +stackhpc_pulp_repo_rocky_9_8_appstream_source_version: 20260716T221817 +stackhpc_pulp_repo_rocky_9_8_appstream_version: 20260717T213842 +stackhpc_pulp_repo_rocky_9_8_baseos_aarch64_version: 20260717T221224 +stackhpc_pulp_repo_rocky_9_8_baseos_source_version: 20260717T220601 +stackhpc_pulp_repo_rocky_9_8_baseos_version: 20260716T222336 +stackhpc_pulp_repo_rocky_9_8_crb_aarch64_version: 20260717T221224 stackhpc_pulp_repo_rocky_9_8_crb_source_version: 20260710T221031 -stackhpc_pulp_repo_rocky_9_8_crb_version: 20260714T213458 +stackhpc_pulp_repo_rocky_9_8_crb_version: 20260716T214748 stackhpc_pulp_repo_rocky_9_8_extras_aarch64_version: 20260528T223707 stackhpc_pulp_repo_rocky_9_8_extras_source_version: 20260528T220844 stackhpc_pulp_repo_rocky_9_8_extras_version: 20260528T220059 -stackhpc_pulp_repo_rocky_9_8_highavailability_aarch64_version: 20260710T224503 -stackhpc_pulp_repo_rocky_9_8_highavailability_source_version: 20260624T223310 -stackhpc_pulp_repo_rocky_9_8_highavailability_version: 20260709T215352 +stackhpc_pulp_repo_rocky_9_8_highavailability_aarch64_version: 20260715T222005 +stackhpc_pulp_repo_rocky_9_8_highavailability_source_version: 20260716T221817 +stackhpc_pulp_repo_rocky_9_8_highavailability_version: 20260715T213338 stackhpc_pulp_repo_rocky_9_8_security_aarch64_version: 20260605T230645 stackhpc_pulp_repo_rocky_9_8_security_source_version: 20260605T225017 stackhpc_pulp_repo_rocky_9_8_security_version: 20260606T214954 diff --git a/releasenotes/notes/rocky-98-python-update-ca5e831d408a67ba.yaml b/releasenotes/notes/rocky-98-python-update-ca5e831d408a67ba.yaml new file mode 100644 index 0000000000..184b7eaf18 --- /dev/null +++ b/releasenotes/notes/rocky-98-python-update-ca5e831d408a67ba.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + An updated OFED kernel module has been built for the latest Rocky 9.8 + kernel (``5.14.0.687.26.1.el9.8``). +fixes: + - | + Python 3.9 has been updated in the Rocky 9.8 host image, which addresses + deployment issues caused by a previously introduced Python bug + (``ASN1: NOT_ENOUGH_DATA``). From 71c66517b9ed657ea3a615ac355b1f0c3670735a Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 21 Jul 2026 16:37:52 +0100 Subject: [PATCH 04/22] Add upgrade-prerequsites.sh for Epoxy to Gazpacho Added Valkey migration script. This checks how `kolla_enable_redis`` is evaluated. If it's true, the script calls K-A's Valkey migration. Otherwise, it's a NOOP. --- tools/upgrade-prerequisites.sh | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 tools/upgrade-prerequisites.sh diff --git a/tools/upgrade-prerequisites.sh b/tools/upgrade-prerequisites.sh new file mode 100755 index 0000000000..4dab8fc5de --- /dev/null +++ b/tools/upgrade-prerequisites.sh @@ -0,0 +1,39 @@ +#! /usr/bin/bash + +# This script is intended to be run in CI to test upgrades. +# It executes any preparation steps that must be perfomed before upgrading +# OpenStack services. + +# NOTE(upgrade): This script is unique to each release. It may not be required +# for some releases. + +set -ex + +function prechecks() { + if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + echo "Ensure your environment is set up to run kayobe commands" + exit 2 + fi + + echo "Installing dependencies..." + if type dnf > /dev/null 2>&1; then + sudo dnf -y install jq + else + sudo apt update + sudo apt -y install jq + fi + +} + +function valkey_migration() { + + enable_redis=$(kayobe configuration dump --var-name kolla_enable_redis -l localhost | jq .localhost) + if [[ $enable_redis == "true" || $enable_redis == "yes" ]]; then + kayobe kolla ansible run migrate-valkey -ke enable_redis=false -ke enable_valkey=true + fi + +} + +prechecks +valkey_migration From 856da813de72d9588518c6ccf91ec7b52499c869 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 22 Jul 2026 14:59:17 +0200 Subject: [PATCH 05/22] Remove unused auto-label configuration The GitHub app using it was turned off last summer [1]. [1] https://github.com/apps/product-auto-label --- .github/auto-label.yaml | 55 ----------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 .github/auto-label.yaml diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml deleted file mode 100644 index 0adaf72840..0000000000 --- a/.github/auto-label.yaml +++ /dev/null @@ -1,55 +0,0 @@ ---- -path: - pullrequest: true - multipleLabelPaths: - - paths: - .automation: 'kayobe-automation' - .automation.conf: 'kayobe-automation' - - paths: - doc: 'documentation' - - paths: - .github: 'workflows' - etc: - kayobe: - enviromnents: - ci-aio: 'workflows' - ci-builder: 'workflows' - ci-multinode: 'workflows' - trivy: 'workflows' - terraform: 'workflows' - tools: - scan-images.sh: "workflows" - tox.ini: 'workflows' - - paths: - etc: - kayobe: - ansible: 'ansible' - ansible.cfg: 'ansible' - - paths: - etc: - kayobe: - kolla: - config: - fluentd: "monitoring" - grafana: "monitoring" - prometheus: "monitoring" - inventory: - group_vars: - prometheus-blackbox-exporter: "monitoring" - - paths: - etc: - kayobe: - kolla: - kolla-build.conf: "kolla" - kolla-image-tags.yml: "kolla" - kolla.yml: "kolla" - tools: - kolla-images.py: "kolla" - -staleness: - pullrequest: true - old: 30 - extraold: 90 - -requestsize: - enabled: true From 04bfff9ae31f7570e9af08d2b1698d4d23e3bb9b Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:29:30 +0000 Subject: [PATCH 06/22] (automated) Bump kolla-ansible to stackhpc/20.4.0.11 --- etc/kayobe/stackhpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 122444a19d..2faa6af08e 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -230,7 +230,7 @@ stackhpc_kolla_source_version: stackhpc/20.4.0.10 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.10 +stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.11 ############################################################################### # Container image registry From d6b67b69781d8f53b4de52aade75d7e81743b134 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Thu, 23 Jul 2026 11:07:26 +0100 Subject: [PATCH 07/22] GT-1835: Fix IPA for stream 10 (#2448) This switches to an IPA builder fork which is required for centos-10 stream IPA images, which are required (currently) to provide a version of qemu-img that can handle 4k sectors from Dell Perc RAID controllers which don't support other block sizes. Anyone not affected can carry on using stream-9 images until Rocky ones exist. We can drop this patch if we can backport https://review.opendev.org/c/openstack/ironic-python-agent-builder/+/995589 --- etc/kayobe/ipa.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml index d0b58729a9..4dd1b6dff8 100644 --- a/etc/kayobe/ipa.yml +++ b/etc/kayobe/ipa.yml @@ -14,10 +14,14 @@ #ipa_build_source_version: # URL of IPA builder source repository. -#ipa_builder_source_url: +# NOTE(dougszu): This fork is required for centos-10 stream IPA images, which +# are required (currently) to provide a version of qemu-img that can handle +# 4k sectors from Dell Perc RAID controllers which don't support other block +# sizes. +ipa_builder_source_url: https://github.com/stackhpc/ironic-python-agent-builder.git # Version of IPA builder source repository. Default is {{ openstack_branch }}. -#ipa_builder_source_version: +ipa_builder_source_version: stackhpc/2025.1 # List of additional build host packages to install. Default is an empty list. ipa_build_dib_host_packages_extra: [ 'zstd' ] From a40a5ec20a6aa7d225d49ef66f3483c71d90071f Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Thu, 23 Jul 2026 14:56:22 +0200 Subject: [PATCH 08/22] Use host facts to select Rocky 9/10 distro version Update kolla_base_distro_version_default_map to select Rocky 9 or Rocky 10 images for each host. --- etc/kayobe/kolla/globals.yml | 2 +- .../notes/r9-r10-kolla-base-distro-75078c4fbae4f785.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/r9-r10-kolla-base-distro-75078c4fbae4f785.yaml diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 250d642c11..b42d0f42b4 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -17,7 +17,7 @@ kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_d kolla_base_distro_version_default_map: { "centos": "stream9", "debian": "bookworm", - "rocky": "{{ os_release }}", + "rocky": "{% raw %}{{ ansible_facts.distribution_major_version }}{% endraw %}", "ubuntu": "noble", } diff --git a/releasenotes/notes/r9-r10-kolla-base-distro-75078c4fbae4f785.yaml b/releasenotes/notes/r9-r10-kolla-base-distro-75078c4fbae4f785.yaml new file mode 100644 index 0000000000..46373b75af --- /dev/null +++ b/releasenotes/notes/r9-r10-kolla-base-distro-75078c4fbae4f785.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Uses host facts in ``kolla_base_distro_version_default_map`` to select + Rocky 9 or Rocky 10 Kolla images for each host. From 00ffa6070518c0dd4c9a78719639237eaccd15f0 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 23 Jul 2026 16:55:40 +0100 Subject: [PATCH 09/22] Add coderabbit review config --- .coderabbit.yaml | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000000..bdc98fc77a --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,67 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +# CodeRabbit review config file + +language: "en-GB" +reviews: + # Options are "quiet", "chill", "assertive" in order of nitpickyness + profile: "assertive" + # Don't automatically approves PRs + request_changes_workflow: false + # Summarise the PR in a comment + high_level_summary: true + # Summary instructions + high_level_summary_instructions: "Give a brief description of the changes in 50 words or less." + # Put summary in a comment, don't override the PR description + high_level_summary_in_walkthrough: true + # Post a comment e.g. when a review is skipped, why so + review_status: true + # Posts extra details about the review (ignored files, extra context used, suppressed comments, etc.) + review_details: true + # Don't suggest labels to apply to PRs + suggested_labels: false + # Don't suggest reviewers to add to the PR + suggested_reviewers: false + # Don't post a "fortune" message while the review is running + in_progress_fortune: false + # Don't include review comments to provide codegen instructions for AI agents. + enable_prompt_for_ai_agents: false + + # Auto review configuration + auto_review: + # Review things + enabled: true + # Review draft PRs + drafts: true + # Don't auto re-review on push (Can always be re-triggered manually) + auto_incremental_review: false + # Branches to auto-review PRs against + # Regex should match branch such as: + # stackhpc/2025.1 + # stackhpc-dev/2026.1 + # stackhpc-rc/2027.1 + base_branches: + - ^stackhpc/202\d\.1$ + - ^stackhpc-rc/202\d\.1$ + - ^stackhpc-dev/202\d\.1$ + # Ignore PRs by the CI bot + ignore_usernames: + - "stackhpc-ci" + + finishing_touches: + # Don't try to add docstrings and unit tests + docstrings: + enabled: false + unit_tests: + enabled: false + + pre_merge_checks: + # Ignore title/description checks + title: + mode: "off" + description: + mode: "off" + +# Additional context options +knowledge_base: + # Auto link to other repos in the org + automatic_repository_linking: true From 9a9341ce8c57ba20a1238381020ec80913df6adc Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 24 Jul 2026 16:56:18 +0200 Subject: [PATCH 10/22] Add hook to reset-bls-entries after host package update Also ensure that the playbook is safe to run on Ubuntu. --- .../ansible/maintenance/reset-bls-entries.yml | 51 ++++++++++--------- .../post.d/20-reset-bls-entries | 1 + .../reset-bls-entries-20375af82dfd3517.yaml | 5 ++ 3 files changed, 33 insertions(+), 24 deletions(-) create mode 120000 etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries create mode 100644 etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml diff --git a/etc/kayobe/ansible/maintenance/reset-bls-entries.yml b/etc/kayobe/ansible/maintenance/reset-bls-entries.yml index e26593b708..3d67798c78 100644 --- a/etc/kayobe/ansible/maintenance/reset-bls-entries.yml +++ b/etc/kayobe/ansible/maintenance/reset-bls-entries.yml @@ -10,30 +10,33 @@ tags: - reset-bls-entries tasks: - - name: Get machine ID - ansible.builtin.command: cat /etc/machine-id - register: machine_id - check_mode: false + - name: Reset BLS entries on RedHat-based systems + when: ansible_facts.os_family == 'RedHat' + block: + - name: Get machine ID + ansible.builtin.command: cat /etc/machine-id + register: machine_id + check_mode: false - - name: Find entries with wrong machine ID - ansible.builtin.find: - paths: /boot/loader/entries - patterns: "*.conf" - register: bls_entries - check_mode: false + - name: Find entries with wrong machine ID + ansible.builtin.find: + paths: /boot/loader/entries + patterns: "*.conf" + register: bls_entries + check_mode: false - # We set force to false to avoid replacing an existing BLS entry with the - # correct machine ID. - - name: Rename entries with wrong machine ID - ansible.builtin.copy: - src: /boot/loader/entries/{{ item }} - dest: /boot/loader/entries/{{ item | ansible.builtin.regex_replace('^[a-f0-9]*', machine_id.stdout) }} - force: false - remote_src: true - with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" + # We set force to false to avoid replacing an existing BLS entry with the + # correct machine ID. + - name: Rename entries with wrong machine ID + ansible.builtin.copy: + src: /boot/loader/entries/{{ item }} + dest: /boot/loader/entries/{{ item | ansible.builtin.regex_replace('^[a-f0-9]*', machine_id.stdout) }} + force: false + remote_src: true + with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" - - name: Remove entries with wrong machine ID - ansible.builtin.file: - path: /boot/loader/entries/{{ item }} - state: absent - with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" + - name: Remove entries with wrong machine ID + ansible.builtin.file: + path: /boot/loader/entries/{{ item }} + state: absent + with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" diff --git a/etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries b/etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries new file mode 120000 index 0000000000..8a594a37e1 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries @@ -0,0 +1 @@ +../../../ansible/maintenance/reset-bls-entries.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml b/etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml new file mode 100644 index 0000000000..e852fb57ad --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Adds a hook to automatically fix BLS configuration files under + ``/boot/loader/entries`` after overcloud host package update. From 03f59cf548823651766d62573b76977569474997 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Jul 2026 12:46:01 +0200 Subject: [PATCH 11/22] Revert "GT-1835: Fix IPA for stream 10 (#2448)" This reverts commit d6b67b69781d8f53b4de52aade75d7e81743b134. Reason for revert: fixes have been merged upstream. --- etc/kayobe/ipa.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml index 4dd1b6dff8..d0b58729a9 100644 --- a/etc/kayobe/ipa.yml +++ b/etc/kayobe/ipa.yml @@ -14,14 +14,10 @@ #ipa_build_source_version: # URL of IPA builder source repository. -# NOTE(dougszu): This fork is required for centos-10 stream IPA images, which -# are required (currently) to provide a version of qemu-img that can handle -# 4k sectors from Dell Perc RAID controllers which don't support other block -# sizes. -ipa_builder_source_url: https://github.com/stackhpc/ironic-python-agent-builder.git +#ipa_builder_source_url: # Version of IPA builder source repository. Default is {{ openstack_branch }}. -ipa_builder_source_version: stackhpc/2025.1 +#ipa_builder_source_version: # List of additional build host packages to install. Default is an empty list. ipa_build_dib_host_packages_extra: [ 'zstd' ] From 8ebfcea4aec01140aa30ea92a1e321a1cde4be85 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Jul 2026 13:24:43 +0200 Subject: [PATCH 12/22] Bump all repositories --- etc/kayobe/pulp-repo-versions.yml | 96 +++++++++++++++---------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index d72b0f9c51..eb762cb377 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -2,26 +2,26 @@ # This file is autogenerated by Ansible using the following workflow: # https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml stackhpc_pulp_repo_almalinux_9_proxysql_2_7_version: 20250627T134211 -stackhpc_pulp_repo_centos_stream_10_docker_aarch64_version: 20260702T230648 -stackhpc_pulp_repo_centos_stream_10_docker_version: 20260702T204052 +stackhpc_pulp_repo_centos_stream_10_docker_aarch64_version: 20260716T231645 +stackhpc_pulp_repo_centos_stream_10_docker_version: 20260716T214407 stackhpc_pulp_repo_centos_stream_10_nfv_openvswitch_aarch64_version: 20260603T233007 stackhpc_pulp_repo_centos_stream_10_nfv_openvswitch_version: 20260602T230909 stackhpc_pulp_repo_centos_stream_10_openstack_epoxy_aarch64_version: 20260227T222400 stackhpc_pulp_repo_centos_stream_10_openstack_epoxy_version: 20260227T214730 stackhpc_pulp_repo_centos_stream_10_ovn_lts_aarch64_version: 20260608T144504 stackhpc_pulp_repo_centos_stream_10_ovn_lts_version: 20260608T144504 -stackhpc_pulp_repo_centos_stream_10_storage_ceph_squid_aarch64_version: 20260603T233007 -stackhpc_pulp_repo_centos_stream_10_storage_ceph_squid_version: 20260602T230909 -stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20260702T222046 -stackhpc_pulp_repo_centos_stream_9_docker_version: 20260702T204052 +stackhpc_pulp_repo_centos_stream_10_storage_ceph_squid_aarch64_version: 20260715T232156 +stackhpc_pulp_repo_centos_stream_10_storage_ceph_squid_version: 20260715T221325 +stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20260716T222443 +stackhpc_pulp_repo_centos_stream_9_docker_version: 20260716T214407 stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20260602T230751 stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20260602T230909 stackhpc_pulp_repo_centos_stream_9_openstack_epoxy_aarch64_version: 20250521T015033 stackhpc_pulp_repo_centos_stream_9_openstack_epoxy_version: 20250521T021940 stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318 -stackhpc_pulp_repo_centos_stream_9_storage_ceph_squid_aarch64_version: 20260603T230620 -stackhpc_pulp_repo_centos_stream_9_storage_ceph_squid_version: 20260603T230019 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_squid_aarch64_version: 20260715T222005 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_squid_version: 20260715T221325 stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245 stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620 stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314 @@ -40,26 +40,26 @@ stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_aarch64_version: 20260720T160316 stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_version: 20260720T160316 stackhpc_pulp_repo_doca_3_2_3_rhel9_aarch64_version: 20260715T143437 stackhpc_pulp_repo_doca_3_2_3_rhel9_version: 20260715T143437 -stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20260704T203807 +stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20260727T204056 stackhpc_pulp_repo_elrepo_10_aarch64_version: 20260220T152827 -stackhpc_pulp_repo_elrepo_10_version: 20260620T002710 +stackhpc_pulp_repo_elrepo_10_version: 20260717T223545 stackhpc_pulp_repo_elrepo_9_aarch64_version: 20250408T030629 -stackhpc_pulp_repo_elrepo_9_version: 20260624T220025 -stackhpc_pulp_repo_epel_10_aarch64_version: 20260705T225747 -stackhpc_pulp_repo_epel_10_version: 20260705T213055 -stackhpc_pulp_repo_epel_9_aarch64_version: 20260704T222111 -stackhpc_pulp_repo_epel_9_version: 20260705T213055 -stackhpc_pulp_repo_grafana_version: 20260701T070549 +stackhpc_pulp_repo_elrepo_9_version: 20260717T214449 +stackhpc_pulp_repo_epel_10_aarch64_version: 20260727T231655 +stackhpc_pulp_repo_epel_10_version: 20260727T215111 +stackhpc_pulp_repo_epel_9_aarch64_version: 20260727T223147 +stackhpc_pulp_repo_epel_9_version: 20260727T215111 +stackhpc_pulp_repo_grafana_version: 20260725T203624 stackhpc_pulp_repo_opensearch_2_x_version: 20260703T211022 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20260703T211022 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20260112T224827 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20260112T224827 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20260616T214234 -stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20260616T214234 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20260724T213912 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20260724T213912 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20260720T210122 +stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20260723T204348 stackhpc_pulp_repo_rhel_10_mariadb_10_11_aarch64_version: 20260527T210633 stackhpc_pulp_repo_rhel_10_mariadb_10_11_version: 20260527T210633 stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20260112T224827 -stackhpc_pulp_repo_rhel_9_influxdb_version: 20260630T222710 +stackhpc_pulp_repo_rhel_9_influxdb_version: 20260721T205558 stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20260527T210633 stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20260527T210633 stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20251219T213344 @@ -82,21 +82,21 @@ stackhpc_pulp_repo_rocky_10_1_highavailability_version: 20260506T223941 stackhpc_pulp_repo_rocky_10_1_security_aarch64_version: 20260510T225242 stackhpc_pulp_repo_rocky_10_1_security_source_version: 20260510T222658 stackhpc_pulp_repo_rocky_10_1_security_version: 20260510T223302 -stackhpc_pulp_repo_rocky_10_2_appstream_aarch64_version: 20260714T230725 -stackhpc_pulp_repo_rocky_10_2_appstream_source_version: 20260714T224455 -stackhpc_pulp_repo_rocky_10_2_appstream_version: 20260714T223154 -stackhpc_pulp_repo_rocky_10_2_baseos_aarch64_version: 20260714T230725 -stackhpc_pulp_repo_rocky_10_2_baseos_source_version: 20260714T224455 -stackhpc_pulp_repo_rocky_10_2_baseos_version: 20260714T225549 -stackhpc_pulp_repo_rocky_10_2_crb_aarch64_version: 20260714T230725 -stackhpc_pulp_repo_rocky_10_2_crb_source_version: 20260710T224916 -stackhpc_pulp_repo_rocky_10_2_crb_version: 20260714T223154 +stackhpc_pulp_repo_rocky_10_2_appstream_aarch64_version: 20260727T231655 +stackhpc_pulp_repo_rocky_10_2_appstream_source_version: 20260723T225406 +stackhpc_pulp_repo_rocky_10_2_appstream_version: 20260727T222657 +stackhpc_pulp_repo_rocky_10_2_baseos_aarch64_version: 20260727T231655 +stackhpc_pulp_repo_rocky_10_2_baseos_source_version: 20260727T225001 +stackhpc_pulp_repo_rocky_10_2_baseos_version: 20260727T225537 +stackhpc_pulp_repo_rocky_10_2_crb_aarch64_version: 20260727T231655 +stackhpc_pulp_repo_rocky_10_2_crb_source_version: 20260727T225001 +stackhpc_pulp_repo_rocky_10_2_crb_version: 20260724T095616 stackhpc_pulp_repo_rocky_10_2_extras_aarch64_version: 20260602T012631 stackhpc_pulp_repo_rocky_10_2_extras_source_version: 20260602T004048 stackhpc_pulp_repo_rocky_10_2_extras_version: 20260602T001113 -stackhpc_pulp_repo_rocky_10_2_highavailability_aarch64_version: 20260626T010026 -stackhpc_pulp_repo_rocky_10_2_highavailability_source_version: 20260624T232023 -stackhpc_pulp_repo_rocky_10_2_highavailability_version: 20260626T002921 +stackhpc_pulp_repo_rocky_10_2_highavailability_aarch64_version: 20260723T231652 +stackhpc_pulp_repo_rocky_10_2_highavailability_source_version: 20260715T225208 +stackhpc_pulp_repo_rocky_10_2_highavailability_version: 20260723T223343 stackhpc_pulp_repo_rocky_10_2_security_aarch64_version: 20260606T230245 stackhpc_pulp_repo_rocky_10_2_security_source_version: 20260605T232846 stackhpc_pulp_repo_rocky_10_2_security_version: 20260606T223509 @@ -158,27 +158,27 @@ stackhpc_pulp_repo_rocky_9_7_highavailability_version: 20260514T220209 stackhpc_pulp_repo_rocky_9_7_security_aarch64_version: 20260519T222921 stackhpc_pulp_repo_rocky_9_7_security_source_version: 20260520T222157 stackhpc_pulp_repo_rocky_9_7_security_version: 20260520T215442 -stackhpc_pulp_repo_rocky_9_8_appstream_aarch64_version: 20260716T222443 -stackhpc_pulp_repo_rocky_9_8_appstream_source_version: 20260716T221817 -stackhpc_pulp_repo_rocky_9_8_appstream_version: 20260717T213842 -stackhpc_pulp_repo_rocky_9_8_baseos_aarch64_version: 20260717T221224 -stackhpc_pulp_repo_rocky_9_8_baseos_source_version: 20260717T220601 -stackhpc_pulp_repo_rocky_9_8_baseos_version: 20260716T222336 -stackhpc_pulp_repo_rocky_9_8_crb_aarch64_version: 20260717T221224 -stackhpc_pulp_repo_rocky_9_8_crb_source_version: 20260710T221031 -stackhpc_pulp_repo_rocky_9_8_crb_version: 20260716T214748 +stackhpc_pulp_repo_rocky_9_8_appstream_aarch64_version: 20260727T223147 +stackhpc_pulp_repo_rocky_9_8_appstream_source_version: 20260727T215716 +stackhpc_pulp_repo_rocky_9_8_appstream_version: 20260723T212946 +stackhpc_pulp_repo_rocky_9_8_baseos_aarch64_version: 20260727T223147 +stackhpc_pulp_repo_rocky_9_8_baseos_source_version: 20260727T215716 +stackhpc_pulp_repo_rocky_9_8_baseos_version: 20260724T095616 +stackhpc_pulp_repo_rocky_9_8_crb_aarch64_version: 20260727T223147 +stackhpc_pulp_repo_rocky_9_8_crb_source_version: 20260727T215716 +stackhpc_pulp_repo_rocky_9_8_crb_version: 20260727T212744 stackhpc_pulp_repo_rocky_9_8_extras_aarch64_version: 20260528T223707 stackhpc_pulp_repo_rocky_9_8_extras_source_version: 20260528T220844 stackhpc_pulp_repo_rocky_9_8_extras_version: 20260528T220059 -stackhpc_pulp_repo_rocky_9_8_highavailability_aarch64_version: 20260715T222005 -stackhpc_pulp_repo_rocky_9_8_highavailability_source_version: 20260716T221817 -stackhpc_pulp_repo_rocky_9_8_highavailability_version: 20260715T213338 +stackhpc_pulp_repo_rocky_9_8_highavailability_aarch64_version: 20260723T222833 +stackhpc_pulp_repo_rocky_9_8_highavailability_source_version: 20260723T215614 +stackhpc_pulp_repo_rocky_9_8_highavailability_version: 20260723T212946 stackhpc_pulp_repo_rocky_9_8_security_aarch64_version: 20260605T230645 stackhpc_pulp_repo_rocky_9_8_security_source_version: 20260605T225017 stackhpc_pulp_repo_rocky_9_8_security_version: 20260606T214954 stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20260714T221521 stackhpc_pulp_repo_rocky_9_sig_security_common_source_version: 20260714T215809 stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20260714T213458 -stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20260705T230107 -stackhpc_pulp_repo_ubuntu_noble_security_version: 20260706T010259 -stackhpc_pulp_repo_ubuntu_noble_version: 20260706T010259 +stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20260727T225859 +stackhpc_pulp_repo_ubuntu_noble_security_version: 20260728T011141 +stackhpc_pulp_repo_ubuntu_noble_version: 20260728T011141 From b2cf0cff291bc327baec5862bbb5e5dab72fa377 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Jul 2026 15:11:57 +0200 Subject: [PATCH 13/22] Bump OFED modules --- etc/kayobe/ofed.yml | 4 ++-- etc/kayobe/pulp-repo-versions.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml index 6407446595..057c4797d8 100644 --- a/etc/kayobe/ofed.yml +++ b/etc/kayobe/ofed.yml @@ -19,8 +19,8 @@ stackhpc_pulp_rocky_10_doca_version: "{{ stackhpc_pulp_doca_version_matrix[doca_ stackhpc_doca_kernel_version_matrix: "9.6": 5.14.0.570.21.1.el9.6 "9.7": 5.14.0.611.55.1.el9.7 - "9.8": 5.14.0.687.26.1.el9.8 - "10.2": 6.12.0.211.33.1.el10.2 + "9.8": 5.14.0.687.30.1.el9.8 + "10.2": 6.12.0.211.39.1.el10.2 ############################################################################### # Pulp configuration for DOCA OFED diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index eb762cb377..5a26a8297c 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -32,12 +32,12 @@ stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_aarch64_version: 20260615T122946 stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_version: 20260615T122946 stackhpc_pulp_repo_doca_3_2_2_rhel9_aarch64_version: 20260326T091359 stackhpc_pulp_repo_doca_3_2_2_rhel9_version: 20260520T124124 -stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_aarch64_version: 20260715T185525 -stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_version: 20260715T185525 +stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_aarch64_version: 20260728T112638 +stackhpc_pulp_repo_doca_3_2_3_rhel10_2_modules_version: 20260728T112638 stackhpc_pulp_repo_doca_3_2_3_rhel10_aarch64_version: 20260715T143437 stackhpc_pulp_repo_doca_3_2_3_rhel10_x86_64_version: 20260715T143437 -stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_aarch64_version: 20260720T160316 -stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_version: 20260720T160316 +stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_aarch64_version: 20260728T112638 +stackhpc_pulp_repo_doca_3_2_3_rhel9_8_modules_version: 20260728T112638 stackhpc_pulp_repo_doca_3_2_3_rhel9_aarch64_version: 20260715T143437 stackhpc_pulp_repo_doca_3_2_3_rhel9_version: 20260715T143437 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20260727T204056 From 707d1ecd60aeb90036301dc0641d7f8c31920725 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 28 Jul 2026 15:29:36 +0200 Subject: [PATCH 14/22] Add release notes for repository bump --- .../repo-bump-20260728-6cba5f6d572ebbfb.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 releasenotes/notes/repo-bump-20260728-6cba5f6d572ebbfb.yaml diff --git a/releasenotes/notes/repo-bump-20260728-6cba5f6d572ebbfb.yaml b/releasenotes/notes/repo-bump-20260728-6cba5f6d572ebbfb.yaml new file mode 100644 index 0000000000..e2bb0fd3bd --- /dev/null +++ b/releasenotes/notes/repo-bump-20260728-6cba5f6d572ebbfb.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Updated OFED kernel modules have been built for the latest Rocky Linux 9.8 + (``5.14.0.687.30.1.el9.8``) and 10.2 kernels (``6.12.0.211.39.1.el10.2``). +security: + - | + The latest Rocky Linux 9.8 kernel (``5.14.0.687.30.1.el9.8``) is now + available, which addresses KVM vulnerabilities `CVE-2025-40026 + `__ and + `CVE-2026-63807 `__. + - | + Updates Docker Engine to `release 29.6.2 + `__, which addresses + multiple vulnerabilities. From 9efdf3976268ca185d1ffc46eb8c408de3caee5a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 29 Jul 2026 11:34:10 +0100 Subject: [PATCH 15/22] feat: move release note to correct folder Signed-off-by: Jack Hodgkiss --- .../notes/reset-bls-entries-20375af82dfd3517.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes => releasenotes}/notes/reset-bls-entries-20375af82dfd3517.yaml (100%) diff --git a/etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml b/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml similarity index 100% rename from etc/kayobe/hooks/overcloud-host-package-update/post.d/releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml rename to releasenotes/notes/reset-bls-entries-20375af82dfd3517.yaml From d02d05dd84163618c47ea5708e5389cde49d4a71 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Mon, 27 Jul 2026 10:29:40 +0100 Subject: [PATCH 16/22] Disable upstream sync workflow This workflow should sync SKC with the upstream kayobe-config template. Unfortunately, it has never workedm because it has a different structure to standard openstack forks. Disabling it until we have time to properly investigate it. --- .github/workflows/upstream-sync.yml | 38 ----------------------------- 1 file changed, 38 deletions(-) delete mode 100644 .github/workflows/upstream-sync.yml diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml deleted file mode 100644 index 92e5c1f548..0000000000 --- a/.github/workflows/upstream-sync.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Upstream Sync -'on': - schedule: - - cron: "15 8 * * 1" - workflow_dispatch: -permissions: - contents: write - pull-requests: write -jobs: - synchronise-2023-1: - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - name: Synchronise 2023.1 - uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main - with: - release_series: 2023.1 - upstream: openstack/kayobe-config - synchronise-2024-1: - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - name: Synchronise 2024.1 - uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main - with: - release_series: 2024.1 - upstream: openstack/kayobe-config - synchronise-2025-1: - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - name: Synchronise 2025.1 - uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main - with: - release_series: 2025.1 - upstream: openstack/kayobe-config - synchronise-master: - if: github.repository == 'stackhpc/stackhpc-kayobe-config' - name: Synchronise master - uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main - with: - release_series: master - upstream: openstack/kayobe-config From 80d199b19b4441da96018c1400b810258a1ea7fb Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:39:22 +0000 Subject: [PATCH 17/22] (automated) Bump kolla to stackhpc/20.4.0.11 --- etc/kayobe/stackhpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 2faa6af08e..006f0d2e4b 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -226,7 +226,7 @@ stackhpc_repo_rhel10_doca_modules_version: "{{ stackhpc_repo_distribution }}" # Kolla source repository. stackhpc_kolla_source_url: "https://github.com/stackhpc/kolla" -stackhpc_kolla_source_version: stackhpc/20.4.0.10 +stackhpc_kolla_source_version: stackhpc/20.4.0.11 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" From d0c2a19ba916077c583c0d07bcd69d43c3ff9c84 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:39:26 +0000 Subject: [PATCH 18/22] (automated) Bump kolla-ansible to stackhpc/20.4.0.15 --- etc/kayobe/stackhpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 006f0d2e4b..d54791e77c 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -230,7 +230,7 @@ stackhpc_kolla_source_version: stackhpc/20.4.0.11 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.11 +stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.15 ############################################################################### # Container image registry From 2a59fbb962dffc602e36e9eafb0a769c50d4e495 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:39:28 +0000 Subject: [PATCH 19/22] (automated) Bump kayobe to stackhpc/18.4.0.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f5e5161d9b..bd3948a1b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.4.0.6 +kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.4.0.7 ansible-modules-hashivault>=5.3.0 pulp-glue==0.33.* pulp-glue-deb==0.3.* From e0fa8ce7c80360b0cd27dc48940056ea2091b127 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 4 Aug 2026 15:15:21 +0200 Subject: [PATCH 20/22] Fix reset-bls-entries hook Kayobe expects hooks to end with a .yml extension. --- .../post.d/{20-reset-bls-entries => 20-reset-bls-entries.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etc/kayobe/hooks/overcloud-host-package-update/post.d/{20-reset-bls-entries => 20-reset-bls-entries.yml} (100%) diff --git a/etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries b/etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries.yml similarity index 100% rename from etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries rename to etc/kayobe/hooks/overcloud-host-package-update/post.d/20-reset-bls-entries.yml From 49f57c7fa1c529c198a2d15d4a1f4657f2709ab1 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:42:54 +0000 Subject: [PATCH 21/22] (automated) Bump kolla-ansible to stackhpc/20.4.0.17 --- etc/kayobe/stackhpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index d54791e77c..cf68be5bc5 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -230,7 +230,7 @@ stackhpc_kolla_source_version: stackhpc/20.4.0.11 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.15 +stackhpc_kolla_ansible_source_version: stackhpc/20.4.0.17 ############################################################################### # Container image registry From 2b09258182919b9198ff39d48810754b9a5aab7a Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:42:56 +0000 Subject: [PATCH 22/22] (automated) Bump kayobe to stackhpc/18.4.0.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd3948a1b5..bf1211cbef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.4.0.7 +kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.4.0.9 ansible-modules-hashivault>=5.3.0 pulp-glue==0.33.* pulp-glue-deb==0.3.*