From c635f1c4f7936b4c3a76190b81af86b2960b99ae Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 3 Aug 2026 10:39:19 +0100 Subject: [PATCH 1/6] Make OpenBao raft leader index configurable Added variable ``raft_leader_index`` to the ``secret-store-deploy-overcloud.yml`` playbook. This variable controls which controller becomes the leader when deploying OpenBao. The default is 0 (Frist controller). This variable needs to track the index of current leader controller when fixing/restarting the cluster to prevent split brain. --- .../secret-store/secret-store-deploy-overcloud.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml index aa9a200d1f..4a14990445 100644 --- a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml +++ b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml @@ -23,11 +23,16 @@ vars: secret_store_bind_interface: "{{ internal_net_name | net_interface }}" secret_store_bind_address: "{{ internal_net_name | net_ip }}" - # This is the IP address of the first controller and therefore the leader within - # OpenBao. This could be replaced with the VIP address of the internal network if + # This is the index of the controller with Raft leadership. + # Default value is 0 (First controller becomes OpenBao raft leader). + # The index should follow current leader when restarting/fixing the cluster + # to prevent having two separate clusters (Split brain). + raft_leader_index: 0 + # This is the IP address of the leader within OpenBao. + # This could be replaced with the VIP address of the internal network if # HAProxy has been configured to load balance the OpenBao API. openbao_raft_leaders: - - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}" + - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][raft_leader_index | int]) }}" tasks: - name: Set a fact about the virtualenv on the remote system ansible.builtin.set_fact: From f888b430d17511a4f2fa37b2aabf4bde97a93e1b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 3 Aug 2026 12:32:27 +0100 Subject: [PATCH 2/6] Add playbook for getting current OpenBao Raft leader --- .../secret-store/get-current-raft-leader.yml | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 etc/kayobe/ansible/secret-store/get-current-raft-leader.yml diff --git a/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml b/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml new file mode 100644 index 0000000000..46a19fe518 --- /dev/null +++ b/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml @@ -0,0 +1,68 @@ +--- +- name: Get current Raft leader + hosts: controllers + vars: + secret_store_bind_interface: "{{ internal_net_name | net_interface | replace('-', '_') }}" + secret_store_bind_address: "{{ ansible_facts[secret_store_bind_interface].ipv4.address }}" + secret_store_api_address: "https://{{ secret_store_bind_address }}:8200" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: present + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Query the HA status + ansible.builtin.uri: + url: "{{ secret_store_api_address }}/v1/sys/ha-status" + headers: + X-Vault-Token: "{{ secret_store_keys.root_token }}" + follow_redirects: none + register: ha_status_query + ignore_errors: true + + - name: Query the leader + ansible.builtin.uri: + url: "{{ secret_store_api_address }}/v1/sys/leader" + headers: + X-Vault-Token: "{{ secret_store_keys.root_token }}" + follow_redirects: none + register: leader_query + ignore_errors: true + + - name: Determine Raft leader + ansible.builtin.group_by: + key: raft_leader + when: + - ha_status_query.json is defined + - leader_query.json is defined + - ha_status_query.json.nodes | length > 1 + - leader_query.json.is_self | bool + + - name: Assert that there is only one leader + ansible.builtin.assert: + that: groups['raft_leader'] | default([]) | length == 1 + fail_msg: There are multiple OpenBao Raft leader. Check the status of the cluster. + + - name: Set Raft leader index fact + ansible.builtin.set_fact: + raft_leader_index: "{{ lookup('ansible.utils.index_of', groups['controllers'], 'eq', (groups['raft_leader'] | first)) }}" + + - name: Display the index of the Raft leader + ansible.builtin.debug: + msg: "raft_leader_index: {{ raft_leader_index }}" From 30290d668f6cec015761f5810d27ab8afb01439b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 3 Aug 2026 12:56:31 +0100 Subject: [PATCH 3/6] Add timeout to first OpenBao unseal --- .../ansible/secret-store/secret-store-deploy-overcloud.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml index 4a14990445..18b7a1fa3d 100644 --- a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml +++ b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml @@ -128,6 +128,7 @@ vault_unseal_token: "{{ secret_store_keys.root_token }}" vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + vault_unseal_timeout: 10 environment: https_proxy: "" run_once: true From f703b5cafc049593f072ca377d38f4203abff89a Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 4 Aug 2026 12:51:40 +0100 Subject: [PATCH 4/6] Add wrapper playbook for fixing overcloud OpenBao --- etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml diff --git a/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml b/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml new file mode 100644 index 0000000000..91bb59467d --- /dev/null +++ b/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml @@ -0,0 +1,6 @@ +--- +- name: Get current OpenBao Raft leader + import_playbook: get-current-raft-leader.yml + +- name: Run deploy playbook with the current raft_leader_index set + import_playbook: secret-store-deploy-overcloud.yml From a10c29f2512bdab9ab98c25bb91035c54457e48b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 3 Aug 2026 13:22:14 +0100 Subject: [PATCH 5/6] Add docs about restoring openbao raft cluster --- doc/source/configuration/openbao.rst | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst index 0248f8bd69..dae91bf3e2 100644 --- a/doc/source/configuration/openbao.rst +++ b/doc/source/configuration/openbao.rst @@ -604,3 +604,60 @@ However, end users of OpenStack will not be affected. Overcloud migration is HA migration and no downtime is expected. It is recommended to run ``vault-bao-migration-change-config.yml`` after all Vault deployments have been migrated to OpenBao. + +.. _openbao-cluster-fix: + +Restoring OpenBao cluster +========================= + +Users can use ``fix-openbao-overcloud.yml`` playbook to restore OpenBao cluster. + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/fix-openbao-overcloud.yml + +.. warning:: + + The playbook ``secret-store-deploy-overcloud.yml`` assumes the first + controller to always be the leader node. + **DO NOT** run ``secret-store-deploy-overcloud.yml`` alone to fix the + cluster. + If the leader OpenBao node (First controller by default) fails, the leader + role gets transferred to one of the other controllers. + Without checking which node became the new leader, there is a risk of having + two separate clusters as a result. + +The ``fix-openbao-overcloud.yml`` playbook runs two playbooks + +1. ``get-current-raft-leader.yml`` +2. ``secret-store-deploy-overcloud.yml`` + +Users can also follow this procedure to fix the OpenBao cluster manually. + +1. Use ``get-current-raft-leader.yml`` playbook to get the index of the leader + controller. + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/get-current-raft-leader.yml + + The last task "Display the index of the Raft leader" will show the index of + the new leader controller in the controllers ansible group. + + .. code-block:: bash + + TASK [Display the index of the Raft leader] *********************************** + Monday 03 August 2026 12:15:02 +0000 (0:00:00.148) 0:00:07.797 ********* + ok: [controller-01] => + msg: 'raft_leader_index: 2' + ok: [controller-02] => + msg: 'raft_leader_index: 2' + ok: [controller-03] => + msg: 'raft_leader_index: 2' + +2. Run ``secret-store-deploy-overcloud.yml`` playbook with the index of the new + leader as an extra variable. + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/secret-store-deploy-overcloud.yml -e raft_leader_index=2 From d21af912b5dfa7981b114ed5c60f10b46592653c Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 11 Aug 2026 17:42:42 +0100 Subject: [PATCH 6/6] Prevent running init on wrong node --- .../ansible/secret-store/secret-store-deploy-overcloud.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml index 18b7a1fa3d..86cd554337 100644 --- a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml +++ b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml @@ -31,8 +31,7 @@ # This is the IP address of the leader within OpenBao. # This could be replaced with the VIP address of the internal network if # HAProxy has been configured to load balance the OpenBao API. - openbao_raft_leaders: - - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][raft_leader_index | int]) }}" + raft_leader_address: "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][raft_leader_index | int]) }}" tasks: - name: Set a fact about the virtualenv on the remote system ansible.builtin.set_fact: @@ -103,6 +102,9 @@ openbao_registry_password: "{{ overcloud_openbao_registry_password }}" openbao_config_dir: /opt/kayobe/openbao openbao_cluster_name: overcloud + openbao_raft_leaders: + - "{{ raft_leader_address }}" + openbao_init_address: "{{ raft_leader_address }}" openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" openbao_docker_image: "{{ overcloud_openbao_docker_image }}" openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}" @@ -132,6 +134,7 @@ environment: https_proxy: "" run_once: true + delegate_to: "{{ groups['controllers'][raft_leader_index | int] }}" # As the first instance is now unsealed the other instances will now need some # time to connect before we can proceed.