From 8039f1870a66a50a182b47d5cf26dcfce31f7c6d Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Thu, 5 Dec 2024 16:57:09 +0000 Subject: [PATCH 1/8] Staggered upgrade procedure --- doc/source/operations/upgrading-openstack.rst | 65 ++++++++++++++ etc/kayobe/ansible/neutron-l3-drain.yml | 23 +++++ .../neutron-namespace-drain/defaults/main.yml | 8 ++ .../tasks/add-dhcp.yml | 51 +++++++++++ .../tasks/add-new-l3.yml | 43 +++++++++ .../tasks/drain-dhcp.yml | 87 ++++++++++++++++++ .../tasks/drain-l3.yml | 90 +++++++++++++++++++ .../tasks/enable-l3.yml | 24 +++++ .../neutron-namespace-drain/tasks/main.yml | 19 ++++ .../neutron-namespace-drain/tasks/setup.yml | 31 +++++++ 10 files changed, 441 insertions(+) create mode 100644 etc/kayobe/ansible/neutron-l3-drain.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml create mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index 9923f3cc2..998d6f8c5 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1063,6 +1063,12 @@ This will block the upgrade, but may be overridden by setting ``etc/kayobe/kolla/globals.yml`` or ``etc/kayobe/environments//kolla/globals.yml``. +Depending on the networking architecture of your cloud, the steps used +to upgrade the containerised services will differ. + +OVN +^^^ + To upgrade the containerised control plane services: .. code-block:: console @@ -1076,6 +1082,65 @@ scope of the upgrade: kayobe overcloud service upgrade --tags config --kolla-tags keystone +OVS (w/ Dedicated network nodes) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You should first stop the Octavia health manager to prevent alerts during +the service upgrade. + +.. code-block:: console + + kayobe overcloud host command run --command "docker stop octavia_health_manager" --limit controllers --become + +Upgrade the control plane services + +.. code-block:: console + + kayobe overcloud serivce upgrade --kolla-limit controllers + +To ensure L3 reliability during the upgrade, we will need to manually drain +the network nodes of all agents, and upgrade the nodes sequentially. + +Kolla credentials will need to be activated before running the neutron-namespace-drain +role. + +.. code-block:: console + + source $KOLLA_CONFIG_PATH/public-openrc.sh + +You should substitute with the first network node to be drained, To set +the node for maintenance and begin draining the agents: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_drain_host= -e maintenance=true -e drain_dhcp_agents=true + +You can monitor the L3/DHCP agents being drained from the node by running: + +.. code-block:: console + + ssh -t watch ip netns ls + +Once all agents have been drained, you can upgrade the containerised services +on the network node. + +.. code-block:: console + + kayobe overcloud service upgrade --kolla-limit + +Following the service upgrade, the agents can be restored on the node by disabling maintenance: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_drain_host= -e maintenance=false -e drain_dhcp_agents=true + +The above steps should be repeated for the remaining network nodes, once all network nodes have been upgraded +the remaining containerised services can be upgraded: + +.. code-block:: console + + kayobe overcloud service upgrade --kolla-tags common,nova,prometheus,openvswitch,neutron --skip-prechecks -kl controllers,compute --limit controllers,compute + Updating the Octavia Amphora Image ---------------------------------- diff --git a/etc/kayobe/ansible/neutron-l3-drain.yml b/etc/kayobe/ansible/neutron-l3-drain.yml new file mode 100644 index 000000000..7640a6920 --- /dev/null +++ b/etc/kayobe/ansible/neutron-l3-drain.yml @@ -0,0 +1,23 @@ +--- +- name: Drain neutron of l3 agents and dhcp agents + hosts: localhost + gather_facts: true + tags: + - neutron-l3-drain + vars: + maintenance: false + drain_ctrl1: false + drain_ctrl2: false + drain_ctrl3: false + tasks: + - import_role: + name: neutron-namespace-drain + tasks_from: main.yml + when: drain_ctrl1 | bool or drain_ctrl2 | bool or drain_ctrl3 | bool or neutron_drain_host is defined + - name: "Print Info" + debug: + msg: + - "{{ neutron_drain_host }} is read for maintenance" + - "rerun this play book with -e maintenance=false to re-add" + - "routers" + when: maintenance | bool diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml new file mode 100644 index 000000000..2d67ee960 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml @@ -0,0 +1,8 @@ +--- +neutron_drain_venv: "{{ virtualenv_path }}/openstack" +neutron_drain_host: "{% if drain_ctrl1 | bool %}{{ groups['controllers'][0] }}{% elif drain_ctrl2 | bool %}{{ groups['controllers'][1] }}{% elif drain_ctrl3 | bool %}{{ groups['controllers'][2] }}{% endif %}" +neutron_drain_venv_python: "{{ 'python' ~ ansible_facts.python.version.major ~ '.' ~ ansible_facts.python.version.minor }}" +drain_ctrl1: false +drain_ctrl2: false +drain_ctrl3: false +drain_dhcp_agents: false diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml new file mode 100644 index 000000000..ed58159e8 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml @@ -0,0 +1,51 @@ +--- +- name: Query source SRC_DHCP_ID + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --host {{ neutron_drain_host }} + --agent-type dhcp -f value -c ID + register: SRC_DHCP_ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set fact containing SRC_DHCP_ID + set_fact: + DHCP_SRC_ID: "{{ SRC_DHCP_ID.stdout }}" + +- name: Enable DHCP agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent set "{{ DHCP_SRC_ID }}" --enable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Get Network IDs + command: > + {{ neutron_drain_venv }}/bin/openstack + network list -f value -c ID + register: network_ids + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set Network IDs + set_fact: + NETWORK_IDS: "{{ network_ids.stdout_lines }}" + +- name: Add DHCP agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent add network {{ DHCP_SRC_ID }} {{ item }} --dhcp + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + with_items: + - "{{ NETWORK_IDS }}" + loop_control: + pause: 10 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml new file mode 100644 index 000000000..0e5fd4f17 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml @@ -0,0 +1,43 @@ +--- +- name: Query L3_IDs + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --agent-type l3 -f value -c ID + register: L3_ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set fact containing SRC_L3_IDs + set_fact: + L3_IDS: "{{ L3_ID.stdout_lines }}" + +- name: Get agents for each router + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --router {{ router_id }} --agent-type l3 -f value -c ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + failed_when: false + register: ROUTER_L3_IDS + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + loop: "{{ ROUTER_IDS }}" + loop_control: + loop_var: router_id + +- name: Add agent to router + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent add router --l3 {{ L3_ADD }} {{ item.router_id }} + loop: "{{ ROUTER_L3_IDS.results }}" + loop_control: + label: "{{ item.router_id }}" + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + L3_ADD_DIFF: "{{ L3_IDS | difference([L3_SRC_ID]) | difference(item.stdout_lines) }}" + L3_ADD: "{{ L3_ADD_DIFF[:1] | first }}" + when: L3_ADD_DIFF | length > 0 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml new file mode 100644 index 000000000..ba28edc57 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml @@ -0,0 +1,87 @@ +--- +- name: Query source SRC_DHCP_ID + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --host {{ neutron_drain_host }} + --agent-type dhcp -f value -c ID + register: SRC_DHCP_ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set fact containing SRC_DHCP_ID + set_fact: + DHCP_SRC_ID: "{{ SRC_DHCP_ID.stdout }}" + +- name: Get DHCP agent network IDs + command: > + {{ neutron_drain_venv }}/bin/openstack + network list --agent {{ DHCP_SRC_ID }} -f value -c ID + register: dhcp_agent_ids + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set DHCP agent network IDs + set_fact: + DHCP_AGENT_IDS: "{{ dhcp_agent_ids.stdout_lines }}" + +- name: Remove DHCP agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent remove network {{ DHCP_SRC_ID }} {{ item }} --dhcp + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + with_items: + - "{{ DHCP_AGENT_IDS }}" + +- name: Wait for no more dhcp agents to be attached to the host + block: + - name: Retry count + set_fact: + retry_count: "{{ 0 if retry_count is undefined or retry_count == 'reset' else retry_count | int + 1 }}" + max_retries: 20 + + - name: Verify dhcp agents exist + command: > + {{ neutron_drain_venv }}/bin/openstack + network list --agent {{ DHCP_SRC_ID }} -f value -c ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + register: agent_status + + - name: Fail if DHCP agent still attached + fail: + msg: a DHCP agent is still attached to "{{ L3_SRC_ID }}" + when: agent_status.stdout | length > 0 + + - name: Reset retry count after success + set_fact: + retry_count: reset + rescue: + - fail: + msg: | + Maximum retries waiting for DHCP agents to be detached reached + when: retry_count | int == max_retries + + - name: Reset retry counter if max retries reached (exit loop) + set_fact: + retry_count: reset + failed_when: retry_count == 'reset' + when: retry_count | int >= max_retries | int + +- name: Disable DHCP agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent set "{{ DHCP_SRC_ID }}" --disable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + when: agent_status.stdout | length == 0 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml new file mode 100644 index 000000000..eebd99d60 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml @@ -0,0 +1,90 @@ +--- +- name: Query source SRC_L3_ID + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --host {{ neutron_drain_host }} + --agent-type l3 -f value -c ID + register: SRC_L3_ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set fact containing SRC_L3_ID + set_fact: + L3_SRC_ID: "{{ SRC_L3_ID.stdout }}" + +- name: Get Router IDs + command: > + {{ neutron_drain_venv }}/bin/openstack + router list --agent {{ L3_SRC_ID }} -f value -c ID + register: router_ids + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set Router IDs + set_fact: + ROUTER_IDS: "{{ router_ids.stdout_lines }}" + +- name: Add agents to router + include_tasks: add-new-l3.yml + when: ROUTER_IDS | length > 0 + +- name: Remove router + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent remove router {{ L3_SRC_ID }} {{ item }} --l3 + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + with_items: + - "{{ ROUTER_IDS }}" + +- name: Wait for no more routers to be attached to the host + block: + - name: Retry count + set_fact: + retry_count: "{{ 0 if retry_count is undefined or retry_count == 'reset' else retry_count | int + 1 }}" + max_retries: 20 + + - name: Verify routers exist + command: > + {{ neutron_drain_venv }}/bin/openstack router list --agent {{ L3_SRC_ID }} -f value -c ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + register: agent_status + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + + - name: Fail if routers still attached + fail: + msg: a Router is still attached to agent "{{ L3_SRC_ID }}" + when: agent_status.stdout | length > 0 + + - name: Reset retry count after success + set_fact: + retry_count: reset + rescue: + - fail: + msg: | + Maximum retries waiting for routers to be detached reached + when: retry_count | int == max_retries + + - name: Reset retry counter if max retries reached (exit loop) + set_fact: + retry_count: reset + failed_when: retry_count == 'reset' + when: retry_count | int >= max_retries | int + +- name: Disable L3 agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent set "{{ L3_SRC_ID }}" --disable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + when: agent_status.stdout | length == 0 + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml new file mode 100644 index 000000000..e6679c40b --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml @@ -0,0 +1,24 @@ +--- +- name: Query source SRC_L3_ID + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent list --host {{ neutron_drain_host }} + --agent-type l3 -f value -c ID + register: SRC_L3_ID + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" + +- name: Set fact containing SRC_L3_ID + set_fact: + L3_SRC_ID: "{{ SRC_L3_ID.stdout }}" + +- name: Enable L3 agent + command: > + {{ neutron_drain_venv }}/bin/openstack + network agent set "{{ L3_SRC_ID }}" --enable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ neutron_drain_host }}" + vars: + ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml new file mode 100644 index 000000000..c33f72731 --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: "Setup OpenStack venv" + import_tasks: setup.yml + +- name: "Drain L3 agents" + import_tasks: drain-l3.yml + when: maintenance | bool + +- name: "Enable L3 agent" + import_tasks: enable-l3.yml + when: not maintenance | bool + +- name: "Drain DHCP agents" + import_tasks: drain-dhcp.yml + when: maintenance | bool and drain_dhcp_agents | bool + +- name: "Add DHCP agents" + import_tasks: add-dhcp.yml + when: not maintenance | bool and drain_dhcp_agents | bool diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml new file mode 100644 index 000000000..909c4959f --- /dev/null +++ b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml @@ -0,0 +1,31 @@ +--- + +- name: Ensure the latest version of pip is installed + pip: + name: + - pip + state: latest + virtualenv: "{{ neutron_drain_venv }}" + virtualenv_command: "{{ neutron_drain_venv_python }} -m venv" + run_once: true + become: true + delegate_to: "{{ neutron_drain_host }}" + vars: + # NOTE: Without this, the delegate ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ neutron_drain_host }}" + +- name: Set up openstack cli virtualenv + pip: + virtualenv: "{{ neutron_drain_venv }}" + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + become: true + delegate_to: "{{ neutron_drain_host }}" + vars: + # NOTE: Without this, the delegate ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ neutron_drain_host }}" From 22e1a846ddc65aecae59eb25f1cd9fee4d77ca32 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 5 Feb 2025 11:49:34 +0000 Subject: [PATCH 2/8] Converged network node support --- doc/source/operations/upgrading-openstack.rst | 26 ++++-- etc/kayobe/ansible/neutron-l3-drain.yml | 12 +-- etc/kayobe/ansible/requirements.yml | 3 + .../neutron-namespace-drain/defaults/main.yml | 8 -- .../tasks/add-dhcp.yml | 51 ----------- .../tasks/add-new-l3.yml | 43 --------- .../tasks/drain-dhcp.yml | 87 ------------------ .../tasks/drain-l3.yml | 90 ------------------- .../tasks/enable-l3.yml | 24 ----- .../neutron-namespace-drain/tasks/main.yml | 19 ---- .../neutron-namespace-drain/tasks/setup.yml | 31 ------- 11 files changed, 29 insertions(+), 365 deletions(-) delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml delete mode 100644 etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index 998d6f8c5..a85b3f543 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1082,8 +1082,8 @@ scope of the upgrade: kayobe overcloud service upgrade --tags config --kolla-tags keystone -OVS (w/ Dedicated network nodes) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +OVS +^^^ You should first stop the Octavia health manager to prevent alerts during the service upgrade. @@ -1092,12 +1092,19 @@ the service upgrade. kayobe overcloud host command run --command "docker stop octavia_health_manager" --limit controllers --become -Upgrade the control plane services +For dedicated network nodes, upgrade the control plane services: .. code-block:: console kayobe overcloud serivce upgrade --kolla-limit controllers +For converged network nodes, you should specify the service limit to only +upgrade the Neutron API service. + +.. code-block:: console + + kayobe overcloud serivce upgrade --kolla-limit controllers -ke neutron_service_limit=neutron-server + To ensure L3 reliability during the upgrade, we will need to manually drain the network nodes of all agents, and upgrade the nodes sequentially. @@ -1113,7 +1120,7 @@ the node for maintenance and begin draining the agents: .. code-block:: console - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_drain_host= -e maintenance=true -e drain_dhcp_agents=true + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_namespace_drain_host= -e maintenance=true -e neutron_namespace_drain_dhcp_agents=true You can monitor the L3/DHCP agents being drained from the node by running: @@ -1122,17 +1129,24 @@ You can monitor the L3/DHCP agents being drained from the node by running: ssh -t watch ip netns ls Once all agents have been drained, you can upgrade the containerised services -on the network node. +on the network node. For dedicated network nodes run: .. code-block:: console kayobe overcloud service upgrade --kolla-limit +Converged network nodes will require specifying the the service limit for the Neutron +agents. + +.. code-block:: console + + kayobe overcloud service upgrade --kolla-limit -ke neutron_service_limit='neutron-openvswitch-agent,neutron-dhcp-agent,neutron-l3-agent,neutron-metadata-agent,ironic-neutron-agent' + Following the service upgrade, the agents can be restored on the node by disabling maintenance: .. code-block:: console - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_drain_host= -e maintenance=false -e drain_dhcp_agents=true + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_namespace_drain_host= -e maintenance=false -e neutron_namespace_drain_dhcp_agents=true The above steps should be repeated for the remaining network nodes, once all network nodes have been upgraded the remaining containerised services can be upgraded: diff --git a/etc/kayobe/ansible/neutron-l3-drain.yml b/etc/kayobe/ansible/neutron-l3-drain.yml index 7640a6920..d9f261d17 100644 --- a/etc/kayobe/ansible/neutron-l3-drain.yml +++ b/etc/kayobe/ansible/neutron-l3-drain.yml @@ -6,18 +6,18 @@ - neutron-l3-drain vars: maintenance: false - drain_ctrl1: false - drain_ctrl2: false - drain_ctrl3: false + neutron_namespace_drain_ctrl1: false + neutron_namespace_drain_ctrl2: false + neutron_namespace_drain_ctrl3: false tasks: - import_role: - name: neutron-namespace-drain + name: stackhpc.openstack_ops.neutron_namespace_drain tasks_from: main.yml - when: drain_ctrl1 | bool or drain_ctrl2 | bool or drain_ctrl3 | bool or neutron_drain_host is defined + when: neutron_namespace_drain_ctrl1 | bool or neutron_namespace_drain_ctrl2 | bool or neutron_namespace_drain_ctrl3 | bool or neutron_namespace_drain_host is defined - name: "Print Info" debug: msg: - - "{{ neutron_drain_host }} is read for maintenance" + - "{{ neutron_namespace_drain_host }} is read for maintenance" - "rerun this play book with -e maintenance=false to re-add" - "routers" when: maintenance | bool diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 72d083a61..fb5012bf5 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -12,6 +12,9 @@ collections: version: 2.5.1 - name: stackhpc.kayobe_workflows version: 1.1.0 + - name: https://github.com/stackhpc/ansible-collection-openstack-ops + type: git + version: feature/neutron-namespace-drain roles: - src: stackhpc.vxlan - name: ansible-lockdown.ubuntu22_cis diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml deleted file mode 100644 index 2d67ee960..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/defaults/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -neutron_drain_venv: "{{ virtualenv_path }}/openstack" -neutron_drain_host: "{% if drain_ctrl1 | bool %}{{ groups['controllers'][0] }}{% elif drain_ctrl2 | bool %}{{ groups['controllers'][1] }}{% elif drain_ctrl3 | bool %}{{ groups['controllers'][2] }}{% endif %}" -neutron_drain_venv_python: "{{ 'python' ~ ansible_facts.python.version.major ~ '.' ~ ansible_facts.python.version.minor }}" -drain_ctrl1: false -drain_ctrl2: false -drain_ctrl3: false -drain_dhcp_agents: false diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml deleted file mode 100644 index ed58159e8..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-dhcp.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- -- name: Query source SRC_DHCP_ID - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --host {{ neutron_drain_host }} - --agent-type dhcp -f value -c ID - register: SRC_DHCP_ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set fact containing SRC_DHCP_ID - set_fact: - DHCP_SRC_ID: "{{ SRC_DHCP_ID.stdout }}" - -- name: Enable DHCP agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent set "{{ DHCP_SRC_ID }}" --enable - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Get Network IDs - command: > - {{ neutron_drain_venv }}/bin/openstack - network list -f value -c ID - register: network_ids - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set Network IDs - set_fact: - NETWORK_IDS: "{{ network_ids.stdout_lines }}" - -- name: Add DHCP agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent add network {{ DHCP_SRC_ID }} {{ item }} --dhcp - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - with_items: - - "{{ NETWORK_IDS }}" - loop_control: - pause: 10 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml deleted file mode 100644 index 0e5fd4f17..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/add-new-l3.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Query L3_IDs - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --agent-type l3 -f value -c ID - register: L3_ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set fact containing SRC_L3_IDs - set_fact: - L3_IDS: "{{ L3_ID.stdout_lines }}" - -- name: Get agents for each router - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --router {{ router_id }} --agent-type l3 -f value -c ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - failed_when: false - register: ROUTER_L3_IDS - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - loop: "{{ ROUTER_IDS }}" - loop_control: - loop_var: router_id - -- name: Add agent to router - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent add router --l3 {{ L3_ADD }} {{ item.router_id }} - loop: "{{ ROUTER_L3_IDS.results }}" - loop_control: - label: "{{ item.router_id }}" - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - L3_ADD_DIFF: "{{ L3_IDS | difference([L3_SRC_ID]) | difference(item.stdout_lines) }}" - L3_ADD: "{{ L3_ADD_DIFF[:1] | first }}" - when: L3_ADD_DIFF | length > 0 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml deleted file mode 100644 index ba28edc57..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-dhcp.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -- name: Query source SRC_DHCP_ID - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --host {{ neutron_drain_host }} - --agent-type dhcp -f value -c ID - register: SRC_DHCP_ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set fact containing SRC_DHCP_ID - set_fact: - DHCP_SRC_ID: "{{ SRC_DHCP_ID.stdout }}" - -- name: Get DHCP agent network IDs - command: > - {{ neutron_drain_venv }}/bin/openstack - network list --agent {{ DHCP_SRC_ID }} -f value -c ID - register: dhcp_agent_ids - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set DHCP agent network IDs - set_fact: - DHCP_AGENT_IDS: "{{ dhcp_agent_ids.stdout_lines }}" - -- name: Remove DHCP agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent remove network {{ DHCP_SRC_ID }} {{ item }} --dhcp - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - with_items: - - "{{ DHCP_AGENT_IDS }}" - -- name: Wait for no more dhcp agents to be attached to the host - block: - - name: Retry count - set_fact: - retry_count: "{{ 0 if retry_count is undefined or retry_count == 'reset' else retry_count | int + 1 }}" - max_retries: 20 - - - name: Verify dhcp agents exist - command: > - {{ neutron_drain_venv }}/bin/openstack - network list --agent {{ DHCP_SRC_ID }} -f value -c ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - register: agent_status - - - name: Fail if DHCP agent still attached - fail: - msg: a DHCP agent is still attached to "{{ L3_SRC_ID }}" - when: agent_status.stdout | length > 0 - - - name: Reset retry count after success - set_fact: - retry_count: reset - rescue: - - fail: - msg: | - Maximum retries waiting for DHCP agents to be detached reached - when: retry_count | int == max_retries - - - name: Reset retry counter if max retries reached (exit loop) - set_fact: - retry_count: reset - failed_when: retry_count == 'reset' - when: retry_count | int >= max_retries | int - -- name: Disable DHCP agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent set "{{ DHCP_SRC_ID }}" --disable - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - when: agent_status.stdout | length == 0 diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml deleted file mode 100644 index eebd99d60..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/drain-l3.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -- name: Query source SRC_L3_ID - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --host {{ neutron_drain_host }} - --agent-type l3 -f value -c ID - register: SRC_L3_ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set fact containing SRC_L3_ID - set_fact: - L3_SRC_ID: "{{ SRC_L3_ID.stdout }}" - -- name: Get Router IDs - command: > - {{ neutron_drain_venv }}/bin/openstack - router list --agent {{ L3_SRC_ID }} -f value -c ID - register: router_ids - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set Router IDs - set_fact: - ROUTER_IDS: "{{ router_ids.stdout_lines }}" - -- name: Add agents to router - include_tasks: add-new-l3.yml - when: ROUTER_IDS | length > 0 - -- name: Remove router - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent remove router {{ L3_SRC_ID }} {{ item }} --l3 - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - with_items: - - "{{ ROUTER_IDS }}" - -- name: Wait for no more routers to be attached to the host - block: - - name: Retry count - set_fact: - retry_count: "{{ 0 if retry_count is undefined or retry_count == 'reset' else retry_count | int + 1 }}" - max_retries: 20 - - - name: Verify routers exist - command: > - {{ neutron_drain_venv }}/bin/openstack router list --agent {{ L3_SRC_ID }} -f value -c ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - register: agent_status - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - - - name: Fail if routers still attached - fail: - msg: a Router is still attached to agent "{{ L3_SRC_ID }}" - when: agent_status.stdout | length > 0 - - - name: Reset retry count after success - set_fact: - retry_count: reset - rescue: - - fail: - msg: | - Maximum retries waiting for routers to be detached reached - when: retry_count | int == max_retries - - - name: Reset retry counter if max retries reached (exit loop) - set_fact: - retry_count: reset - failed_when: retry_count == 'reset' - when: retry_count | int >= max_retries | int - -- name: Disable L3 agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent set "{{ L3_SRC_ID }}" --disable - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - when: agent_status.stdout | length == 0 - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml deleted file mode 100644 index e6679c40b..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/enable-l3.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Query source SRC_L3_ID - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent list --host {{ neutron_drain_host }} - --agent-type l3 -f value -c ID - register: SRC_L3_ID - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" - -- name: Set fact containing SRC_L3_ID - set_fact: - L3_SRC_ID: "{{ SRC_L3_ID.stdout }}" - -- name: Enable L3 agent - command: > - {{ neutron_drain_venv }}/bin/openstack - network agent set "{{ L3_SRC_ID }}" --enable - environment: "{{ openstack_auth_env }}" - delegate_to: "{{ neutron_drain_host }}" - vars: - ansible_host: "{{ hostvars[neutron_drain_host].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml deleted file mode 100644 index c33f72731..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: "Setup OpenStack venv" - import_tasks: setup.yml - -- name: "Drain L3 agents" - import_tasks: drain-l3.yml - when: maintenance | bool - -- name: "Enable L3 agent" - import_tasks: enable-l3.yml - when: not maintenance | bool - -- name: "Drain DHCP agents" - import_tasks: drain-dhcp.yml - when: maintenance | bool and drain_dhcp_agents | bool - -- name: "Add DHCP agents" - import_tasks: add-dhcp.yml - when: not maintenance | bool and drain_dhcp_agents | bool diff --git a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml deleted file mode 100644 index 909c4959f..000000000 --- a/etc/kayobe/ansible/roles/neutron-namespace-drain/tasks/setup.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Ensure the latest version of pip is installed - pip: - name: - - pip - state: latest - virtualenv: "{{ neutron_drain_venv }}" - virtualenv_command: "{{ neutron_drain_venv_python }} -m venv" - run_once: true - become: true - delegate_to: "{{ neutron_drain_host }}" - vars: - # NOTE: Without this, the delegate ansible_host variable will not - # be respected when using delegate_to. - ansible_host: "{{ neutron_drain_host }}" - -- name: Set up openstack cli virtualenv - pip: - virtualenv: "{{ neutron_drain_venv }}" - name: - - python-openstackclient - state: latest - extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" - run_once: true - become: true - delegate_to: "{{ neutron_drain_host }}" - vars: - # NOTE: Without this, the delegate ansible_host variable will not - # be respected when using delegate_to. - ansible_host: "{{ neutron_drain_host }}" From 13d0fb1cb6cc0db50870e993877d692410e1df90 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 5 Feb 2025 12:19:59 +0000 Subject: [PATCH 3/8] Fix linting --- etc/kayobe/ansible/neutron-l3-drain.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/neutron-l3-drain.yml b/etc/kayobe/ansible/neutron-l3-drain.yml index d9f261d17..4b8c71280 100644 --- a/etc/kayobe/ansible/neutron-l3-drain.yml +++ b/etc/kayobe/ansible/neutron-l3-drain.yml @@ -10,12 +10,14 @@ neutron_namespace_drain_ctrl2: false neutron_namespace_drain_ctrl3: false tasks: - - import_role: + - name: Drain hosts + ansible.builtin.import_role: name: stackhpc.openstack_ops.neutron_namespace_drain tasks_from: main.yml when: neutron_namespace_drain_ctrl1 | bool or neutron_namespace_drain_ctrl2 | bool or neutron_namespace_drain_ctrl3 | bool or neutron_namespace_drain_host is defined - - name: "Print Info" - debug: + + - name: Print info + ansible.builtin.debug: msg: - "{{ neutron_namespace_drain_host }} is read for maintenance" - "rerun this play book with -e maintenance=false to re-add" From 51647498f26bf007f01e97c66447a29b1df62807 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 5 Feb 2025 14:07:51 +0000 Subject: [PATCH 4/8] Fix whitespace --- doc/source/operations/upgrading-openstack.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index a85b3f543..032d2459e 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1125,7 +1125,7 @@ the node for maintenance and begin draining the agents: You can monitor the L3/DHCP agents being drained from the node by running: .. code-block:: console - + ssh -t watch ip netns ls Once all agents have been drained, you can upgrade the containerised services @@ -1143,7 +1143,7 @@ agents. kayobe overcloud service upgrade --kolla-limit -ke neutron_service_limit='neutron-openvswitch-agent,neutron-dhcp-agent,neutron-l3-agent,neutron-metadata-agent,ironic-neutron-agent' Following the service upgrade, the agents can be restored on the node by disabling maintenance: - + .. code-block:: console kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/neutron-l3-drain.yml -e neutron_namespace_drain_host= -e maintenance=false -e neutron_namespace_drain_dhcp_agents=true From 1248d7c3477cd88b883f235e3904ade2264852cd Mon Sep 17 00:00:00 2001 From: Jake Hutchinson <39007539+assumptionsandg@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:56:06 +0000 Subject: [PATCH 5/8] Update doc/source/operations/upgrading-openstack.rst Co-authored-by: Matt Anson --- doc/source/operations/upgrading-openstack.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index 032d2459e..a00f22abe 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1096,7 +1096,7 @@ For dedicated network nodes, upgrade the control plane services: .. code-block:: console - kayobe overcloud serivce upgrade --kolla-limit controllers + kayobe overcloud service upgrade --kolla-limit controllers For converged network nodes, you should specify the service limit to only upgrade the Neutron API service. From c69e8cb6bed75d91c5503ef7c27cb68a1290aa3e Mon Sep 17 00:00:00 2001 From: Jake Hutchinson <39007539+assumptionsandg@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:56:17 +0000 Subject: [PATCH 6/8] Update doc/source/operations/upgrading-openstack.rst Co-authored-by: Matt Anson --- doc/source/operations/upgrading-openstack.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index a00f22abe..5510e629e 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1103,7 +1103,7 @@ upgrade the Neutron API service. .. code-block:: console - kayobe overcloud serivce upgrade --kolla-limit controllers -ke neutron_service_limit=neutron-server + kayobe overcloud service upgrade --kolla-limit controllers -ke neutron_service_limit=neutron-server To ensure L3 reliability during the upgrade, we will need to manually drain the network nodes of all agents, and upgrade the nodes sequentially. From 65fbc5c3485566e04665e560903a6e27a0eb12a5 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson <39007539+assumptionsandg@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:56:30 +0000 Subject: [PATCH 7/8] Update doc/source/operations/upgrading-openstack.rst Co-authored-by: Matt Anson --- doc/source/operations/upgrading-openstack.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index 5510e629e..1f3d123f0 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1105,7 +1105,8 @@ upgrade the Neutron API service. kayobe overcloud service upgrade --kolla-limit controllers -ke neutron_service_limit=neutron-server -To ensure L3 reliability during the upgrade, we will need to manually drain +To ensure L3 reliability during the upgrade, we will need to sequentially drain +and upgrade each network node by first disabling agents and then running a targeted upgrade. the network nodes of all agents, and upgrade the nodes sequentially. Kolla credentials will need to be activated before running the neutron-namespace-drain From 907ff358d54e6e26cde8f4dda004343f19591ed7 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Tue, 25 Feb 2025 14:25:52 +0000 Subject: [PATCH 8/8] Kolla tag neutron for faster upgrade --- doc/source/operations/upgrading-openstack.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst index 032d2459e..ce78e91b2 100644 --- a/doc/source/operations/upgrading-openstack.rst +++ b/doc/source/operations/upgrading-openstack.rst @@ -1140,7 +1140,7 @@ agents. .. code-block:: console - kayobe overcloud service upgrade --kolla-limit -ke neutron_service_limit='neutron-openvswitch-agent,neutron-dhcp-agent,neutron-l3-agent,neutron-metadata-agent,ironic-neutron-agent' + kayobe overcloud service upgrade --kolla-limit -kt neutron -ke neutron_service_limit='neutron-openvswitch-agent,neutron-dhcp-agent,neutron-l3-agent,neutron-metadata-agent,ironic-neutron-agent' Following the service upgrade, the agents can be restored on the node by disabling maintenance: