Skip to content

US1b - Add role for updating single-node systems. #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Mar 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelogs/fragments/version_update_single_node_role.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
major_changes:
- Added a role for updating single-node systems.
(https://github.com/ScaleComputing/HyperCoreAnsibleCollection/pull/136)
17 changes: 17 additions & 0 deletions examples/version_update_single_node.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
- name: Update a HyperCore single-node system
hosts: localhost
connection: local
gather_facts: false
# Comment out check_mode=true if you really want to start update.
check_mode: true

vars:
desired_version: 9.1.23.210897

tasks:
- name: Update HyperCore single-node system to a desired version
include_role:
name: scale_computing.hypercore.version_update_single_node
vars:
scale_computing_hypercore_desired_version: "{{ desired_version }}"
14 changes: 14 additions & 0 deletions roles/version_update_single_node/meta/argument_specs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
argument_specs:
main:
short_description: Update single-node systems
description:
- Role version_update_single_node can be use to to update a single-node HyperCore system to a desired HyperCore version.
options:
scale_computing_hypercore_desired_version:
description:
- The desired HyperCore version we wish to update to.
- If already on desired version, the updates will not be applied.
- If there is no desired version present in the list of available updates, no updates will be applied.
- If multi-node system was detected, no update will be applied.
required: true
type: str
90 changes: 90 additions & 0 deletions roles/version_update_single_node/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
---
- name: Check if there is already an update in progress
scale_computing.hypercore.version_update_status_info:
register: update_status_before_update

- name: Current update status
ansible.builtin.debug:
var: update_status_before_update

- name: Get cluster info
scale_computing.hypercore.cluster_info:
register: cluster_info

- name: Show cluster info
ansible.builtin.debug:
var: cluster_info

- name: Get node info
scale_computing.hypercore.node_info:
register: node_info

- name: Show node info
ansible.builtin.debug:
var: node_info

- name: Check if single-node system - fail if not
ansible.builtin.fail:
msg: >-
The role should be used only with single node systems.
This system does have {{ node_info.records | length }} nodes.
when: node_info.records | length > 1

# ===================================================================

- name: Update
block:
- name: Get available updates
scale_computing.hypercore.version_update_info:
register: available_updates

- name: Show available updates
ansible.builtin.debug:
var: available_updates

- name: Check if desired update is available - fail if not available
ansible.builtin.fail:
msg: >-
Requested update {{ scale_computing_hypercore_desired_version }} is not
in available_updates {{ available_updates.records | map(attribute='uuid') | list }}
when: not scale_computing_hypercore_desired_version in (available_updates.records | map(attribute='uuid') | list)

- name: Get all available running VMs
scale_computing.hypercore.vm_info:
register: vm_info

- name: Shutdown all running VMs
include_tasks: shutdown_vms.yml
vars:
vms: "{{ vm_info }}"
when: vms.records != []

# ----------------- UPDATE --------------------

- name: Update single-node system
scale_computing.hypercore.version_update:
icos_version: "{{ scale_computing_hypercore_desired_version }}"
register: update_result

- name: Check update status
include_tasks: update_status_check.yml

- name: Show update result
ansible.builtin.debug:
var: update_result

# ---------------------------------------------

- name: Restart previously running VMs
include_tasks: restart_vms.yml
vars:
vms: "{{ vm_info }}"
when: vms.records != []

- name: Check if updating to desired version failed
ansible.builtin.fail:
msg: Update to version "{{ scale_computing_hypercore_desired_version }}" failed.
when: update_result.record.uuid != scale_computing_hypercore_desired_version
when:
- cluster_info.record.icos_version != scale_computing_hypercore_desired_version
- update_status_before_update.record.update_status == "COMPLETED" or update_status_before_update.record.update_status != "IN PROGRESS"
12 changes: 12 additions & 0 deletions roles/version_update_single_node/tasks/restart_vms.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
- name: Start all VMs that were initially started
scale_computing.hypercore.vm_params:
vm_name: "{{ item.vm_name }}"
power_state: start
when: item.power_state == 'started'
loop: "{{ vms.records }}"
register: vm_start_result

- name: Show restart results
ansible.builtin.debug:
var: vm_start_result
19 changes: 19 additions & 0 deletions roles/version_update_single_node/tasks/shutdown_vms.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
- name: Show all running VMs
ansible.builtin.debug:
msg: "{{ item.vm_name }}"
when: item.power_state == 'started'
loop: "{{ vms.records }}"
register: running_vms

- name: Shutdown running VMs
scale_computing.hypercore.vm_params:
vm_name: "{{ item.vm_name }}"
power_state: stop
when: item.power_state == 'started'
loop: "{{ vms.records }}"
register: vm_shutdown_result

- name: Show shutdown results
ansible.builtin.debug:
var: vm_shutdown_result
32 changes: 32 additions & 0 deletions roles/version_update_single_node/tasks/update_status_check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
---
- name: Check update status on HC3
block:
- name: Increment retry_count
ansible.builtin.set_fact:
retry_count: "{{ 0 if retry_count is undefined else retries | int + 1 }}"

# We might be able to remove this task
- name: Pause before checking update status - checks will report FAILED-RETRYING until update COMPLETE/TERMINATED
ansible.builtin.wait_for:
timeout: 60
delegate_to: localhost

- name: Check update status - will report FAILED-RETRYING until update COMPLETE/TERMINATED
scale_computing.hypercore.version_update_status_info:
register: update_status
until: update_status.record.update_status == "COMPLETE" | default(omit) or update_status.record.update_status == "TERMINATING" | default(omit)
retries: 100
delay: 30
ignore_unreachable: true

rescue:
- name: Fail if retries reached 20
ansible.builtin.fail:
msg: Maximum retries of grouped tasks reached
when: retry_count | int == 20

- name: Log
ansible.builtin.debug:
msg: Update status check failed due to server down / restart - retrying

- include_tasks: update_status_check.yml # Recursion
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
- name: Get cluster HC3 API version before update
scale_computing.hypercore.cluster_info:
register: api_version_before
- ansible.builtin.debug:
var: api_version_before

# --------------------------------------------------------------

- name: Run single-system update role
ansible.builtin.include_role:
name: scale_computing.hypercore.version_update_single_node
vars:
scale_computing_hypercore_desired_version: "{{ desired_version_apply }}"

# --------------------------------------------------------------

- name: Get cluster HC3 API version after update
scale_computing.hypercore.cluster_info:
register: api_version_after
- ansible.builtin.debug:
var: api_version_after

# --------------------------------------------------------------

- name: Assert that api_version_after != api_version_before
ansible.builtin.assert:
that:
- api_version_after != api_version_before
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
- name: Get cluster HC3 API version before update
scale_computing.hypercore.cluster_info:
register: api_version_before
- ansible.builtin.debug:
var: api_version_before

# --------------------------------------------------------------

- name: Run single-system update role - idempotence
ansible.builtin.include_role:
name: scale_computing.hypercore.version_update_single_node
vars:
scale_computing_hypercore_desired_version: "{{ desired_version_apply }}"

# --------------------------------------------------------------

- name: Get cluster HC3 API version after update
scale_computing.hypercore.cluster_info:
register: api_version_after
- ansible.builtin.debug:
var: api_version_after

# --------------------------------------------------------------

- name: Assert that api_version_after == api_version_before
ansible.builtin.assert:
that:
- api_version_after == api_version_before
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
---
- environment:
SC_HOST: "{{ sc_host }}"
SC_USERNAME: "{{ sc_username }}"
SC_PASSWORD: "{{ sc_password }}"
SC_TIMEOUT: "{{ sc_timeout }}"

vars:
desired_version_new: 9.1.23.210897
desired_version_current: 9.1.18.209840

block:
- name: Get available updates
scale_computing.hypercore.version_update_info:
register: updates
- ansible.builtin.debug:
var: updates

- name: Test role version_update_single_node - no updates available
include_tasks: 02_update_not_available.yml
vars:
desired_version_apply: "{{ desired_version_current }}"
when: updates.records == []

- name: Test role version_update_single_node
include_tasks: 01_update.yml
vars:
desired_version_apply: "{{ desired_version_new }}"
when: updates.records != []

- name: Test role version_update_single_node - idempotence
include_tasks: 02_update_not_available.yml
vars:
desired_version_apply: "{{ desired_version_new }}"
when: updates.records != []