Merge pull request #3793 from jarrpa/glusterfs-registry

Merged by openshift-bot

filter_plugins/oo_filters.py

@@ -773,6 +773,23 @@ def oo_persistent_volumes(hostvars, groups, persistent_volumes=None):
                                     fsType=filesystem,
                                     volumeID=volume_id)))
                         persistent_volumes.append(persistent_volume)
+                    elif kind == 'glusterfs':
+                        volume = params['volume']['name']
+                        size = params['volume']['size']
+                        access_modes = params['access']['modes']
+                        endpoints = params['glusterfs']['endpoints']
+                        path = params['glusterfs']['path']
+                        read_only = params['glusterfs']['readOnly']
+                        persistent_volume = dict(
+                            name="{0}-volume".format(volume),
+                            capacity=size,
+                            access_modes=access_modes,
+                            storage=dict(
+                                glusterfs=dict(
+                                    endpoints=endpoints,
+                                    path=path,
+                                    readOnly=read_only)))
+                        persistent_volumes.append(persistent_volume)
                     elif not (kind == 'object' or kind == 'dynamic'):
                         msg = "|failed invalid storage kind '{0}' for component '{1}'".format(
                             kind,

inventory/byo/hosts.origin.example

@@ -426,6 +426,9 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 #openshift_hosted_registry_storage_openstack_volumeID=3a650b4f-c8c5-4e0a-8ca5-eaee11f16c57
 #openshift_hosted_registry_storage_volume_size=10Gi
 #
+# Native GlusterFS Registry Storage
+#openshift_hosted_registry_storage_kind=glusterfs
+#
 # AWS S3
 # S3 bucket must already exist.
 #openshift_hosted_registry_storage_kind=object

inventory/byo/hosts.ose.example

@@ -426,6 +426,9 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 #openshift_hosted_registry_storage_openstack_volumeID=3a650b4f-c8c5-4e0a-8ca5-eaee11f16c57
 #openshift_hosted_registry_storage_volume_size=10Gi
 #
+# Native GlusterFS Registry Storage
+#openshift_hosted_registry_storage_kind=glusterfs
+#
 # AWS S3
 #
 # S3 bucket must already exist.

playbooks/byo/openshift-cluster/cluster_hosts.yml

@@ -13,6 +13,8 @@ g_new_node_hosts: "{{ groups.new_nodes | default([]) }}"
 
 g_nfs_hosts: "{{ groups.nfs | default([]) }}"
 
+g_glusterfs_hosts: "{{ groups.glusterfs | default([]) }}"
+
 g_all_hosts: "{{ g_master_hosts | union(g_node_hosts) | union(g_etcd_hosts)
                  | union(g_lb_hosts) | union(g_nfs_hosts)
                  | union(g_new_node_hosts)| union(g_new_master_hosts)

playbooks/common/openshift-cluster/config.yml

@@ -31,6 +31,10 @@
   tags:
   - node
 
+- include: ../openshift-glusterfs/config.yml
+  tags:
+  - glusterfs
+
 - include: openshift_hosted.yml
   tags:
   - hosted

playbooks/common/openshift-cluster/evaluate_groups.yml

@@ -29,6 +29,10 @@
       msg: The nfs group must be limited to one host
     when: "{{ (groups[g_nfs_hosts] | default([])) | length > 1 }}"
 
+  - fail:
+      msg: This playbook requires g_glusterfs_hosts to be set
+    when: "{{ g_glusterfs_hosts is not defined }}"
+
   - name: Evaluate oo_all_hosts
     add_host:
       name: "{{ item }}"
@@ -119,3 +123,12 @@
       ansible_become: "{{ g_sudo | default(omit) }}"
     with_items: "{{ g_nfs_hosts | default([]) }}"
     changed_when: no
+
+  - name: Evaluate oo_glusterfs_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_glusterfs_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_become: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_glusterfs_hosts | default([]) }}"
+    changed_when: no

playbooks/common/openshift-glusterfs/config.yml

@@ -0,0 +1,21 @@
+---
+- name: Open firewall ports for GlusterFS
+  hosts: oo_glusterfs_to_config
+  vars:
+    os_firewall_allow:
+    - service: glusterfs_sshd
+      port: "2222/tcp"
+    - service: glusterfs_daemon
+      port: "24007/tcp"
+    - service: glusterfs_management
+      port: "24008/tcp"
+    - service: glusterfs_bricks
+      port: "49152-49251/tcp"
+  roles:
+  - os_firewall
+
+- name: Configure GlusterFS
+  hosts: oo_first_master
+  roles:
+  - role: openshift_storage_glusterfs
+    when: groups.oo_glusterfs_to_config | default([]) | count > 0

playbooks/common/openshift-glusterfs/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins

playbooks/common/openshift-glusterfs/lookup_plugins

@@ -0,0 +1 @@
+../../../lookup_plugins

playbooks/common/openshift-glusterfs/roles

@@ -0,0 +1 @@
+../../../roles/

roles/openshift_facts/library/openshift_facts.py

@@ -2155,6 +2155,10 @@ def get_defaults(self, roles, deployment_type, deployment_subtype):
                         nfs=dict(
                             directory='/exports',
                             options='*(rw,root_squash)'),
+                        glusterfs=dict(
+                            endpoints='glusterfs-registry-endpoints',
+                            path='glusterfs-registry-volume',
+                            readOnly=False),
                         host=None,
                         access=dict(
                             modes=['ReadWriteMany']

roles/openshift_hosted/tasks/registry/registry.yml

@@ -109,7 +109,7 @@
       type: persistentVolumeClaim
       claim_name: "{{ openshift.hosted.registry.storage.volume.name }}-claim"
   when:
-  - openshift.hosted.registry.storage.kind | default(none) in ['nfs', 'openstack']
+  - openshift.hosted.registry.storage.kind | default(none) in ['nfs', 'openstack', 'glusterfs']
 
 - name: Create OpenShift registry
   oc_adm_registry:
@@ -123,3 +123,7 @@
     volume_mounts: "{{ openshift_hosted_registry_volumes }}"
     edits: "{{ openshift_hosted_registry_edits }}"
     force: "{{ True|bool in openshift_hosted_registry_force }}"
+
+- include: storage/glusterfs.yml
+  when:
+  - openshift.hosted.registry.storage.kind | default(none) == 'glusterfs'

roles/openshift_hosted/tasks/registry/storage/glusterfs.yml

@@ -0,0 +1,51 @@
+---
+- name: Wait for registry pods
+  oc_obj:
+    namespace: "{{ openshift_hosted_registry_namespace }}"
+    state: list
+    kind: pod
+    selector: "{{ openshift_hosted_registry_name }}={{ openshift_hosted_registry_namespace }}"
+  register: registry_pods
+  until:
+  - "registry_pods.results.results[0]['items'] | count > 0"
+  # There must be as many matching pods with 'Ready' status True as there are expected replicas
+  - "registry_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == openshift_hosted_registry_replicas | int"
+  delay: 10
+  retries: "{{ (600 / 10) | int }}"
+
+- name: Determine registry fsGroup
+  set_fact:
+    openshift_hosted_registry_fsgroup: "{{ registry_pods.results.results[0]['items'][0].spec.securityContext.fsGroup }}"
+
+- name: Create temp mount directory
+  command: mktemp -d /tmp/openshift-glusterfs-registry-XXXXXX
+  register: mktemp
+  changed_when: False
+  check_mode: no
+
+- name: Mount registry volume
+  mount:
+    state: mounted
+    fstype: glusterfs
+    src: "{{ groups.oo_glusterfs_to_config[0] }}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
+    name: "{{ mktemp.stdout }}"
+
+- name: Set registry volume permissions
+  file:
+    dest: "{{ mktemp.stdout }}"
+    state: directory
+    group: "{{ openshift_hosted_registry_fsgroup }}"
+    mode: "2775"
+    recurse: True
+
+- name: Unmount registry volume
+  mount:
+    state: unmounted
+    name: "{{ mktemp.stdout }}"
+
+- name: Delete temp mount directory
+  file:
+    dest: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False
+  check_mode: no

roles/openshift_storage_glusterfs/README.md

@@ -0,0 +1,60 @@
+OpenShift GlusterFS Cluster
+===========================
+
+OpenShift GlusterFS Cluster Installation
+
+Requirements
+------------
+
+* Ansible 2.2
+
+Role Variables
+--------------
+
+From this role:
+
+| Name                                             | Default value           |                                         |
+|--------------------------------------------------|-------------------------|-----------------------------------------|
+| openshift_storage_glusterfs_timeout              | 300                     | Seconds to wait for pods to become ready
+| openshift_storage_glusterfs_namespace            | 'default'               | Namespace in which to create GlusterFS resources
+| openshift_storage_glusterfs_is_native            | True                    | GlusterFS should be containerized
+| openshift_storage_glusterfs_nodeselector         | 'storagenode=glusterfs' | Selector to determine which nodes will host GlusterFS pods in native mode
+| openshift_storage_glusterfs_image                | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7'
+| openshift_storage_glusterfs_version              | 'latest'                | Container image version to use for GlusterFS pods
+| openshift_storage_glusterfs_wipe                 | False                   | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.**
+| openshift_storage_glusterfs_heketi_is_native     | True                    | heketi should be containerized
+| openshift_storage_glusterfs_heketi_image         | 'heketi/heketi'         | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7'
+| openshift_storage_glusterfs_heketi_version       | 'latest'                | Container image version to use for heketi pods
+| openshift_storage_glusterfs_heketi_admin_key     | ''                      | String to use as secret key for performing heketi commands as admin
+| openshift_storage_glusterfs_heketi_user_key      | ''                      | String to use as secret key for performing heketi commands as user that can only view or modify volumes
+| openshift_storage_glusterfs_heketi_topology_load | True                    | Load the GlusterFS topology information into heketi
+| openshift_storage_glusterfs_heketi_url           | Undefined               | URL for the heketi REST API, dynamically determined in native mode
+| openshift_storage_glusterfs_heketi_wipe          | False                   | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
+
+Dependencies
+------------
+
+* os_firewall
+* openshift_hosted_facts
+* openshift_repos
+* lib_openshift
+
+Example Playbook
+----------------
+
+```
+- name: Configure GlusterFS hosts
+  hosts: oo_first_master
+  roles:
+  - role: openshift_storage_glusterfs
+```
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Jose A. Rivera ([email protected])

roles/openshift_storage_glusterfs/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+openshift_storage_glusterfs_timeout: 300
+openshift_storage_glusterfs_namespace: 'default'
+openshift_storage_glusterfs_is_native: True
+openshift_storage_glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector_label | default('storagenode=glusterfs') | map_from_pairs }}"
+openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}"
+openshift_storage_glusterfs_version: 'latest'
+openshift_storage_glusterfs_wipe: False
+openshift_storage_glusterfs_heketi_is_native: True
+openshift_storage_glusterfs_heketi_is_missing: True
+openshift_storage_glusterfs_heketi_deploy_is_missing: True
+openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}"
+openshift_storage_glusterfs_heketi_version: 'latest'
+openshift_storage_glusterfs_heketi_admin_key: ''
+openshift_storage_glusterfs_heketi_user_key: ''
+openshift_storage_glusterfs_heketi_topology_load: True
+openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"

roles/openshift_storage_glusterfs/files/v1.6/deploy-heketi-template.yml

@@ -0,0 +1,115 @@
+---
+kind: Template
+apiVersion: v1
+metadata:
+  name: deploy-heketi
+  labels:
+    glusterfs: heketi-template
+    deploy-heketi: support
+  annotations:
+    description: Bootstrap Heketi installation
+    tags: glusterfs,heketi,installation
+labels:
+  template: deploy-heketi
+objects:
+- kind: Service
+  apiVersion: v1
+  metadata:
+    name: deploy-heketi
+    labels:
+      glusterfs: deploy-heketi-service
+      deploy-heketi: support
+    annotations:
+      description: Exposes Heketi service
+  spec:
+    ports:
+    - name: deploy-heketi
+      port: 8080
+      targetPort: 8080
+    selector:
+      name: deploy-heketi
+- kind: Route
+  apiVersion: v1
+  metadata:
+    name: deploy-heketi
+    labels:
+      glusterfs: deploy-heketi-route
+      deploy-heketi: support
+  spec:
+    to:
+      kind: Service
+      name: deploy-heketi
+- kind: DeploymentConfig
+  apiVersion: v1
+  metadata:
+    name: deploy-heketi
+    labels:
+      glusterfs: deploy-heketi-dc
+      deploy-heketi: support
+    annotations:
+      description: Defines how to deploy Heketi
+  spec:
+    replicas: 1
+    selector:
+      name: deploy-heketi
+    triggers:
+    - type: ConfigChange
+    strategy:
+      type: Recreate
+    template:
+      metadata:
+        name: deploy-heketi
+        labels:
+          name: deploy-heketi
+          glusterfs: deploy-heketi-pod
+          deploy-heketi: support
+      spec:
+        serviceAccountName: heketi-service-account
+        containers:
+        - name: deploy-heketi
+          image: ${IMAGE_NAME}:${IMAGE_VERSION}
+          env:
+          - name: HEKETI_USER_KEY
+            value: ${HEKETI_USER_KEY}
+          - name: HEKETI_ADMIN_KEY
+            value: ${HEKETI_ADMIN_KEY}
+          - name: HEKETI_EXECUTOR
+            value: kubernetes
+          - name: HEKETI_FSTAB
+            value: /var/lib/heketi/fstab
+          - name: HEKETI_SNAPSHOT_LIMIT
+            value: '14'
+          - name: HEKETI_KUBE_GLUSTER_DAEMONSET
+            value: '1'
+          ports:
+          - containerPort: 8080
+          volumeMounts:
+          - name: db
+            mountPath: /var/lib/heketi
+          readinessProbe:
+            timeoutSeconds: 3
+            initialDelaySeconds: 3
+            httpGet:
+              path: /hello
+              port: 8080
+          livenessProbe:
+            timeoutSeconds: 3
+            initialDelaySeconds: 30
+            httpGet:
+              path: /hello
+              port: 8080
+        volumes:
+        - name: db
+parameters:
+- name: HEKETI_USER_KEY
+  displayName: Heketi User Secret
+  description: Set secret for those creating volumes as type _user_
+- name: HEKETI_ADMIN_KEY
+  displayName: Heketi Administrator Secret
+  description: Set secret for administration of the Heketi service as user _admin_
+- name: IMAGE_NAME
+  displayName: GlusterFS container name
+  required: True
+- name: IMAGE_VERSION
+  displayName: GlusterFS container versiona
+  required: True