Merge dev GA v1.0.0 to master branch

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
 go:
-  - 1.7
+  - 1.9
 
 install:
   - sh scripts/run_glide_up

Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.9.1
+WORKDIR /go/src/github.com/IBM/ubiquity/
+COPY . .
+RUN go get -v github.com/Masterminds/glide
+RUN glide up
+RUN CGO_ENABLED=1 GOOS=linux go build -tags netgo -v -a --ldflags '-w -linkmode external -extldflags "-static"' -installsuffix cgo -o ubiquity main.go
+
+
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates=20161130-r2 openssl=1.0.2k-r0
+WORKDIR /root/
+COPY --from=0 /go/src/github.com/IBM/ubiquity/ubiquity .
+COPY --from=0 /go/src/github.com/IBM/ubiquity/LICENSE .
+COPY --from=0 /go/src/github.com/IBM/ubiquity/scripts/notices_file_for_ibm_storage_enabler_for_containers ./NOTICES
+
+COPY docker-entrypoint.sh .
+RUN chmod 755 docker-entrypoint.sh
+
+# comments below should be removed when we implement the new SSL_MODE env variable
+ENV PATH=/root:$PATH \
+    UBIQUITY_SERVER_CERT_PRIVATE=/var/lib/ubiquity/ssl/private/ubiquity.key \
+    UBIQUITY_SERVER_CERT_PUBLIC=/var/lib/ubiquity/ssl/private/ubiquity.crt \
+    UBIQUITY_SERVER_VERIFY_SCBE_CERT=/var/lib/ubiquity/ssl/public/scbe-trusted-ca.crt \
+    UBIQUITY_DB_SSL_ROOT_CERT=/var/lib/ubiquity/ssl/public/ubiquity-db-trusted-ca.crt \
+    SSL_MODE=verify-full
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["ubiquity"]
+

README.md

@@ -5,99 +5,41 @@
 [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0)
 [![Go Report Card](https://goreportcard.com/badge/github.com/IBM/ubiquity)](https://goreportcard.com/report/github.com/IBM/ubiquity)
 
-The Ubiquity project enables persistent storage for the Kubernetes and Docker container frameworks. 
-It is a pluggable framework available for different storage systems. The framework interfaces with the storage systems, using their plugins. The [Available Storage Systems](supportedStorage.md) section describes the storage system  configuration and deployment options. Different container frameworks can use Ubiquity concurrently, allowing access to different storage systems. 
 
-
-![Ubiquity Overview](images/UbiquityOverview.jpg)
+The Ubiquity project enables persistent storage for the Kubernetes and Docker container frameworks. It is a pluggable framework available for different storage systems. The framework interfaces with the storage systems, using their plugins. Different container frameworks can use Ubiquity concurrently, allowing access to different storage systems.
+Ubiquity supports the Kubernetes and Docker frameworks, using the following plugins:
 
 Ubiquity supports the Kubernetes and Docker frameworks, using the following plugins:
 
-- [Ubiquity Docker volume plugin](https://github.com/IBM/ubiquity-docker-plugin)
 - [Ubiquity plugin for Kubernetes](https://github.com/IBM/ubiquity-k8s) (Dynamic Provisioner and FlexVolume)
+- [Ubiquity Docker volume plugin](https://github.com/IBM/ubiquity-docker-plugin), for testing only.
 
-The code is provided as is, without warranty. Any issue will be handled on a best-effort basis.
+Currently, the following storage systems use Ubiquity:
+* IBM block storage.
 
-## Installing the Ubiquity service
-
-### 1. Prerequisites
-  * Ubiquity is available on the following operating systems:
-    - RHEL 7+
-    - SUSE 12+
-    - Ubuntu 16+
-  * The following sudoers configuration `/etc/sudoers` is required to run the Ubiquity process as root user: 
-
-     ```
-        Defaults !requiretty
-     ```
-     For non-root users, such as USER, configure the sudoers as follows: 
-
-     ```
-         USER ALL= NOPASSWD: /usr/bin/, /bin/
-         Defaults:%USER !requiretty
-         Defaults:%USER secure_path = /sbin:/bin:/usr/sbin:/usr/bin
-     ```
-
-
-  * Configure your [storage backend](supportedStorage.md). 
-
-### 2. Downloading and installing the Ubiquity service 
-
-  * Download and unpack the application package.
-```bash
-mkdir -p /etc/ubiquity
-cd /etc/ubiquity
-curl -L https://github.com/IBM/ubiquity/releases/download/v0.4.0/ubiquity-0.4.0.tar.gz | tar xzf -
-chmod u+x ubiquity
-cp ubiquity /usr/bin/ubiquity                
-#chown USER:GROUP /usr/bin/ubiquity  # Run this command only if non-root should run ubiquity (fill up the USER and GROUP)
-cp ubiquity.service /usr/lib/systemd/system/ 
-```
-   * To run the ubiquity as non-root users, you must add to the `/usr/lib/systemd/system/ubiquity.service` file this line `User=USER` under the [Service] item.
-
-   * Enable the Ubiquity service
+   The IBM block storage is supported for Kubernetes via IBM Spectrum Control Base Edition (SCBE) version 3.3.0 and later. Ubiquity communicates with the IBM storage systems through SCBE. SCBE creates a storage profile (for example, gold, silver or bronze) and makes it available for Kubernetes. For details about supported storage systems, refer to the latest SCBE release notes.
 
-```bash 
-systemctl enable ubiquity.service         
-```
+   The IBM official solution for Kubernetes, based on the Ubiquity project, is referred to as IBM Storage Enabler for Containers. You can download the installation package and its documentation from [IBM Fix Central](https://www-945.ibm.com/support/fixcentral/swg/selectFixes?parent=Software%2Bdefined%2Bstorage&product=ibm/StorageSoftware/IBM+Spectrum+Control&release=All&platform=Linux&function=all). For details on the IBM Storage Enabler for Containers, see the relevant sections in the SCBE user guide.   
 
-### 3. Configuring the Ubiquity service
-Before running the Ubiquity service, you must create and configure the `/etc/ubiquity/ubiquity-server.conf` file, according to your storage system type.
-Follow the configuration procedures detailed in the [Available Storage Systems](supportedStorage.md) section.
-
-
-### 4. Running the Ubiquity service
-  * Run the service.
-```bash
-systemctl start ubiquity    
-```
+* IBM Spectrum Scale, for testing only.
 
+The code is provided as is, without warranty. Any issue will be handled on a best-effort basis.
 
-### 5. Installing Ubiquity plugins for Docker or Kubernetes
-To use the active Ubiquity service, install Ubiquity plugins for the relevant container framework. The plugin packages and installation instructions are available in seperate github repositories.
-  * [Ubiquity Docker volume plugin](https://github.com/IBM/ubiquity-docker-plugin)
-  * [Ubiquity Kubernetes plugin](https://github.com/IBM/ubiquity-k8s)
+## Solution overview
 
+![Ubiquity Overview](images/ubiquity_architecture_draft_for_github.jpg)
 
-## Roadmap
- * Make Ubiquity Docker volume plugin in Docker store
- * Containerize Ubiquity service for Docker and Kubernetes
- * Add more IBM storage systems as Ubiquity backends
- * Add OpenStack Manila storage as Ubiquity backend
- * Add Cloud Foundry, as a container framework
- * Secure communication between plugins and Ubiquity service, using certificates
- * Enable volume sharing between multiple nodes
- * And more goodies to come...
+Description of Ubiquity Kubernetes deployment:
+   *   Ubiquity Kubernetes Dynamic Provisioner (ubiquity-k8s-provisioner) runs as a Kubernetes deployment with replica=1.
+   *   Ubiquity Kubernetes FlexVolume (ubiquity-k8s-flex) runs as a Kubernetes daemonset on all the worker and master nodes.
+   *   Ubiquity (ubiquity) runs as a Kubernetes deployment with replica=1.
+   *   Ubiquity database (ubiquity-db) runs as a Kubernetes deployment with replica=1.
 
 
 ## Contribution
 To contribute, follow the guidelines in [Contribution guide](contribution-guide.md)
 
 
-## Troubleshooting
-* Review the Ubiquity logs for any issues:
-    * [logPath]/ubiquity.log   ([logPath] configured in the ubiquity-server.conf)
-    * /var/log/messages        
 
 ## Support
 For any questions, suggestions, or issues, use github.

database/connection.go

@@ -0,0 +1,123 @@
+/**
+ * Copyright 2017 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package database
+
+import (
+    "github.com/jinzhu/gorm"
+    _ "github.com/jinzhu/gorm/dialects/postgres"
+    _ "github.com/jinzhu/gorm/dialects/sqlite"
+    "github.com/IBM/ubiquity/utils/logs"
+    "errors"
+)
+
+var globalConnectionFactory ConnectionFactory = nil
+
+func initConnectionFactory(connectionFactory ConnectionFactory) func() {
+    if globalConnectionFactory != nil {
+        panic("globalConnectionFactory already initialized")
+    }
+    globalConnectionFactory = connectionFactory
+    return func() { globalConnectionFactory = nil }
+}
+
+type ConnectionFactory interface {
+    newConnection() (*gorm.DB, error)
+}
+
+type postgresFactory struct {
+    psql     string
+    psqlLog  string
+}
+
+type sqliteFactory struct {
+    path     string
+}
+
+type testErrorFactory struct {
+}
+
+func (f *postgresFactory) newConnection() (*gorm.DB, error) {
+    logger := logs.GetLogger()
+    logger.Debug("", logs.Args{{"psql", f.psqlLog}})
+    return gorm.Open("postgres", f.psql)
+}
+
+func (f *sqliteFactory) newConnection() (*gorm.DB, error) {
+    return gorm.Open("sqlite3", f.path)
+}
+
+func (f *testErrorFactory) newConnection() (*gorm.DB, error) {
+    return nil, errors.New("testErrorFactory")
+}
+
+type Connection struct {
+    factory  ConnectionFactory
+    logger   logs.Logger
+    db       *gorm.DB
+}
+
+func NewConnection() Connection {
+    return Connection{logger: logs.GetLogger(), factory: globalConnectionFactory}
+}
+
+func (c *Connection) Open() (error) {
+    defer c.logger.Trace(logs.DEBUG)()
+    var err error
+
+    // sanity
+    if c.db != nil {
+        return c.logger.ErrorRet(errors.New("Connection already open"), "failed")
+    }
+
+    // open db connection
+    if c.db, err = c.factory.newConnection(); err != nil {
+        return c.logger.ErrorRet(err, "failed")
+    }
+
+    // do migrations
+    if err = doMigrations(*c); err != nil {
+        defer c.Close()
+        return c.logger.ErrorRet(err, "doMigrations failed")
+    }
+
+    return nil
+}
+
+func (c *Connection) Close() (error) {
+    defer c.logger.Trace(logs.DEBUG)()
+    var err error
+
+    // sanity
+    if c.db == nil {
+        return c.logger.ErrorRet(errors.New("Connection already closed"), "failed")
+    }
+
+    // close db connection
+    err = c.db.Close()
+    c.db = nil
+    if err != nil {
+        return c.logger.ErrorRet(err, "failed")
+    }
+
+    return nil
+}
+
+func (c *Connection) GetDb() (*gorm.DB) {
+    defer c.logger.Trace(logs.DEBUG)()
+
+    return c.db
+}

database/connection_test.go

@@ -0,0 +1,64 @@
+/**
+ * Copyright 2017 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package database_test
+
+
+import (
+    . "github.com/onsi/ginkgo"
+    . "github.com/onsi/gomega"
+    "github.com/IBM/ubiquity/database"
+)
+
+
+var _ = Describe("Connection", func() {
+    var (
+        err error
+    )
+    BeforeEach(func() {
+    })
+
+    Context(".Open and Close", func() {
+        It("open and close success", func() {
+            dbConnection := database.NewConnection()
+            Expect(dbConnection.GetDb()).To(BeNil())
+            err = dbConnection.Open()
+            Expect(err).To(Not(HaveOccurred()))
+            Expect(dbConnection.GetDb()).To(Not(BeNil()))
+            err = dbConnection.Close()
+            Expect(err).To(Not(HaveOccurred()))
+            Expect(dbConnection.GetDb()).To(BeNil())
+        })
+        It("open fail", func() {
+            dbConnection := database.NewConnection()
+            Expect(dbConnection.GetDb()).To(BeNil())
+            err = dbConnection.Open()
+            Expect(err).To(Not(HaveOccurred()))
+            Expect(dbConnection.GetDb()).To(Not(BeNil()))
+            err = dbConnection.Open()
+            Expect(err).To(HaveOccurred())
+            err = dbConnection.Close()
+            Expect(err).To(Not(HaveOccurred()))
+            Expect(dbConnection.GetDb()).To(BeNil())
+        })
+        It("close fail", func() {
+            dbConnection := database.NewConnection()
+            Expect(dbConnection.GetDb()).To(BeNil())
+            err = dbConnection.Close()
+            Expect(err).To(HaveOccurred())
+        })
+    })
+})