EP-2061 -> master | Prep vfs for opensource

LICENSE → License.md

@@ -1,6 +1,6 @@
-The MIT License
+The MIT License (MIT)
 
-Copyright (c) 2011-2012 C2FO
+Copyright (c) 2019 C2FO, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

backend/all/all.go

@@ -0,0 +1,8 @@
+// Package all imports all VFS implementations.
+package all
+
+import (
+	_ "github.com/c2fo/vfs/backend/gs"
+	_ "github.com/c2fo/vfs/backend/os"
+	_ "github.com/c2fo/vfs/backend/s3"
+)

backend/backend.go

@@ -0,0 +1,56 @@
+package backend
+
+import (
+	"sort"
+	"sync"
+
+	"github.com/c2fo/vfs"
+)
+
+var mmu sync.RWMutex
+var m map[string]vfs.FileSystem
+
+// Register a new filesystem in backend map
+func Register(name string, v vfs.FileSystem) {
+	mmu.Lock()
+	m[name] = v
+	mmu.Unlock()
+}
+
+// Unregister unregisters a filesystem from backend map
+func Unregister(name string) {
+	mmu.Lock()
+	delete(m, name)
+	mmu.Unlock()
+}
+
+// UnregisterAll unregisters all filesystems from backend map
+func UnregisterAll() {
+	// mainly for tests
+	mmu.Lock()
+	m = make(map[string]vfs.FileSystem)
+	mmu.Unlock()
+}
+
+// Backend returns the backend filesystem by name
+func Backend(name string) vfs.FileSystem {
+	mmu.RLock()
+	defer mmu.RUnlock()
+	return m[name]
+}
+
+// RegisteredBackends returns an array of backend names
+func RegisteredBackends() []string {
+	var f []string
+	mmu.RLock()
+	for k := range m {
+		f = append(f, k)
+	}
+	mmu.RUnlock()
+	sort.Strings(f)
+	return f
+}
+
+func init() {
+	m = make(map[string]vfs.FileSystem)
+}

backend/doc.go

@@ -0,0 +1,78 @@
+/*
+Package backend provides a means of allowing backend filesystems to self-register on load via an init() call to
+backend.Register("some name", vfs.Filesystem)
+
+In this way, a caller of vfs backends can simply load the backend filesystem (and ONLY those needed) and begin using it:
+
+  package main
+
+  // import backend and each backend you intend to use
+  import(
+      "github.com/c2fo/vfs/backend"
+      _ "github.com/c2fo/vfs/backend/os"
+      _ "github.com/c2fo/vfs/backend/s3"
+  )
+
+  func main() {
+     var err error
+     var osfile, s3file vfs.File
+
+      // THEN begin using the filesystems
+      osfile, err = backend.Backend(os.Scheme).NewFile("", "/path/to/file.txt")
+      if err != nil {
+          panic(err)
+      }
+
+      s3file, err = backend.Backend(os.Scheme).NewFile("", "/some/file.txt")
+      if err != nil {
+          panic(err)
+      }
+
+      err = osfile.CopyTo(s3file)
+      if err != nil {
+          panic(err)
+      }
+  }
+
+Development
+
+To create your own backend, you must create a package that implements the interfaces: vfs.Filesystem, vfs.Location, and vfs.File.
+Then ensure it registers itself on load:
+
+  pacakge myexoticfilesystem
+
+  import(
+      ...
+      "github.com/c2fo/vfs"
+      "github.com/c2fo/vfs/backend"
+  )
+
+  // IMPLEMENT vfs interfaces
+  ...
+
+  // register backend
+  func init() {
+      backend.Register(
+          "My Exotic Filesystem",
+          &MyExoticFilesystem{},
+      )
+  }
+
+Then do use it in some other package do
+  pacakge MyExoticFilesystem
+
+  import(
+      "github.com/c2fo/vfs/backend"
+      _ "github.com/acme/myexoticfilesystem"
+  )
+
+  ...
+
+  func useNewBackend() error {
+      myExoticFs, err = backend.Backend(myexoticfilesystem.Scheme)
+      ...
+  }
+
+Thats it.  Simple.
+*/
+package backend

backend/gs/doc.go

@@ -0,0 +1,75 @@
+/*
+Package gs Google Cloud Storage VFS implementation.
+
+Usage
+
+Rely on github.com/c2fo/vfs/backend
+
+  import(
+      "github.com/c2fo/vfs/backend"
+      _ "github.com/c2fo/vfs/backend/gs"
+  )
+
+  func UseFs() error {
+      fs, err := backend.Backend("Google Cloud Storage")
+      ...
+  }
+
+Or call directly:
+
+  import "github.com/c2fo/vfs/backend/gs"
+
+  func DoSomething() {
+      fs := gs.NewFilesystem()
+      ...
+  }
+
+gs can be augmented with the following implementation-specific methods.  Backend returns vfs.Filesystem interface so it
+would have to be cast as gs.Filesystem to use the following:
+
+  func DoSomething() {
+
+      ...
+
+      // cast if fs was created using backend.Backend().  Not necessary if created directly from gs.NewFilsystem().
+      fs = fs.(gs.Filesystem)
+
+      // to use your own "context"
+      ctx := context.Background()
+      fs = fs.WithContext(ctx)
+
+      // to pass in client options
+      fs = fs.WithOptions(
+          gs.Options{
+              CredentialFile: "/root/.gcloud/account.json",
+              Scopes:         []string{"ScopeReadOnly"},
+              //default scope is "ScopeFullControl"
+          },
+      )
+
+      // to pass specific client, for instance no-auth client
+      ctx := context.Background()
+      client, _ := storage.NewClient(ctx, option.WithoutAuthentication())
+      fs = fs.WithClient(client)
+  }
+
+Authentication
+
+Authentication, by default, occurs automatically when Client() is called. It looks for credentials in the following places,
+preferring the first location found:
+
+  1. A JSON file whose path is specified by the GOOGLE_APPLICATION_CREDENTIALS environment variable
+  2. A JSON file in a location known to the gcloud command-line tool.
+     On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
+     On other systems, $HOME/.config/gcloud/application_default_credentials.json.
+  3. On Google App Engine it uses the appengine.AccessToken function.
+  4. On Google Compute Engine and Google App Engine Managed VMs, it fetches credentials from the metadata server.
+
+See https://cloud.google.com/docs/authentication/production for more autho info
+
+See Also
+
+See: https://github.com/googleapis/google-cloud-go/tree/master/storage
+
+*/
+package gs