From f53aae9f7ff1fd2cde78b5005d8d708528449851 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 9 Apr 2021 08:38:10 +0200 Subject: [PATCH] Add wrappers for EOS and EOS Home storage drivers (#1624) --- changelog/unreleased/eos-wrappers.md | 9 ++ .../storageprovider/storageprovider.go | 5 +- pkg/cbox/loader/loader.go | 2 + pkg/cbox/publicshare/sql/sql.go | 2 +- .../storage/eoshomewrapper/eoshomewrapper.go | 123 ++++++++++++++++++ pkg/cbox/storage/eoswrapper/eoswrapper.go | 123 ++++++++++++++++++ 6 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/eos-wrappers.md create mode 100644 pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go create mode 100644 pkg/cbox/storage/eoswrapper/eoswrapper.go diff --git a/changelog/unreleased/eos-wrappers.md b/changelog/unreleased/eos-wrappers.md new file mode 100644 index 00000000000..05abb449751 --- /dev/null +++ b/changelog/unreleased/eos-wrappers.md @@ -0,0 +1,9 @@ +Enhancement: Add wrappers for EOS and EOS Home storage drivers + +For CERNBox, we need the mount ID to be configured according to the owner of a +resource. Setting this in the storageprovider means having different instances +of this service to cater to different users, which does not scale. This driver +forms a wrapper around the EOS driver and sets the mount ID according to a +configurable mapping based on the owner of the resource. + +https://github.com/cs3org/reva/pull/1624 diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 424e24e0086..729e660eee2 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -1142,7 +1142,10 @@ func (s *service) trimMountPrefix(fn string) (string, error) { } func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo) error { - ri.Id.StorageId = s.mountID + if ri.Id.StorageId == "" { + // For wrapper drivers, the storage ID might already be set. In that case, skip setting it + ri.Id.StorageId = s.mountID + } ri.Path = path.Join(s.mountPath, ri.Path) return nil } diff --git a/pkg/cbox/loader/loader.go b/pkg/cbox/loader/loader.go index a46b40c90a8..a215a8aa03e 100644 --- a/pkg/cbox/loader/loader.go +++ b/pkg/cbox/loader/loader.go @@ -23,5 +23,7 @@ import ( _ "github.com/cs3org/reva/pkg/cbox/group/rest" _ "github.com/cs3org/reva/pkg/cbox/publicshare/sql" _ "github.com/cs3org/reva/pkg/cbox/share/sql" + _ "github.com/cs3org/reva/pkg/cbox/storage/eoshomewrapper" + _ "github.com/cs3org/reva/pkg/cbox/storage/eoswrapper" _ "github.com/cs3org/reva/pkg/cbox/user/rest" ) diff --git a/pkg/cbox/publicshare/sql/sql.go b/pkg/cbox/publicshare/sql/sql.go index 152d3c42b23..cfc0ba6796b 100644 --- a/pkg/cbox/publicshare/sql/sql.go +++ b/pkg/cbox/publicshare/sql/sql.go @@ -437,7 +437,7 @@ func (m *manager) cleanupExpiredShares() error { return nil } - query := "delete from oc_share where expiration IS NOT NULL AND expiration < ?" + query := "update oc_share set orphan = 1 where expiration IS NOT NULL AND expiration < ?" params := []interface{}{time.Now().Format("2006-01-02 03:04:05")} stmt, err := m.db.Prepare(query) diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go new file mode 100644 index 00000000000..eda1f55ab5a --- /dev/null +++ b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go @@ -0,0 +1,123 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eoshome + +import ( + "bytes" + "context" + "text/template" + + "github.com/Masterminds/sprig" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/eosfs" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("eoshomewrapper", New) +} + +type wrapper struct { + storage.FS + mountIDTemplate *template.Template +} + +func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { + c := &eosfs.Config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, "", err + } + + // default to version invariance if not configured + if _, ok := m["version_invariant"]; !ok { + c.VersionInvariant = true + } + + t, ok := m["mount_id_template"].(string) + if !ok || t == "" { + t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}" + } + + return c, t, nil +} + +// New returns an implementation of the storage.FS interface that forms a wrapper +// around separate connections to EOS. +func New(m map[string]interface{}) (storage.FS, error) { + c, t, err := parseConfig(m) + if err != nil { + return nil, err + } + c.EnableHome = true + + eos, err := eosfs.NewEOSFS(c) + if err != nil { + return nil, err + } + + mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t) + if err != nil { + return nil, err + } + + return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil +} + +// We need to override the two methods, GetMD and ListFolder to fill the +// StorageId in the ResourceInfo objects. + +func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { + res, err := w.FS.GetMD(ctx, ref, mdKeys) + if err != nil { + return nil, err + } + + // We need to extract the mount ID based on the mapping template. + // + // Take the first letter of the resource path after the namespace has been removed. + // If it's empty, leave it empty to be filled by storageprovider. + res.Id.StorageId = w.getMountID(ctx, res) + return res, nil +} + +func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { + res, err := w.FS.ListFolder(ctx, ref, mdKeys) + if err != nil { + return nil, err + } + for _, r := range res { + r.Id.StorageId = w.getMountID(ctx, r) + } + return res, nil +} + +func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string { + if r == nil { + return "" + } + b := bytes.Buffer{} + if err := w.mountIDTemplate.Execute(&b, r); err != nil { + return "" + } + return b.String() +} diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go new file mode 100644 index 00000000000..2f27c422305 --- /dev/null +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -0,0 +1,123 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eoshome + +import ( + "bytes" + "context" + "text/template" + + "github.com/Masterminds/sprig" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/eosfs" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("eoswrapper", New) +} + +type wrapper struct { + storage.FS + mountIDTemplate *template.Template +} + +func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { + c := &eosfs.Config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, "", err + } + + // default to version invariance if not configured + if _, ok := m["version_invariant"]; !ok { + c.VersionInvariant = true + } + + t, ok := m["mount_id_template"].(string) + if !ok || t == "" { + t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}" + } + + return c, t, nil +} + +// New returns an implementation of the storage.FS interface that forms a wrapper +// around separate connections to EOS. +func New(m map[string]interface{}) (storage.FS, error) { + c, t, err := parseConfig(m) + if err != nil { + return nil, err + } + + eos, err := eosfs.NewEOSFS(c) + if err != nil { + return nil, err + } + + mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t) + if err != nil { + return nil, err + } + + return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil +} + +// We need to override the two methods, GetMD and ListFolder to fill the +// StorageId in the ResourceInfo objects. + +func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { + res, err := w.FS.GetMD(ctx, ref, mdKeys) + if err != nil { + return nil, err + } + + // We need to extract the mount ID based on the mapping template. + // + // Take the first letter of the resource path after the namespace has been removed. + // If it's empty, leave it empty to be filled by storageprovider. + res.Id.StorageId = w.getMountID(ctx, res) + return res, nil + +} + +func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { + res, err := w.FS.ListFolder(ctx, ref, mdKeys) + if err != nil { + return nil, err + } + for _, r := range res { + r.Id.StorageId = w.getMountID(ctx, r) + } + return res, nil +} + +func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string { + if r == nil { + return "" + } + b := bytes.Buffer{} + if err := w.mountIDTemplate.Execute(&b, r); err != nil { + return "" + } + return b.String() +}