diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index afec9427c3c..a9984a952b9 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -46,14 +46,13 @@ func init() { type config struct { MountPath string `mapstructure:"mount_path"` - MountID string `mapstructure:"mount_id"` GatewayAddr string `mapstructure:"gateway_addr"` } type service struct { - conf *config - mountPath, mountID string - gateway gateway.GatewayAPIClient + conf *config + mountPath string + gateway gateway.GatewayAPIClient } func (s *service) Close() error { @@ -85,7 +84,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { } mountPath := c.MountPath - mountID := c.MountID gateway, err := pool.GetGatewayServiceClient(c.GatewayAddr) if err != nil { @@ -95,7 +93,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, mountPath: mountPath, - mountID: mountID, gateway: gateway, } diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 914d537cce9..744265bc8ff 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -28,6 +28,5 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/s3" - _ "github.com/cs3org/reva/pkg/storage/fs/stub" // Add your own here ) diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go new file mode 100644 index 00000000000..53569ae9663 --- /dev/null +++ b/pkg/storage/fs/ocis/node.go @@ -0,0 +1,58 @@ +package ocis + +import ( + "os" + "path/filepath" + + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// NodeInfo allows referencing a node by id and optionally a relative path +type NodeInfo struct { + ParentID string + ID string + Name string + Exists bool +} + +// BecomeParent rewrites the internal state to point to the parent id +func (n *NodeInfo) BecomeParent() { + n.ID = n.ParentID + n.ParentID = "" + n.Name = "" + n.Exists = false +} + +// Create creates a new node in the given root and add symlinks to parent node +// TODO use a reference to the tree to access tho root? +func (n *NodeInfo) Create(root string) (err error) { + + if n.ID != "" { + return errors.Wrap(err, "ocisfs: node already his an id") + } + // create a new file node + n.ID = uuid.New().String() + + nodePath := filepath.Join(root, "nodes", n.ID) + + err = os.MkdirAll(nodePath, 0700) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node dir") + } + // create back link + // we are not only linking back to the parent, but also to the filename + link := "../" + n.ParentID + "/children/" + n.Name + err = os.Symlink(link, filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent node") + } + + // link child name to node + err = os.Symlink("../../"+n.ID, filepath.Join(root, "nodes", n.ParentID, "children", n.Name)) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } + + return nil +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index d2a9d925bbd..fef47b9d726 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -20,12 +20,10 @@ package ocis import ( "context" - "fmt" "io" - "io/ioutil" "net/url" "os" - "path" + "path/filepath" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -35,7 +33,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/mitchellh/mapstructure" @@ -49,7 +47,7 @@ func init() { type config struct { // ocis fs works on top of a dir of uuid nodes - DataDirectory string `mapstructure:"data_directory"` + Root string `mapstructure:"root"` // UserLayout wraps the internal path with user information. // Example: if conf.Namespace is /ocis/user and received path is /docs @@ -72,10 +70,10 @@ func parseConfig(m map[string]interface{}) (*config, error) { func (c *config) init(m map[string]interface{}) { if c.UserLayout == "" { - c.UserLayout = "{{.Username}}" + c.UserLayout = "{{.Id.OpaqueId}}" } // c.DataDirectory should never end in / unless it is the root - c.DataDirectory = path.Clean(c.DataDirectory) + c.Root = filepath.Clean(c.Root) // TODO we need a lot more mimetypes mime.RegisterMime(".txt", "text/plain") @@ -91,10 +89,13 @@ func New(m map[string]interface{}) (storage.FS, error) { c.init(m) dataPaths := []string{ - path.Join(c.DataDirectory, "users"), - path.Join(c.DataDirectory, "nodes"), - path.Join(c.DataDirectory, "trash/files"), - path.Join(c.DataDirectory, "trash/info"), + filepath.Join(c.Root, "users"), + filepath.Join(c.Root, "nodes"), + // notes contain symlinks from nodes//uploads/ to ../../uploads/ + // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir + filepath.Join(c.Root, "uploads"), + filepath.Join(c.Root, "trash/files"), + filepath.Join(c.Root, "trash/info"), } for _, v := range dataPaths { if err := os.MkdirAll(v, 0700); err != nil { @@ -105,12 +106,12 @@ func New(m map[string]interface{}) (storage.FS, error) { } pw := &Path{ - DataDirectory: c.DataDirectory, - EnableHome: c.EnableHome, - UserLayout: c.UserLayout, + Root: c.Root, + EnableHome: c.EnableHome, + UserLayout: c.UserLayout, } - tp, err := NewTree(pw, c.DataDirectory) + tp, err := NewTree(pw, c.Root) if err != nil { return nil, err } @@ -136,8 +137,7 @@ func (fs *ocisfs) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, nil } -// Home discovery - +// CreateHome creates a new root node that has no parent id func (fs *ocisfs) CreateHome(ctx context.Context) error { if !fs.conf.EnableHome || fs.conf.UserLayout == "" { return errtypes.NotSupported("ocisfs: create home not supported") @@ -149,7 +149,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { return err } layout := templates.WithUser(u, fs.conf.UserLayout) - home := path.Join(fs.conf.DataDirectory, "users", layout) + home := filepath.Join(fs.conf.Root, "users", layout) _, err = os.Stat(home) if err == nil { // home already exists @@ -157,7 +157,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { } // create the users dir - parent := path.Dir(home) + parent := filepath.Dir(home) err = os.MkdirAll(parent, 0700) if err != nil { // MkdirAll will return success on mkdir over an existing directory. @@ -166,7 +166,7 @@ func (fs *ocisfs) CreateHome(ctx context.Context) error { // create a directory node (with children subfolder) nodeID := uuid.Must(uuid.NewV4()).String() - err = os.MkdirAll(path.Join(fs.conf.DataDirectory, "nodes", nodeID, "children"), 0700) + err = os.MkdirAll(filepath.Join(fs.conf.Root, "nodes", nodeID, "children"), 0700) if err != nil { return errors.Wrap(err, "ocisfs: error node dir") } @@ -187,7 +187,7 @@ func (fs *ocisfs) GetHome(ctx context.Context) (string, error) { return "", err } layout := templates.WithUser(u, fs.conf.UserLayout) - return path.Join(fs.conf.DataDirectory, layout), nil // TODO use a namespace? + return filepath.Join(fs.conf.Root, layout), nil // TODO use a namespace? } // Tree persistence @@ -198,12 +198,11 @@ func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str } func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { - parent := path.Dir(fn) - var in string - if in, err = fs.pw.Wrap(ctx, parent); err != nil { + var node *NodeInfo + if node, err = fs.pw.Wrap(ctx, fn); err != nil { return } - return fs.tp.CreateDir(ctx, in, path.Base(fn)) + return fs.tp.CreateDir(ctx, node) } func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { @@ -211,45 +210,51 @@ func (fs *ocisfs) CreateReference(ctx context.Context, path string, targetURI *u } func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { - var oldInternal, newInternal string - if oldInternal, err = fs.pw.Resolve(ctx, oldRef); err != nil { + var oldNode, newNode *NodeInfo + if oldNode, err = fs.pw.Resolve(ctx, oldRef); err != nil { + return + } + if !oldNode.Exists { + err = errtypes.NotFound(filepath.Join(oldNode.ParentID, oldNode.Name)) return } - if newInternal, err = fs.pw.Resolve(ctx, newRef); err != nil { - // TODO might not exist ... + if newNode, err = fs.pw.Resolve(ctx, newRef); err != nil { return } - return fs.tp.Move(ctx, oldInternal, newInternal) + return fs.tp.Move(ctx, oldNode, newNode) } -func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference) (ri *provider.ResourceInfo, err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { +func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { return } - var md os.FileInfo - md, err = fs.tp.GetMD(ctx, in) - if err != nil { - return nil, err + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return } - return fs.normalize(ctx, md, in) + return fs.normalize(ctx, node) } -func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference) (finfos []*provider.ResourceInfo, err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { +func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - var mds []os.FileInfo - mds, err = fs.tp.ListFolder(ctx, in) + var children []*NodeInfo + children, err = fs.tp.ListFolder(ctx, node) if err != nil { return } - for _, md := range mds { + for _, child := range children { var ri *provider.ResourceInfo - ri, err = fs.normalize(ctx, md, path.Join(in, "children", md.Name())) + ri, err = fs.normalize(ctx, child) if err != nil { return } @@ -259,11 +264,15 @@ func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference) (finf } func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { - var in string - if in, err = fs.pw.Resolve(ctx, ref); err != nil { + var node *NodeInfo + if node, err = fs.pw.Resolve(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) return } - return fs.tp.Delete(ctx, in) + return fs.tp.Delete(ctx, node) } // arbitrary metadata persistence @@ -278,92 +287,22 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe // Data persistence -func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - var in string // the internal path of the file node - - p := ref.GetPath() - if p != "" { - if p == "/" { - return fmt.Errorf("cannot upload into folder node /") - } - parent := path.Dir(p) - name := path.Base(p) - - inp, err := fs.pw.Wrap(ctx, parent) - - if _, err := os.Stat(path.Join(inp, "children")); os.IsNotExist(err) { - // TODO double check if node is a file - return fmt.Errorf("cannot upload into folder node " + path.Join(inp, "children")) - } - childEntry := path.Join(inp, "children", name) - - // try to determine nodeID by reading link - link, err := os.Readlink(childEntry) - if os.IsNotExist(err) { - // create a new file node - nodeID := uuid.Must(uuid.NewV4()).String() - - in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) - - err = os.MkdirAll(in, 0700) - if err != nil { - return errors.Wrap(err, "ocisfs: could not create node dir") - } - // create back link - // we are not only linking back to the parent, but also to the filename - link = "../" + path.Base(inp) + "/children/" + name - err = os.Symlink(link, path.Join(in, "parentname")) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink parent node") - } - - // link child name to node - err = os.Symlink("../../"+nodeID, path.Join(inp, "children", name)) - if err != nil { - return errors.Wrap(err, "ocisfs: could not symlink child entry") - } - } else { - // the nodeID is in the link - // TODO check if link has correct beginning? - nodeID := path.Base(link) - in = path.Join(fs.conf.DataDirectory, "nodes", nodeID) - } - } else if ref.GetId() != nil { - var err error - if in, err = fs.pw.WrapID(ctx, ref.GetId()); err != nil { - return err - } - } else { - return fmt.Errorf("invalid reference %+v", ref) - } - - tmp, err := ioutil.TempFile(in, "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "ocisfs: error creating tmp fn at "+in) - } - - _, err = io.Copy(tmp, r) - if err != nil { - return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) - } - - // TODO move old content to version - _ = os.RemoveAll(path.Join(in, "content")) - - err = os.Rename(tmp.Name(), path.Join(in, "content")) - if err != nil { - return err - } - return fs.tp.Propagate(ctx, in) +func (fs *ocisfs) ContentPath(node *NodeInfo) string { + return filepath.Join(fs.conf.Root, "nodes", node.ID, "content") } func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - in, err := fs.pw.Resolve(ctx, ref) + node, err := fs.pw.Resolve(ctx, ref) if err != nil { return nil, errors.Wrap(err, "ocisfs: error resolving ref") } - contentPath := path.Join(in, "content") + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return nil, err + } + + contentPath := fs.ContentPath(node) r, err := os.Open(contentPath) if err != nil { @@ -435,30 +374,33 @@ func getUser(ctx context.Context) (*userpb.User, error) { return u, nil } -func (fs *ocisfs) normalize(ctx context.Context, fi os.FileInfo, internal string) (ri *provider.ResourceInfo, err error) { +func (fs *ocisfs) normalize(ctx context.Context, node *NodeInfo) (ri *provider.ResourceInfo, err error) { var fn string - fn, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) + fn, err = fs.pw.Unwrap(ctx, node) if err != nil { return nil, err } - // TODO GetMD should return the correct fileinfo + nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) + + var fi os.FileInfo nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID - if fi, err = os.Stat(path.Join(internal, "content")); err == nil { + if fi, err = os.Stat(filepath.Join(nodePath, "content")); err == nil { nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - } else if fi, err = os.Stat(path.Join(internal, "children")); err == nil { + } else if fi, err = os.Stat(filepath.Join(nodePath, "children")); err == nil { nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else if fi, err = os.Stat(path.Join(internal, "reference")); err == nil { + } else if fi, err = os.Stat(filepath.Join(nodePath, "reference")); err == nil { // TODO handle references nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE } var etag []byte - if etag, err = xattr.Get(internal, "user.ocis.etag"); err != nil { + // TODO store etag in node folder or on content and child nodes? + if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { logger.New().Error().Err(err).Msg("could not read etag") } ri = &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: path.Base(internal)}, + Id: &provider.ResourceId{OpaqueId: node.ID}, Path: fn, Type: nodeType, Etag: string(etag), diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go index 0dba13d5d74..63ed78568b9 100644 --- a/pkg/storage/fs/ocis/path.go +++ b/pkg/storage/fs/ocis/path.go @@ -4,19 +4,20 @@ import ( "context" "fmt" "os" - "path" + "path/filepath" "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/pkg/errors" ) +// Path implements transformations from filepath to node and back type Path struct { // ocis fs works on top of a dir of uuid nodes - DataDirectory string `mapstructure:"data_directory"` + Root string `mapstructure:"root"` // UserLayout wraps the internal path with user information. // Example: if conf.Namespace is /ocis/user and received path is /docs @@ -28,8 +29,8 @@ type Path struct { EnableHome bool `mapstructure:"enable_home"` } -// Resolve takes in a request path or request id and converts it to an internal path. -func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (string, error) { +// Resolve takes in a request path or request id and converts it to a NodeInfo +func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (*NodeInfo, error) { if ref.GetPath() != "" { return pw.Wrap(ctx, ref.GetPath()) } @@ -39,11 +40,12 @@ func (pw *Path) Resolve(ctx context.Context, ref *provider.Reference) (string, e } // reference is invalid - return "", fmt.Errorf("invalid reference %+v", ref) + return nil, fmt.Errorf("invalid reference %+v", ref) } -func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error) { - var link, nodeID, root string +// Wrap converts a filename into a NodeInfo +func (pw *Path) Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) { + var link, root string if fn == "" { fn = "/" } @@ -58,33 +60,16 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error } layout := templates.WithUser(u, pw.UserLayout) - root = path.Join(pw.DataDirectory, "users", layout) + root = filepath.Join(pw.Root, "users", layout) } else { // start at the storage root node - root = path.Join(pw.DataDirectory, "nodes", "root") + root = filepath.Join(pw.Root, "nodes/root") } + node, err = pw.ReadRootLink(root) // The symlink contains the nodeID - link, err = os.Readlink(root) - if os.IsNotExist(err) { - err = errtypes.NotFound(fn) - return - } if err != nil { - err = errors.Wrap(err, "ocisfs: Wrap: readlink error") - return - } - - // extract the nodeID - if strings.HasPrefix(link, "../nodes/") { // TODO does not take into account the template - nodeID = link[9:] - if strings.Contains(nodeID, "/") { - err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) // TODO allow this to distribute nodeids over multiple folders - return - } - } else { - err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) return } @@ -92,9 +77,20 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error // we need to walk the path segments := strings.Split(strings.TrimLeft(fn, "/"), "/") for i := range segments { - link, err = os.Readlink(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + node.ParentID = node.ID + node.ID = "" + node.Name = segments[i] + + link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) if os.IsNotExist(err) { - err = errtypes.NotFound(path.Join(pw.DataDirectory, "nodes", nodeID, "children", segments[i])) + node.Exists = false + // if this is the last segment we can use it as the node name + if i == len(segments)-1 { + err = nil + return + } + + err = errtypes.NotFound(filepath.Join(pw.Root, "nodes", node.ParentID, "children", node.Name)) return } if err != nil { @@ -102,11 +98,7 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error return } if strings.HasPrefix(link, "../../") { - nodeID = link[6:] - if strings.Contains(nodeID, "/") { - err = fmt.Errorf("ocisfs: node id must not contain / %+v", nodeID) - return - } + node.ID = filepath.Base(link) } else { err = fmt.Errorf("ocisfs: expected '../../ prefix, got' %+v", link) return @@ -114,55 +106,84 @@ func (pw *Path) Wrap(ctx context.Context, fn string) (internal string, err error } } - internal = path.Join(pw.DataDirectory, "nodes", nodeID) return } // WrapID returns the internal path for the id -func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (string, error) { - if id == nil || id.GetOpaqueId() == "" { - return "", fmt.Errorf("invalid resource id %+v", id) +func (pw *Path) WrapID(ctx context.Context, id *provider.ResourceId) (*NodeInfo, error) { + if id == nil || id.OpaqueId == "" { + return nil, fmt.Errorf("invalid resource id %+v", id) } - return path.Join(pw.DataDirectory, "nodes", id.GetOpaqueId()), nil + return &NodeInfo{ID: id.OpaqueId}, nil } -func (pw *Path) Unwrap(ctx context.Context, internal string) (external string, err error) { - var link string +func (pw *Path) Unwrap(ctx context.Context, ni *NodeInfo) (external string, err error) { for err == nil { - link, err = os.Readlink(path.Join(internal, "parentname")) + err = pw.FillParentAndName(ni) if os.IsNotExist(err) { err = nil return } if err != nil { - err = errors.Wrap(err, "ocisfs: getNode: readlink error") + err = errors.Wrap(err, "ocisfs: Unwrap: could not fill node") return } - parentID := path.Base(path.Dir(path.Dir(link))) - internal = path.Join(pw.DataDirectory, "nodes", parentID) - external = path.Join(path.Base(link), external) + external = filepath.Join(ni.Name, external) + ni.BecomeParent() } return } -// ReadParentName reads the symbolic link and extracts the parnetNodeID and the name of the child -func (pw *Path) ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) { +// FillParentAndName reads the symbolic link and extracts the parent ID and the name of the node if necessary +func (pw *Path) FillParentAndName(node *NodeInfo) (err error) { + + if node == nil || node.ID == "" { + err = fmt.Errorf("ocisfs: invalid node info '%+v'", node) + } + // check if node is already filled + if node.ParentID != "" && node.Name != "" { + return + } + + var link string // The parentname symlink looks like `../76455834-769e-412a-8a01-68f265365b79/children/myname.txt` - link, err := os.Readlink(path.Join(internal, "parentname")) - if os.IsNotExist(err) { - err = errtypes.NotFound(internal) + link, err = os.Readlink(filepath.Join(pw.Root, "nodes", node.ID, "parentname")) + if err != nil { return } // check the link follows the correct schema // TODO count slashes if strings.HasPrefix(link, "../") { - name = path.Base(link) - parentNodeID = path.Base(path.Dir(path.Dir(link))) + node.Name = filepath.Base(link) + node.ParentID = filepath.Base(filepath.Dir(filepath.Dir(link))) + node.Exists = true } else { err = fmt.Errorf("ocisfs: expected '../' prefix, got '%+v'", link) return } return } + +// ReadRootLink reads the symbolic link and extracts the node id +func (pw *Path) ReadRootLink(root string) (node *NodeInfo, err error) { + + // A root symlink looks like `../nodes/76455834-769e-412a-8a01-68f265365b79` + link, err := os.Readlink(root) + if os.IsNotExist(err) { + err = errtypes.NotFound(root) + return + } + + // extract the nodeID + if strings.HasPrefix(link, "../nodes/") { + node = &NodeInfo{ + ID: filepath.Base(link), + Exists: true, + } + } else { + err = fmt.Errorf("ocisfs: expected '../nodes/ prefix, got' %+v", link) + } + return +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go index 9e2fae05f34..0698585a3a3 100644 --- a/pkg/storage/fs/ocis/persistence.go +++ b/pkg/storage/fs/ocis/persistence.go @@ -8,22 +8,29 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) +// TreePersistence is used to manage a tree hierarchy type TreePersistence interface { GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - GetMD(ctx context.Context, internal string) (os.FileInfo, error) - ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) - CreateDir(ctx context.Context, internal string, newName string) (err error) + GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) + ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) + CreateDir(ctx context.Context, node *NodeInfo) (err error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error - Move(ctx context.Context, oldInternal string, newInternal string) (err error) - Delete(ctx context.Context, internal string) (err error) + Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) + Delete(ctx context.Context, node *NodeInfo) (err error) - Propagate(ctx context.Context, internal string) (err error) + Propagate(ctx context.Context, node *NodeInfo) (err error) } +// PathWrapper is used to encapsulate path transformations type PathWrapper interface { - Resolve(ctx context.Context, ref *provider.Reference) (internal string, err error) - WrapID(ctx context.Context, id *provider.ResourceId) (internal string, err error) - Wrap(ctx context.Context, fn string) (internal string, err error) - Unwrap(ctx context.Context, np string) (external string, err error) - ReadParentName(ctx context.Context, internal string) (parentNodeID string, name string, err error) // Tree persistence? + Resolve(ctx context.Context, ref *provider.Reference) (node *NodeInfo, err error) + WrapID(ctx context.Context, id *provider.ResourceId) (node *NodeInfo, err error) + + // Wrap returns a NodeInfo object: + // - if the node exists with the node id, name and parent + // - if only the parent exists, the node id is empty + Wrap(ctx context.Context, fn string) (node *NodeInfo, err error) + Unwrap(ctx context.Context, node *NodeInfo) (external string, err error) + FillParentAndName(node *NodeInfo) (err error) // Tree persistence? + ReadRootLink(root string) (node *NodeInfo, err error) } diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index ceb1d98b081..39520c410cc 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -3,13 +3,11 @@ package ocis import ( "context" "encoding/hex" - "fmt" "io/ioutil" "math/rand" "net/url" "os" - "path" - "strings" + "path/filepath" "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -20,11 +18,13 @@ import ( "github.com/rs/zerolog/log" ) +// Tree manages a hierarchical tree type Tree struct { pw PathWrapper DataDirectory string } +// NewTree creates a new Tree instance func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { return &Tree{ pw: pw, @@ -32,13 +32,14 @@ func NewTree(pw PathWrapper, dataDirectory string) (TreePersistence, error) { }, nil } -func (fs *Tree) GetMD(ctx context.Context, internal string) (os.FileInfo, error) { - md, err := os.Stat(internal) +// GetMD returns the metadata of a node in the tree +func (fs *Tree) GetMD(ctx context.Context, node *NodeInfo) (os.FileInfo, error) { + md, err := os.Stat(filepath.Join(fs.DataDirectory, "nodes", node.ID)) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(internal) + return nil, errtypes.NotFound(node.ID) } - return nil, errors.Wrap(err, "tree: error stating "+internal) + return nil, errors.Wrap(err, "tree: error stating "+node.ID) } return md, nil @@ -46,78 +47,149 @@ func (fs *Tree) GetMD(ctx context.Context, internal string) (os.FileInfo, error) // GetPathByID returns the fn pointed by the file id, without the internal namespace func (fs *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { - var internal string - internal, err = fs.pw.WrapID(ctx, id) + var node *NodeInfo + node, err = fs.pw.WrapID(ctx, id) if err != nil { return } - relativeExternalPath, err = fs.pw.Unwrap(ctx, path.Join("/", internal)) - if !strings.HasPrefix(relativeExternalPath, fs.DataDirectory) { - return "", fmt.Errorf("ocisfs: GetPathByID wrong prefix") - } - - relativeExternalPath = strings.TrimPrefix(relativeExternalPath, fs.DataDirectory) + relativeExternalPath, err = fs.pw.Unwrap(ctx, node) return } -func (fs *Tree) CreateDir(ctx context.Context, internal string, newName string) (err error) { +// CreateDir creates a new directory entry in the tree +func (fs *Tree) CreateDir(ctx context.Context, node *NodeInfo) (err error) { - internalChild := path.Join(internal, "children", newName) - _, err = os.Stat(internalChild) - if err == nil { // child already exists - return nil + // TODO always try to fill node? + if node.Exists || node.ID != "" { // child already exists + return } // create a directory node (with children subfolder) - nodeID := uuid.Must(uuid.NewV4()).String() + node.ID = uuid.Must(uuid.NewV4()).String() - nodePath := path.Join(fs.DataDirectory, "nodes", nodeID) + newPath := filepath.Join(fs.DataDirectory, "nodes", node.ID) - err = os.MkdirAll(path.Join(nodePath, "children"), 0700) + err = os.MkdirAll(filepath.Join(newPath, "children"), 0700) if err != nil { return errors.Wrap(err, "ocisfs: could not create node dir") } // create back link // we are not only linking back to the parent, but also to the filename - err = os.Symlink("../"+path.Base(internal)+"/children/"+newName, path.Join(nodePath, "parentname")) + err = os.Symlink("../"+node.ParentID+"/children/"+node.Name, filepath.Join(newPath, "parentname")) if err != nil { return errors.Wrap(err, "ocisfs: could not symlink parent node") } - // link child name to node - err = os.Symlink("../../"+nodeID, internalChild) + // make child appear in listings + err = os.Symlink("../../"+node.ID, filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) if err != nil { return } - return fs.Propagate(ctx, nodePath) + return fs.Propagate(ctx, node) } +// CreateReference creates a new reference entry in the tree func (fs *Tree) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { return errtypes.NotSupported("operation not supported: CreateReference") } -func (fs *Tree) Move(ctx context.Context, oldInternal string, newInternal string) (err error) { - oldParentID, oldName, err = fs.pw.ReadParentName(ctx, oldInternal) +// Move replaces the target with the source +func (fs *Tree) Move(ctx context.Context, oldNode *NodeInfo, newNode *NodeInfo) (err error) { + err = fs.pw.FillParentAndName(newNode) + if os.IsNotExist(err) { + err = nil + return + } + + // if target exists delete it without trashing it + if newNode.Exists { + err = fs.pw.FillParentAndName(newNode) + if os.IsNotExist(err) { + err = nil + return + } + if err := os.RemoveAll(filepath.Join(fs.DataDirectory, "nodes", newNode.ID)); err != nil { + return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) + } + } + // are we renaming? + if oldNode.ParentID == newNode.ParentID { + + nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) + + // update back link + // we are not only linking back to the parent, but also to the filename + err = os.Remove(filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not remove parent link") + } + err = os.Symlink("../"+oldNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) + if err != nil { + return errors.Wrap(err, "ocisfs: could not symlink parent") + } + + // rename child + err = os.Rename( + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not rename symlink") + } + return fs.Propagate(ctx, oldNode) + } + // we are moving the node to a new parent, any target has been removed + // bring old node to the new parent + + nodePath := filepath.Join(fs.DataDirectory, "nodes", oldNode.ID) + + // update back link + // we are not only linking back to the parent, but also to the filename + err = os.Remove(filepath.Join(nodePath, "parentname")) if err != nil { - return err + return errors.Wrap(err, "ocisfs: could not remove parent link") } - newParentID, newName, err = fs.pw.ReadParentName(ctx, oldInternal) + err = os.Symlink("../"+newNode.ParentID+"/children/"+newNode.Name, filepath.Join(nodePath, "parentname")) if err != nil { - return err + return errors.Wrap(err, "ocisfs: could not symlink parent") + } + + // rename child + err = os.Rename( + filepath.Join(fs.DataDirectory, "nodes", oldNode.ParentID, "children", oldNode.Name), + filepath.Join(fs.DataDirectory, "nodes", newNode.ParentID, "children", newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not rename symlink") + } + + // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? + // collect in a list, then only stat each node once + // also do this in a go routine ... webdav should check the etag async + err = fs.Propagate(ctx, oldNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } - if err := os.Rename(oldInternal, newInternal); err != nil { - return errors.Wrap(err, "localfs: error moving "+oldInternal+" to "+newInternal) + err = fs.Propagate(ctx, newNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate old node") } - return errtypes.NotSupported("operation not supported: Move") + return nil } -func (fs *Tree) ListFolder(ctx context.Context, internal string) ([]os.FileInfo, error) { +// ChildrenPath returns the absolute path to childrens in a node +// TODO move to node? +func (fs *Tree) ChildrenPath(node *NodeInfo) string { + return filepath.Join(fs.DataDirectory, "nodes", node.ID, "children") +} - children := path.Join(internal, "children") +// ListFolder lists the children inside a folder +func (fs *Tree) ListFolder(ctx context.Context, node *NodeInfo) ([]*NodeInfo, error) { - mds, err := ioutil.ReadDir(children) + children := fs.ChildrenPath(node) + f, err := os.Open(children) if err != nil { if os.IsNotExist(err) { return nil, errtypes.NotFound(children) @@ -125,29 +197,44 @@ func (fs *Tree) ListFolder(ctx context.Context, internal string) ([]os.FileInfo, return nil, errors.Wrap(err, "tree: error listing "+children) } - return mds, nil -} + names, err := f.Readdirnames(0) + nodes := []*NodeInfo{} + for i := range names { + link, err := os.Readlink(filepath.Join(children, names[i])) + if err != nil { + // TODO log + continue + } + n := &NodeInfo{ + ParentID: node.ID, + ID: filepath.Base(link), + Name: names[i], + Exists: true, // TODO + } -func (fs *Tree) Delete(ctx context.Context, internal string) (err error) { - // resolve the parent + nodes = append(nodes, n) + } + return nodes, nil +} - // The nodes parentname symlink contains the nodeID and the file name - link, err := os.Readlink(path.Join(internal, "parentname")) +// Delete deletes a node in the tree +func (fs *Tree) Delete(ctx context.Context, node *NodeInfo) (err error) { + err = fs.pw.FillParentAndName(node) if os.IsNotExist(err) { - err = errtypes.NotFound(internal) + err = nil + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Delete: FillParentAndName error") return } // remove child entry from dir - childName := path.Base(link) - parentNodeID := path.Base(path.Dir(path.Dir(link))) - os.Remove(path.Join(fs.DataDirectory, "nodes", parentNodeID, "children", childName)) + os.Remove(filepath.Join(fs.DataDirectory, "nodes", node.ParentID, "children", node.Name)) - nodeID := path.Base(internal) - - src := path.Join(fs.DataDirectory, "nodes", nodeID) - trashpath := path.Join(fs.DataDirectory, "trash/files", nodeID) + src := filepath.Join(fs.DataDirectory, "nodes", node.ID) + trashpath := filepath.Join(fs.DataDirectory, "trash/files", node.ID) err = os.Rename(src, trashpath) if err != nil { return @@ -156,16 +243,18 @@ func (fs *Tree) Delete(ctx context.Context, internal string) (err error) { // write a trash info ... slightly violating the freedesktop trash spec t := time.Now() // TODO store the original Path - info := []byte("[Trash Info]\nParentID=" + parentNodeID + "\nDeletionDate=" + t.Format(time.RFC3339)) - infoPath := path.Join(fs.DataDirectory, "trash/info", nodeID+".trashinfo") + info := []byte("[Trash Info]\nParentID=" + node.ParentID + "\nDeletionDate=" + t.Format(time.RFC3339)) + infoPath := filepath.Join(fs.DataDirectory, "trash/info", node.ID+".trashinfo") err = ioutil.WriteFile(infoPath, info, 0700) if err != nil { return } - return fs.Propagate(ctx, path.Join(fs.DataDirectory, "nodes", parentNodeID)) + + return fs.Propagate(ctx, &NodeInfo{ID: node.ParentID}) } -func (fs *Tree) Propagate(ctx context.Context, internal string) (err error) { +// Propagate propagates changes to the root of the tree +func (fs *Tree) Propagate(ctx context.Context, node *NodeInfo) (err error) { // generate an etag bytes := make([]byte, 16) if _, err := rand.Read(bytes); err != nil { @@ -173,22 +262,21 @@ func (fs *Tree) Propagate(ctx context.Context, internal string) (err error) { } // store in extended attribute etag := hex.EncodeToString(bytes) - var link string for err == nil { - if err := xattr.Set(internal, "user.ocis.etag", []byte(etag)); err != nil { + if err := xattr.Set(filepath.Join(fs.DataDirectory, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { log.Error().Err(err).Msg("error storing file id") } - link, err = os.Readlink(path.Join(internal, "parentname")) + err = fs.pw.FillParentAndName(node) if os.IsNotExist(err) { err = nil return } if err != nil { - err = errors.Wrap(err, "ocisfs: getNode: readlink error") + err = errors.Wrap(err, "ocisfs: Propagate: readlink error") return } - parentID := path.Base(path.Dir(path.Dir(link))) - internal = path.Join(fs.DataDirectory, "nodes", parentID) + + node.BecomeParent() } return } diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go new file mode 100644 index 00000000000..53e8a4f4101 --- /dev/null +++ b/pkg/storage/fs/ocis/upload.go @@ -0,0 +1,438 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus + +func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + + node, err := fs.pw.Resolve(ctx, ref) + if err != nil { + return err + } + + if !node.Exists { + err = node.Create(fs.conf.Root) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node") + } + } + nodePath := filepath.Join(fs.conf.Root, "nodes", node.ID) + + tmp, err := ioutil.TempFile(nodePath, "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocisfs: error creating tmp fn at "+nodePath) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) + } + + // TODO move old content to version + //_ = os.RemoveAll(path.Join(nodePath, "content")) + + err = os.Rename(tmp.Name(), filepath.Join(nodePath, "content")) + if err != nil { + return err + } + return fs.tp.Propagate(ctx, node) + +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (uploadID string, err error) { + var relative string // the internal path of the file node + + node, err := fs.pw.Resolve(ctx, ref) + if err != nil { + return "", err + } + relative, err = fs.pw.Unwrap(ctx, node) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": node.Name, + "dir": filepath.Dir(relative), + }, + Size: uploadLength, + } + + if metadata != nil && metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *ocisfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + composer.UseConcater(fs) + composer.UseLengthDeferrer(fs) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("ocisfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("ocisfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("ocisfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + node, err := fs.pw.Wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Interface("node", node).Msg("ocisfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error resolving upload path") + } + usr := user.ContextMustGetUser(ctx) + info.Storage = map[string]string{ + "Type": "OCISStore", + "BinPath": binPath, + + "NodeId": node.ID, + "NodeParentId": node.ParentID, + "NodeName": node.Name, + + "Idp": usr.Id.Idp, + "UserId": usr.Id.OpaqueId, + "UserName": usr.Username, + + "LogLevel": log.GetLevel().String(), + } + // Create binary file in the upload folder with no content + log.Debug().Interface("info", info).Msg("ocisfs: built storage info") + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: filepath.Join(fs.conf.Root, "uploads", info.ID+".info"), + fs: fs, + ctx: ctx, + } + + if !info.SizeIsDeferred && info.Size == 0 { + log.Debug().Interface("info", info).Msg("ocisfs: finishing upload for empty file") + // no need to create info file and finish directly + err := u.FinishUpload(ctx) + if err != nil { + return nil, err + } + return u, nil + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *ocisfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + return "", err + } + layout := templates.WithUser(u, fs.conf.UserLayout) + return filepath.Join(fs.conf.Root, layout, "uploads", uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *ocisfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + infoPath := filepath.Join(fs.conf.Root, "uploads", id+".info") + + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(info.Storage["BinPath"]) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + + ctx = user.ContextSetUser(ctx, u) + // TODO configure the logger the same way ... store and add traceid in file info + + var opts []logger.Option + opts = append(opts, logger.WithLevel(info.Storage["LogLevel"])) + opts = append(opts, logger.WithWriter(os.Stderr, logger.ConsoleMode)) + l := logger.New(opts...) + + sub := l.With().Int("pid", os.Getpid()).Logger() + + ctx = appctx.WithLogger(ctx, &sub) + + return &fileUpload{ + info: info, + binPath: info.Storage["BinPath"], + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *ocisfs + // a context with a user + // TODO add logger as well? + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk + + return n, err +} + +// GetReader returns an io.Reader for the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + node := &NodeInfo{ + ID: upload.info.Storage["NodeId"], + ParentID: upload.info.Storage["NodeParentId"], + Name: upload.info.Storage["NodeName"], + } + + if node.ID == "" { + err := node.Create(upload.fs.conf.Root) + if err != nil { + return errors.Wrap(err, "ocisfs: could not create node") + } + } + contentPath := filepath.Join(upload.fs.conf.Root, "nodes", node.ID, "content") + + log := appctx.GetLogger(upload.ctx) + err := os.Rename(upload.binPath, contentPath) + if err != nil { + log.Err(err).Interface("info", upload.info). + Str("binPath", upload.binPath). + Str("contentPath", contentPath). + Msg("ocisfs: could not rename") + return err + } + + // only delete the upload if it was successfully written to the storage + if err := os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not delete upload info") + return err + } + } + // use set arbitrary metadata? + /*if upload.info.MetaData["mtime"] != "" { + err := upload.fs.SetMtime(ctx, np, upload.info.MetaData["mtime"]) + if err != nil { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not set mtime metadata") + return err + } + }*/ + + return upload.fs.tp.Propagate(upload.ctx, node) +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returns a TerminatableUpload +func (fs *ocisfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + if err := os.Remove(upload.binPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + return nil +} + +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// - the storage needs to implement AsLengthDeclarableUpload +// - the upload needs to implement DeclareLength + +// AsLengthDeclarableUpload returns a LengthDeclarableUpload +func (fs *ocisfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// DeclareLength updates the upload length information +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// - the storage needs to implement AsConcatableUpload +// - the upload needs to implement ConcatUploads + +// AsConcatableUpload returns a ConcatableUpload +func (fs *ocisfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { + return upload.(*fileUpload) +} + +// ConcatUploads concatenates multiple uploads +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +}