diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9f2f66..96d62005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added NewFile option for Content-Type. - Windows support in the os backend. ### Fixed - Ability to run all unit tests on Windows. +- Deprecated delete.WithDeleteAllVersions in favor of delete.WithAllVersions. ## [6.24.0] - 2024-12-16 ### Security diff --git a/backend/azure/client.go b/backend/azure/client.go index 83023749..e037626e 100644 --- a/backend/azure/client.go +++ b/backend/azure/client.go @@ -28,7 +28,7 @@ type Client interface { // Upload should create or update the blob specified by the file parameter with the contents of the content // parameter - Upload(file vfs.File, content io.ReadSeeker) error + Upload(file vfs.File, content io.ReadSeeker, contentType string) error // Download should return a reader for the blob specified by the file parameter Download(file vfs.File) (io.ReadCloser, error) @@ -102,7 +102,7 @@ func (a *DefaultClient) Properties(containerURI, filePath string) (*BlobProperti } // Upload uploads a new file to Azure Blob Storage -func (a *DefaultClient) Upload(file vfs.File, content io.ReadSeeker) error { +func (a *DefaultClient) Upload(file vfs.File, content io.ReadSeeker, contentType string) error { URL, err := url.Parse(file.Location().(*Location).ContainerURL()) if err != nil { return err @@ -110,7 +110,7 @@ func (a *DefaultClient) Upload(file vfs.File, content io.ReadSeeker) error { containerURL := azblob.NewContainerURL(*URL, a.pipeline) blobURL := containerURL.NewBlockBlobURL(utils.RemoveLeadingSlash(file.Path())) - _, err = blobURL.Upload(context.Background(), content, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, + _, err = blobURL.Upload(context.Background(), content, azblob.BlobHTTPHeaders{ContentType: contentType}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.DefaultAccessTier, nil, azblob.ClientProvidedKeyOptions{}, azblob.ImmutabilityPolicyOptions{}) return err } diff --git a/backend/azure/client_integration_test.go b/backend/azure/client_integration_test.go index 8e4a628b..b1139a83 100644 --- a/backend/azure/client_integration_test.go +++ b/backend/azure/client_integration_test.go @@ -6,6 +6,7 @@ package azure import ( "context" "fmt" + "io" "net/url" "os" "strings" @@ -64,7 +65,7 @@ func (s *ClientIntegrationTestSuite) TestAllTheThings_FileWithNoPath() { s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials") // Create the new file - err = client.Upload(f, strings.NewReader("Hello world!")) + err = client.Upload(f, strings.NewReader("Hello world!"), "") s.NoError(err, "The file should be successfully uploaded to azure") // make sure it exists @@ -112,7 +113,7 @@ func (s *ClientIntegrationTestSuite) TestAllTheThings_FileWithPath() { s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials") // create a new file - err = client.Upload(f, strings.NewReader("Hello world!")) + err = client.Upload(f, strings.NewReader("Hello world!"), "") s.NoError(err, "The file should be successfully uploaded to azure") // check to see if it exists @@ -144,11 +145,11 @@ func (s *ClientIntegrationTestSuite) TestDeleteAllVersions() { s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials") // Create the new file - err = client.Upload(f, strings.NewReader("Hello!")) + err = client.Upload(f, strings.NewReader("Hello!"), "") s.NoError(err, "The file should be successfully uploaded to azure") // Recreate the file - err = client.Upload(f, strings.NewReader("Hello world!")) + err = client.Upload(f, strings.NewReader("Hello world!"), "") s.NoError(err, "The file should be successfully uploaded to azure") // make sure it exists @@ -172,7 +173,7 @@ func (s *ClientIntegrationTestSuite) TestProperties() { client, err := fs.Client() s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials") - err = client.Upload(f, strings.NewReader("Hello world!")) + err = client.Upload(f, strings.NewReader("Hello world!"), "") s.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error") props, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path()) s.NoError(err, "The file exists so we shouldn't get an error") @@ -188,7 +189,7 @@ func (s *ClientIntegrationTestSuite) TestProperties_Location() { l, _ := fs.NewLocation("test-container", "/") client, _ := fs.Client() - err = client.Upload(f, strings.NewReader("Hello world!")) + err = client.Upload(f, strings.NewReader("Hello world!"), "") s.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error") props, err := client.Properties(l.URI(), "") @@ -226,7 +227,7 @@ func (s *ClientIntegrationTestSuite) TestTouch_NonexistentContainer() { client, err := fs.Client() s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials") - err = client.Upload(f, strings.NewReader("")) + err = client.Upload(f, strings.NewReader(""), "") s.Error(err, "The container doesn't exist so we should get an error") } @@ -237,7 +238,7 @@ func (s *ClientIntegrationTestSuite) TestTouch_FileAlreadyExists() { client, err := fs.Client() s.NoError(err) - err = client.Upload(f, strings.NewReader("One fish, two fish, red fish, blue fish.")) + err = client.Upload(f, strings.NewReader("One fish, two fish, red fish, blue fish."), "") s.NoError(err) originalProps, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path()) s.NoError(err, "Should get properties back from azure with no error") diff --git a/backend/azure/file.go b/backend/azure/file.go index 1c74a066..58316607 100644 --- a/backend/azure/file.go +++ b/backend/azure/file.go @@ -15,6 +15,7 @@ import ( "github.com/c2fo/vfs/v6/backend" "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -23,6 +24,7 @@ type File struct { fileSystem *FileSystem container string name string + opts []options.NewFileOption tempFile *os.File isDirty bool } @@ -47,7 +49,16 @@ func (f *File) Close() error { } if f.isDirty { - if err := client.Upload(f, f.tempFile); err != nil { + var contentType string + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + contentType = *(*string)(o) + default: + } + } + + if err := client.Upload(f, f.tempFile, contentType); err != nil { return utils.WrapCloseError(err) } } @@ -141,7 +152,7 @@ func (f *File) Location() vfs.Location { // name at the given location. If the given location is also azure, the azure API for copying // files will be utilized, otherwise, standard io.Copy will be done to the new file. func (f *File) CopyToLocation(location vfs.Location) (vfs.File, error) { - newFile, err := location.NewFile(utils.RemoveLeadingSlash(f.Name())) + newFile, err := location.NewFile(utils.RemoveLeadingSlash(f.Name()), f.opts...) if err != nil { return nil, err } @@ -225,7 +236,7 @@ func (f *File) MoveToFile(file vfs.File) error { } // Delete deletes the file. -// If DeleteAllVersions option is provided, each version of the file is deleted. NOTE: if soft deletion is enabled, +// If delete.AllVersions option is provided, each version of the file is deleted. NOTE: if soft deletion is enabled, // it will mark all versions as soft deleted, and they will be removed by Azure as per soft deletion policy. // Returns any error returned by the API. func (f *File) Delete(opts ...options.DeleteOption) error { @@ -238,11 +249,11 @@ func (f *File) Delete(opts ...options.DeleteOption) error { return err } - var deleteAllVersions bool + var allVersions bool for _, o := range opts { switch o.(type) { - case delete.DeleteAllVersions: - deleteAllVersions = true + case delete.AllVersions, delete.DeleteAllVersions: + allVersions = true default: } } @@ -251,7 +262,7 @@ func (f *File) Delete(opts ...options.DeleteOption) error { return err } - if deleteAllVersions { + if allVersions { return client.DeleteAllVersions(f) } @@ -308,7 +319,16 @@ func (f *File) Touch() error { } if !exists { - return client.Upload(f, strings.NewReader("")) + var contentType string + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + contentType = *(*string)(o) + default: + } + } + + return client.Upload(f, strings.NewReader(""), contentType) } props, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path()) diff --git a/backend/azure/fileSystem.go b/backend/azure/fileSystem.go index 87bcb189..c6d5437c 100644 --- a/backend/azure/fileSystem.go +++ b/backend/azure/fileSystem.go @@ -10,6 +10,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -57,7 +58,7 @@ func (fs *FileSystem) Client() (Client, error) { } // NewFile returns the azure implementation of vfs.File -func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) { +func (fs *FileSystem) NewFile(volume, absFilePath string, opts ...options.NewFileOption) (vfs.File, error) { if fs == nil { return nil, errors.New(errNilFileSystemReceiver) } @@ -74,6 +75,7 @@ func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) { fileSystem: fs, container: volume, name: path.Clean(absFilePath), + opts: opts, }, nil } diff --git a/backend/azure/file_test.go b/backend/azure/file_test.go index 76db4f70..4d5a364a 100644 --- a/backend/azure/file_test.go +++ b/backend/azure/file_test.go @@ -11,6 +11,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -114,6 +115,15 @@ func (s *FileTestSuite) TestExists_NonExistentFile() { s.False(exists) } +func (s *FileTestSuite) TestCloseWithContentType() { + client := MockAzureClient{PropertiesError: MockStorageError{}} + fs := NewFileSystem().WithClient(&client) + f, _ := fs.NewFile("test-container", "/foo.txt", newfile.WithContentType("text/plain")) + _, _ = f.Write([]byte("Hello, World!")) + s.NoError(f.Close()) + s.Equal("text/plain", client.UploadContentType) +} + func (s *FileTestSuite) TestLocation() { fs := NewFileSystem().WithOptions(Options{AccountName: "test-account"}) f, _ := fs.NewFile("test-container", "/file.txt") @@ -189,22 +199,22 @@ func (s *FileTestSuite) TestDelete() { s.NoError(f.Delete(), "The delete should succeed so there should be no error") } -func (s *FileTestSuite) TestDeleteWithDeleteAllVersionsOption() { +func (s *FileTestSuite) TestDeleteWithAllVersionsOption() { client := MockAzureClient{} fs := NewFileSystem().WithClient(&client) f, err := fs.NewFile("test-container", "/foo.txt") s.NoError(err, "The path is valid so no error should be returned") - s.NoError(f.Delete(delete.WithDeleteAllVersions()), "The delete should succeed so there should be no error") + s.NoError(f.Delete(delete.WithAllVersions()), "The delete should succeed so there should be no error") } -func (s *FileTestSuite) TestDeleteWithDeleteAllVersionsOption_Error() { +func (s *FileTestSuite) TestDeleteWithAllVersionsOption_Error() { client := MockAzureClient{ExpectedError: errors.New("i always error")} fs := NewFileSystem().WithClient(&client) f, err := fs.NewFile("test-container", "/foo.txt") s.NoError(err, "The path is valid so no error should be returned") - err = f.Delete(delete.WithDeleteAllVersions()) + err = f.Delete(delete.WithAllVersions()) s.Error(err, "If the file does not exist we get an error") } @@ -287,6 +297,16 @@ func (s *FileTestSuite) TestTouch_NonexistentContainer() { s.Error(f.Touch(), "The container does not exist so creating the new file should error") } +func (s *FileTestSuite) TestTouchWithContentType() { + client := MockAzureClient{ExpectedResult: &BlobProperties{}, PropertiesError: MockStorageError{}} + fs := NewFileSystem().WithClient(&client) + + f, err := fs.NewFile("test-container", "/foo.txt", newfile.WithContentType("text/plain")) + s.NoError(err, "The path is valid so no error should be returned") + s.NoError(f.Touch()) + s.Equal("text/plain", client.UploadContentType) +} + func (s *FileTestSuite) TestURI() { fs := NewFileSystem().WithOptions(Options{AccountName: "test-container"}) f, _ := fs.NewFile("temp", "/foo/bar/blah.txt") diff --git a/backend/azure/location.go b/backend/azure/location.go index 967a64d2..0637bbab 100644 --- a/backend/azure/location.go +++ b/backend/azure/location.go @@ -167,7 +167,7 @@ func (l *Location) FileSystem() vfs.FileSystem { } // NewFile returns a new file instance at the given path, relative to the current location. -func (l *Location) NewFile(relFilePath string) (vfs.File, error) { +func (l *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (vfs.File, error) { if l == nil { return nil, errors.New(errNilLocationReceiver) } @@ -180,6 +180,7 @@ func (l *Location) NewFile(relFilePath string) (vfs.File, error) { name: utils.EnsureLeadingSlash(path.Join(l.path, relFilePath)), container: l.container, fileSystem: l.fileSystem, + opts: opts, }, nil } diff --git a/backend/azure/mock_client.go b/backend/azure/mock_client.go index 55e1be82..13a8d3fa 100644 --- a/backend/azure/mock_client.go +++ b/backend/azure/mock_client.go @@ -11,10 +11,11 @@ import ( // MockAzureClient is a mock implementation of azure.Client. type MockAzureClient struct { - PropertiesError error - PropertiesResult *BlobProperties - ExpectedError error - ExpectedResult interface{} + PropertiesError error + PropertiesResult *BlobProperties + ExpectedError error + ExpectedResult interface{} + UploadContentType string } // Properties returns a PropertiesResult if it exists, otherwise it will return the value of PropertiesError @@ -31,7 +32,8 @@ func (a *MockAzureClient) SetMetadata(file vfs.File, metadata map[string]string) } // Upload returns the value of ExpectedError -func (a *MockAzureClient) Upload(file vfs.File, content io.ReadSeeker) error { +func (a *MockAzureClient) Upload(file vfs.File, content io.ReadSeeker, contentType string) error { + a.UploadContentType = contentType return a.ExpectedError } diff --git a/backend/ftp/file.go b/backend/ftp/file.go index 54844db0..15cd7469 100644 --- a/backend/ftp/file.go +++ b/backend/ftp/file.go @@ -33,6 +33,7 @@ type File struct { fileSystem *FileSystem authority utils.Authority path string + opts []options.NewFileOption offset int64 } diff --git a/backend/ftp/fileSystem.go b/backend/ftp/fileSystem.go index 969d6409..a4f42bf0 100644 --- a/backend/ftp/fileSystem.go +++ b/backend/ftp/fileSystem.go @@ -9,6 +9,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" "github.com/c2fo/vfs/v6/backend/ftp/types" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -34,7 +35,7 @@ func (fs *FileSystem) Retry() vfs.Retry { } // NewFile function returns the FTP implementation of vfs.File. -func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) { +func (fs *FileSystem) NewFile(authority, filePath string, opts ...options.NewFileOption) (vfs.File, error) { if fs == nil { return nil, errors.New("non-nil ftp.FileSystem pointer is required") } @@ -54,6 +55,7 @@ func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) { fileSystem: fs, authority: auth, path: path.Clean(filePath), + opts: opts, }, nil } diff --git a/backend/ftp/location.go b/backend/ftp/location.go index 572714dc..ce95c1f4 100644 --- a/backend/ftp/location.go +++ b/backend/ftp/location.go @@ -193,7 +193,7 @@ func (l *Location) ChangeDir(relativePath string) error { // NewFile uses the properties of the calling location to generate a vfs.File (backed by an ftp.File). The filePath // argument is expected to be a relative path to the location's current path. -func (l *Location) NewFile(filePath string) (vfs.File, error) { +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) { err := utils.ValidateRelativeFilePath(filePath) if err != nil { return nil, err @@ -202,6 +202,7 @@ func (l *Location) NewFile(filePath string) (vfs.File, error) { fileSystem: l.fileSystem, authority: l.Authority, path: utils.EnsureLeadingSlash(path.Join(l.path, filePath)), + opts: opts, } return newFile, nil } diff --git a/backend/gs/file.go b/backend/gs/file.go index ed8600f0..211f65ec 100644 --- a/backend/gs/file.go +++ b/backend/gs/file.go @@ -16,6 +16,7 @@ import ( "github.com/c2fo/vfs/v6/backend" "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -28,6 +29,7 @@ type File struct { fileSystem *FileSystem bucket string key string + opts []options.NewFileOption // seek-related fields cursorPos int64 @@ -106,6 +108,14 @@ func (f *File) tempToGCS() error { w := handle.NewWriter(f.fileSystem.ctx) defer func() { _ = w.Close() }() + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + w.ContentType = *(*string)(o) + default: + } + } + _, err = f.tempFileWriter.Seek(0, io.SeekStart) if err != nil { return err @@ -327,6 +337,14 @@ func (f *File) initWriters() error { return err } + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + w.ContentType = *(*string)(o) + default: + } + } + // set gcsWriter f.gcsWriter = w } @@ -403,6 +421,11 @@ func (f *File) CopyToFile(file vfs.File) (err error) { // do native copy if same location/auth if tf, ok := file.(*File); ok { + // If the target file has no newfile options, use the source file's options (used if not same auth) + if len(f.opts) == 0 { + tf.opts = f.opts + } + opts, ok := tf.Location().FileSystem().(*FileSystem).options.(Options) if ok { if f.isSameAuth(&opts) { @@ -454,18 +477,18 @@ func (f *File) MoveToFile(file vfs.File) error { } // Delete clears any local temp file, or write buffer from read/writes to the file, then makes -// a DeleteObject call to GCS for the file. If DeleteAllVersions option is provided, +// a DeleteObject call to GCS for the file. If delete.AllVersions option is provided, // DeleteObject call is made to GCS for each version of the file. Returns any error returned by the API. func (f *File) Delete(opts ...options.DeleteOption) error { if err := f.Close(); err != nil { return err } - var deleteAllVersions bool + var allVersions bool for _, o := range opts { switch o.(type) { - case delete.DeleteAllVersions: - deleteAllVersions = true + case delete.AllVersions, delete.DeleteAllVersions: + allVersions = true default: } } @@ -479,7 +502,7 @@ func (f *File) Delete(opts ...options.DeleteOption) error { return err } - if deleteAllVersions { + if allVersions { handles, err := f.getObjectGenerationHandles() if err != nil { return err @@ -589,6 +612,15 @@ func (f *File) createEmptyFile() error { defer cancel() w := handle.NewWriter(ctx) + + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + w.ContentType = *(*string)(o) + default: + } + } + defer func() { _ = w.Close() }() if _, err := w.Write(make([]byte, 0)); err != nil { return err diff --git a/backend/gs/fileSystem.go b/backend/gs/fileSystem.go index add14f37..9ccc3919 100644 --- a/backend/gs/fileSystem.go +++ b/backend/gs/fileSystem.go @@ -10,6 +10,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -27,14 +28,14 @@ type FileSystem struct { // Retry will return a retrier provided via options, or a no-op if none is provided. func (fs *FileSystem) Retry() vfs.Retry { - if options, _ := fs.options.(Options); options.Retry != nil { - return options.Retry + if opts, _ := fs.options.(Options); opts.Retry != nil { + return opts.Retry } return vfs.DefaultRetryer() } // NewFile function returns the gcs implementation of vfs.File. -func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { +func (fs *FileSystem) NewFile(volume, name string, opts ...options.NewFileOption) (vfs.File, error) { if fs == nil { return nil, errors.New("non-nil gs.FileSystem pointer is required") } @@ -48,6 +49,7 @@ func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { fileSystem: fs, bucket: volume, key: path.Clean(name), + opts: opts, }, nil } diff --git a/backend/gs/file_test.go b/backend/gs/file_test.go index c43ec292..75a09eda 100644 --- a/backend/gs/file_test.go +++ b/backend/gs/file_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -198,7 +199,7 @@ func (ts *fileTestSuite) TestDeleteRemoveAllVersions() { ts.Require().NoError(err, "Shouldn't fail getting object generation handles") ts.Len(handles, 1) - err = file.Delete(delete.WithDeleteAllVersions()) + err = file.Delete(delete.WithAllVersions()) ts.Require().NoError(err, "Shouldn't fail deleting the file") bucket := client.Bucket(bucketName) @@ -225,6 +226,56 @@ func (ts *fileTestSuite) TestWrite() { ts.NoError(err, "Error should be nil when calling Write") } +func (ts *fileTestSuite) TestWriteWithContentType() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer(Objects{}) + defer server.Stop() + client := server.Client() + bucket := client.Bucket(bucketName) + ctx := context.Background() + err := bucket.Create(ctx, "", nil) + ts.Require().NoError(err) + fs := NewFileSystem().WithClient(client) + + file, err := fs.NewFile(bucketName, "/"+objectName, newfile.WithContentType("text/plain")) + ts.NoError(err, "Shouldn't fail creating new file") + + _, err = file.Write([]byte(contents)) + ts.NoError(err, "Error should be nil when calling Write") + + err = file.Close() + ts.NoError(err, "Error should be nil when calling Close") + + attrs, err := bucket.Object(objectName).Attrs(ctx) + ts.Require().NoError(err) + ts.Equal("text/plain", attrs.ContentType) +} + +func (ts *fileTestSuite) TestTouchWithContentType() { + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer(Objects{}) + defer server.Stop() + client := server.Client() + bucket := client.Bucket(bucketName) + ctx := context.Background() + err := bucket.Create(ctx, "", nil) + ts.Require().NoError(err) + fs := NewFileSystem().WithClient(client) + + file, err := fs.NewFile(bucketName, "/"+objectName, newfile.WithContentType("text/plain")) + ts.NoError(err, "Shouldn't fail creating new file") + + err = file.Touch() + ts.NoError(err, "Error should be nil when calling Touch") + + attrs, err := bucket.Object(objectName).Attrs(ctx) + ts.Require().NoError(err) + ts.Equal("text/plain", attrs.ContentType) +} + func (ts *fileTestSuite) TestGetLocation() { server := fakestorage.NewServer(Objects{}) defer server.Stop() diff --git a/backend/gs/location.go b/backend/gs/location.go index f4ad7d30..38a5a5e6 100644 --- a/backend/gs/location.go +++ b/backend/gs/location.go @@ -157,7 +157,7 @@ func (l *Location) FileSystem() vfs.FileSystem { } // NewFile returns a new file instance at the given path, relative to the current location. -func (l *Location) NewFile(filePath string) (vfs.File, error) { +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) { if l == nil { return nil, errors.New("non-nil gs.Location pointer is required") } @@ -172,6 +172,7 @@ func (l *Location) NewFile(filePath string) (vfs.File, error) { fileSystem: l.fileSystem, bucket: l.bucket, key: utils.EnsureLeadingSlash(path.Join(l.prefix, filePath)), + opts: opts, } return newFile, nil } diff --git a/backend/mem/file.go b/backend/mem/file.go index a15b943b..ebd5cd73 100644 --- a/backend/mem/file.go +++ b/backend/mem/file.go @@ -40,6 +40,7 @@ type File struct { memFile *memFile readWriteSeeker *ReadWriteSeeker name string // the base name of the file + opts []options.NewFileOption cursor int writeMode mode isOpen bool diff --git a/backend/mem/fileSystem.go b/backend/mem/fileSystem.go index 212216aa..99f65a74 100644 --- a/backend/mem/fileSystem.go +++ b/backend/mem/fileSystem.go @@ -6,6 +6,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -38,7 +39,7 @@ func (fs *FileSystem) Retry() vfs.Retry { // If a file is written to before a touch call, Write() will take care of that call. This is // true for other functions as well and existence only poses a problem in the context of deletion // or copying FROM a non-existent file. -func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) { +func (fs *FileSystem) NewFile(volume, absFilePath string, opts ...options.NewFileOption) (vfs.File, error) { err := utils.ValidateAbsoluteFilePath(absFilePath) if err != nil { return nil, err @@ -59,6 +60,7 @@ func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) { name: obj.i.(*memFile).name, memFile: obj.i.(*memFile), readWriteSeeker: NewReadWriteSeekerWithData(obj.i.(*memFile).contents), + opts: opts, } return vfsFile, nil } @@ -68,6 +70,7 @@ func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) { // validateAbsFile path will throw an error if there was a trailing slash, hence not calling path.Clean() file := &File{ name: path.Base(absFilePath), + opts: opts, } memFile := newMemFile(file, location.(*Location)) diff --git a/backend/mem/location.go b/backend/mem/location.go index c22fb14b..ea9d5538 100644 --- a/backend/mem/location.go +++ b/backend/mem/location.go @@ -147,7 +147,7 @@ func (l *Location) FileSystem() vfs.FileSystem { } // NewFile creates a vfs.File given its relative path and tags it onto "l's" path -func (l *Location) NewFile(relFilePath string) (vfs.File, error) { +func (l *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (vfs.File, error) { if relFilePath == "" { return nil, errors.New("cannot use empty name for file") } @@ -178,6 +178,7 @@ func (l *Location) NewFile(relFilePath string) (vfs.File, error) { file := &File{ name: path.Base(nameStr), + opts: opts, } newLoc := *l newLoc.name = relativeLocationPath diff --git a/backend/os/file.go b/backend/os/file.go index 6d7b2fc5..0e38b631 100644 --- a/backend/os/file.go +++ b/backend/os/file.go @@ -26,6 +26,7 @@ type File struct { volume string name string filesystem *FileSystem + opts []options.NewFileOption cursorPos int64 tempFile *os.File useTempFile bool diff --git a/backend/os/fileSystem.go b/backend/os/fileSystem.go index 07ca8c05..b11a2e6a 100644 --- a/backend/os/fileSystem.go +++ b/backend/os/fileSystem.go @@ -7,6 +7,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -23,7 +24,7 @@ func (fs *FileSystem) Retry() vfs.Retry { } // NewFile function returns the os implementation of vfs.File. -func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { +func (fs *FileSystem) NewFile(volume, name string, opts ...options.NewFileOption) (vfs.File, error) { if runtime.GOOS == "windows" && filepath.IsAbs(name) { if v := filepath.VolumeName(name); v != "" { volume = v @@ -36,7 +37,7 @@ func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { if err != nil { return nil, err } - return &File{volume: volume, name: name, filesystem: fs}, nil + return &File{volume: volume, name: name, filesystem: fs, opts: opts}, nil } // NewLocation function returns the os implementation of vfs.Location. diff --git a/backend/os/location.go b/backend/os/location.go index 158e6b78..023f4bf8 100644 --- a/backend/os/location.go +++ b/backend/os/location.go @@ -23,7 +23,7 @@ type Location struct { // NewFile uses the properties of the calling location to generate a vfs.File (backed by an os.File). A string // argument is expected to be a relative path to the location's current path. -func (l *Location) NewFile(fileName string) (vfs.File, error) { +func (l *Location) NewFile(fileName string, opts ...options.NewFileOption) (vfs.File, error) { if l == nil { return nil, errors.New("non-nil os.Location pointer is required") } @@ -36,7 +36,7 @@ func (l *Location) NewFile(fileName string) (vfs.File, error) { return nil, err } fileName = utils.EnsureLeadingSlash(path.Clean(path.Join(l.name, fileName))) - return l.fileSystem.NewFile(l.Volume(), fileName) + return l.fileSystem.NewFile(l.Volume(), fileName, opts...) } // DeleteFile deletes the file of the given name at the location. This is meant to be a short cut for instantiating a diff --git a/backend/s3/file.go b/backend/s3/file.go index f065b889..4592aebb 100644 --- a/backend/s3/file.go +++ b/backend/s3/file.go @@ -21,6 +21,7 @@ import ( "github.com/c2fo/vfs/v6/mocks" "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -31,6 +32,7 @@ type File struct { fileSystem *FileSystem bucket string key string + opts []options.NewFileOption // seek-related fields cursorPos int64 @@ -202,7 +204,7 @@ func (f *File) CopyToLocation(location vfs.Location) (vfs.File, error) { // CRUD Operations // Delete clears any local temp file, or write buffer from read/writes to the file, then makes -// a DeleteObject call to s3 for the file. If DeleteAllVersions option is provided, +// a DeleteObject call to s3 for the file. If delete.AllVersions option is provided, // DeleteObject call is made to s3 for each version of the file. Returns any error returned by the API. func (f *File) Delete(opts ...options.DeleteOption) error { if err := f.Close(); err != nil { @@ -214,11 +216,11 @@ func (f *File) Delete(opts ...options.DeleteOption) error { return err } - var deleteAllVersions bool + var allVersions bool for _, o := range opts { switch o.(type) { - case delete.DeleteAllVersions: - deleteAllVersions = true + case delete.AllVersions, delete.DeleteAllVersions: + allVersions = true default: } } @@ -231,7 +233,7 @@ func (f *File) Delete(opts ...options.DeleteOption) error { return err } - if deleteAllVersions { + if allVersions { objectVersions, err := f.getAllObjectVersions(client) if err != nil { return err @@ -569,6 +571,18 @@ func (f *File) getCopyObjectInput(targetFile *File) (*s3.CopyObjectInput, error) isSameAccount := false var ACL string + // get content type from source + var contentType string + if targetFile.opts == nil && f.opts != nil { + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + contentType = string(*o) + default: + } + } + } + fileOptions := f.Location().FileSystem().(*FileSystem).options targetOptions := targetFile.Location().FileSystem().(*FileSystem).options @@ -605,6 +619,11 @@ func (f *File) getCopyObjectInput(targetFile *File) (*s3.CopyObjectInput, error) SetBucket(targetFile.bucket). SetCopySource(copySourceKey) + // set content type if it exists + if contentType != "" { + copyInput.SetContentType(contentType) + } + if f.fileSystem.options != nil && f.fileSystem.options.(Options).DisableServerSideEncryption { copyInput.ServerSideEncryption = nil } @@ -659,6 +678,14 @@ func uploadInput(f *File) *s3manager.UploadInput { } } + for _, o := range f.opts { + switch o := o.(type) { + case *newfile.ContentType: + input.ContentType = (*string)(o) + default: + } + } + return input } diff --git a/backend/s3/fileSystem.go b/backend/s3/fileSystem.go index a5bfb0dd..ade41f98 100644 --- a/backend/s3/fileSystem.go +++ b/backend/s3/fileSystem.go @@ -10,6 +10,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -30,7 +31,7 @@ func (fs *FileSystem) Retry() vfs.Retry { } // NewFile function returns the s3 implementation of vfs.File. -func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { +func (fs *FileSystem) NewFile(volume, name string, opts ...options.NewFileOption) (vfs.File, error) { if fs == nil { return nil, errors.New("non-nil s3.FileSystem pointer is required") } @@ -45,6 +46,7 @@ func (fs *FileSystem) NewFile(volume, name string) (vfs.File, error) { fileSystem: fs, bucket: utils.RemoveTrailingSlash(volume), key: path.Clean(name), + opts: opts, }, nil } diff --git a/backend/s3/file_test.go b/backend/s3/file_test.go index 55f060e4..9d3d8f98 100644 --- a/backend/s3/file_test.go +++ b/backend/s3/file_test.go @@ -24,6 +24,7 @@ import ( "github.com/c2fo/vfs/v6/backend/s3/mocks" vfsmocks "github.com/c2fo/vfs/v6/mocks" "github.com/c2fo/vfs/v6/options/delete" + "github.com/c2fo/vfs/v6/options/newfile" "github.com/c2fo/vfs/v6/utils" ) @@ -593,7 +594,7 @@ func (ts *fileTestSuite) TestDeleteError() { s3apiMock.AssertExpectations(ts.T()) } -func (ts *fileTestSuite) TestDeleteWithDeleteAllVersionsOption() { +func (ts *fileTestSuite) TestDeleteWithAllVersionsOption() { var versions []*s3.ObjectVersion verIds := [...]string{"ver1", "ver2"} for i := range verIds { @@ -605,13 +606,13 @@ func (ts *fileTestSuite) TestDeleteWithDeleteAllVersionsOption() { s3apiMock.On("ListObjectVersions", mock.AnythingOfType("*s3.ListObjectVersionsInput")).Return(&versOutput, nil) s3apiMock.On("DeleteObject", mock.AnythingOfType("*s3.DeleteObjectInput")).Return(&s3.DeleteObjectOutput{}, nil) - err := testFile.Delete(delete.WithDeleteAllVersions()) + err := testFile.Delete(delete.WithAllVersions()) ts.NoError(err, "Successful delete should not return an error.") s3apiMock.AssertExpectations(ts.T()) s3apiMock.AssertNumberOfCalls(ts.T(), "DeleteObject", 3) } -func (ts *fileTestSuite) TestDeleteWithDeleteAllVersionsOptionError() { +func (ts *fileTestSuite) TestDeleteWithAllVersionsOptionError() { var versions []*s3.ObjectVersion verIds := [...]string{"ver1", "ver2"} for i := range verIds { @@ -625,7 +626,7 @@ func (ts *fileTestSuite) TestDeleteWithDeleteAllVersionsOptionError() { s3apiMock.On("DeleteObject", &s3.DeleteObjectInput{Key: &testFileName, Bucket: &bucket, VersionId: &verIds[0]}). Return(nil, errors.New("something went wrong")) - err := testFile.Delete(delete.WithDeleteAllVersions()) + err := testFile.Delete(delete.WithAllVersions()) ts.Error(err, "Delete should return an error if s3 api had error.") s3apiMock.AssertExpectations(ts.T()) s3apiMock.AssertNumberOfCalls(ts.T(), "DeleteObject", 2) @@ -702,6 +703,13 @@ func (ts *fileTestSuite) TestUploadInputDisableSSE() { ts.Equal("mybucket", *input.Bucket, "bucket was set") } +func (ts *fileTestSuite) TestUploadInputContentType() { + fs = FileSystem{client: &mocks.S3API{}} + file, _ := fs.NewFile("mybucket", "/some/file/test.txt", newfile.WithContentType("text/plain")) + input := uploadInput(file.(*File)) + ts.Equal("text/plain", *input.ContentType) +} + func (ts *fileTestSuite) TestNewFile() { fs := &FileSystem{} // fs is nil diff --git a/backend/s3/location.go b/backend/s3/location.go index 730eb56c..cbec6f19 100644 --- a/backend/s3/location.go +++ b/backend/s3/location.go @@ -123,7 +123,7 @@ func (l *Location) ChangeDir(relativePath string) error { // NewFile uses the properties of the calling location to generate a vfs.File (backed by an s3.File). The filePath // argument is expected to be a relative path to the location's current path. -func (l *Location) NewFile(filePath string) (vfs.File, error) { +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) { if l == nil { return nil, errors.New("non-nil s3.Location pointer is required") } @@ -138,6 +138,7 @@ func (l *Location) NewFile(filePath string) (vfs.File, error) { fileSystem: l.fileSystem, bucket: l.bucket, key: utils.EnsureLeadingSlash(path.Join(l.prefix, filePath)), + opts: opts, } return newFile, nil } diff --git a/backend/s3/location_test.go b/backend/s3/location_test.go index c2b41eb2..9d026109 100644 --- a/backend/s3/location_test.go +++ b/backend/s3/location_test.go @@ -300,7 +300,7 @@ func (lt *locationTestSuite) TestDeleteFile() { lt.s3apiMock.AssertExpectations(lt.T()) } -func (lt *locationTestSuite) TestDeleteFileWithDeleteAllVersionsOption() { +func (lt *locationTestSuite) TestDeleteFileWithAllVersionsOption() { var versions []*s3.ObjectVersion verIds := [...]string{"ver1", "ver2"} for i := range verIds { @@ -314,7 +314,7 @@ func (lt *locationTestSuite) TestDeleteFileWithDeleteAllVersionsOption() { loc, err := lt.fs.NewLocation("bucket", "/old/") lt.NoError(err) - err = loc.DeleteFile("filename.txt", delete.WithDeleteAllVersions()) + err = loc.DeleteFile("filename.txt", delete.WithAllVersions()) lt.NoError(err, "Successful delete should not return an error.") lt.s3apiMock.AssertExpectations(lt.T()) lt.s3apiMock.AssertNumberOfCalls(lt.T(), "DeleteObject", 3) diff --git a/backend/sftp/file.go b/backend/sftp/file.go index f25b3290..5daedf00 100644 --- a/backend/sftp/file.go +++ b/backend/sftp/file.go @@ -18,6 +18,7 @@ type File struct { fileSystem *FileSystem Authority utils.Authority path string + opts []options.NewFileOption sftpfile ReadWriteSeekCloser opener fileOpener seekCalled bool @@ -253,7 +254,7 @@ func (f *File) CopyToLocation(location vfs.Location) (vfs.File, error) { // CRUD Operations // Delete removes the remote file. Error is returned, if any. -func (f *File) Delete(opts ...options.DeleteOption) error { +func (f *File) Delete(_ ...options.DeleteOption) error { client, err := f.fileSystem.Client(f.Authority) if err != nil { return err diff --git a/backend/sftp/fileSystem.go b/backend/sftp/fileSystem.go index e51bf4bc..2a85b786 100644 --- a/backend/sftp/fileSystem.go +++ b/backend/sftp/fileSystem.go @@ -14,6 +14,7 @@ import ( "github.com/c2fo/vfs/v6" "github.com/c2fo/vfs/v6/backend" + "github.com/c2fo/vfs/v6/options" "github.com/c2fo/vfs/v6/utils" ) @@ -40,7 +41,7 @@ func (fs *FileSystem) Retry() vfs.Retry { } // NewFile function returns the SFTP implementation of vfs.File. -func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) { +func (fs *FileSystem) NewFile(authority, filePath string, opts ...options.NewFileOption) (vfs.File, error) { if fs == nil { return nil, errors.New("non-nil sftp.FileSystem pointer is required") } @@ -60,6 +61,7 @@ func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) { fileSystem: fs, Authority: auth, path: path.Clean(filePath), + opts: opts, }, nil } diff --git a/backend/sftp/location.go b/backend/sftp/location.go index ebbe162e..1200e127 100644 --- a/backend/sftp/location.go +++ b/backend/sftp/location.go @@ -173,7 +173,7 @@ func (l *Location) ChangeDir(relativePath string) error { // NewFile uses the properties of the calling location to generate a vfs.File (backed by an sftp.File). The filePath // argument is expected to be a relative path to the location's current path. -func (l *Location) NewFile(filePath string) (vfs.File, error) { +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) { if l == nil { return nil, errors.New("non-nil sftp.Location pointer receiver is required") } @@ -188,6 +188,7 @@ func (l *Location) NewFile(filePath string) (vfs.File, error) { fileSystem: l.fileSystem, Authority: l.Authority, path: utils.EnsureLeadingSlash(path.Join(l.path, filePath)), + opts: opts, } return newFile, nil } diff --git a/backend/testsuite/io_integration_test.go b/backend/testsuite/io_integration_test.go index ffda5fbe..3f1fb921 100644 --- a/backend/testsuite/io_integration_test.go +++ b/backend/testsuite/io_integration_test.go @@ -120,7 +120,7 @@ type ReadWriteSeekCloseURINamer interface { io.Closer Name() string URI() string - Delete(deleteOpts ...options.DeleteOption) error + Delete(opts ...options.DeleteOption) error } type ioTestSuite struct { diff --git a/docs/azure.md b/docs/azure.md index a021645f..0c85906c 100644 --- a/docs/azure.md +++ b/docs/azure.md @@ -295,7 +295,7 @@ be done to the new file. ```go func (f *File) Delete(opts ...options.DeleteOption) error ``` -Deletes the file using Azure's delete blob api. If opts is of type DeleteAllVersions, after deleting the blob, each version of the blob is deleted using Azure's delete api. NOTE that if soft deletion is enabled for the blobs, each version will be marked as deleted and will get permanently deleted by Azure as per the soft deletion policy. Returns any error returned by the API. +Deletes the file using Azure's delete blob api. If opts is of type delete.AllVersions, after deleting the blob, each version of the blob is deleted using Azure's delete api. NOTE that if soft deletion is enabled for the blobs, each version will be marked as deleted and will get permanently deleted by Azure as per the soft deletion policy. Returns any error returned by the API. #### func (*File) Exists @@ -449,7 +449,7 @@ Name returns "azure" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(volume, absFilePath string) (vfs.File, error) +func (fs *FileSystem) NewFile(volume, absFilePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile returns the azure implementation of vfs.File @@ -558,7 +558,7 @@ ListByRegex returns a list of base names that match the given regular expression #### func (*Location) NewFile ```go -func (l *Location) NewFile(relFilePath string) (vfs.File, error) +func (l *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile returns a new file instance at the given path, relative to the current location. diff --git a/docs/delete_options.md b/docs/delete_options.md index ec626009..5a6b99f2 100644 --- a/docs/delete_options.md +++ b/docs/delete_options.md @@ -5,7 +5,7 @@ Package delete consists of custom delete options ## DeleteAllVersions -Currently, we have DeleteAllVersions option that can be used to remove all the versions of a file upon delete. +Currently, we have delete.AllVersions option that can be used to remove all the versions of a file upon delete. This is supported for all filesystems that have file versioning (E.g: S3, GS etc.) ### Usage @@ -21,7 +21,7 @@ Delete file using file.delete(): func DeleteFile() error { file, err := fs.NewFile(bucketName, fileName) ... - err = file.Delete(delete.WithDeleteAllVersions()) + err = file.Delete(delete.WithAllVersions()) ... } ``` @@ -37,7 +37,7 @@ Delete file using location.delete(): ) func DeleteFileUsingLocation() error { - err = location.DeleteFile("filename.txt", delete.WithDeleteAllVersions()) + err = location.DeleteFile("filename.txt", delete.WithAllVersions()) ... } ``` diff --git a/docs/ftp.md b/docs/ftp.md index d339f5c7..6076875c 100644 --- a/docs/ftp.md +++ b/docs/ftp.md @@ -381,7 +381,7 @@ Name returns "Secure File Transfer Protocol" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) +func (fs *FileSystem) NewFile(authority, filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the FTP implementation of vfs.File. @@ -498,7 +498,7 @@ considerations of List() apply here as well. #### func (*Location) NewFile ```go -func (l *Location) NewFile(filePath string) (vfs.File, error) +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile uses the properties of the calling location to generate a vfs.File (backed by an ftp.File). The filePath argument is expected to be a relative path diff --git a/docs/gs.md b/docs/gs.md index 3b2d615a..42e9dd28 100644 --- a/docs/gs.md +++ b/docs/gs.md @@ -130,7 +130,7 @@ to the new file. func (f *File) Delete(opts ...options.DeleteOption) error ``` Delete clears any local temp file, or write buffer from read/writes to the file, -then makes a DeleteObject call to GCS for the file. If opts is of type DeleteAllVersions, DeleteObject call is made to +then makes a DeleteObject call to GCS for the file. If opts is of type delete.AllVersions, DeleteObject call is made to GCS for each version of the file. Returns any error returned by the API. #### func (*File) Exists @@ -269,7 +269,7 @@ Name returns "Google Cloud Storage" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(volume string, name string) (vfs.File, error) +func (fs *FileSystem) NewFile(volume string, name string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the gcs implementation of [vfs.File](../README.md#type-file). @@ -374,7 +374,7 @@ provided regular expression. #### func (*Location) NewFile ```go -func (l *Location) NewFile(filePath string) (vfs.File, error) +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile returns a new file instance at the given path, relative to the current location. diff --git a/docs/mem.md b/docs/mem.md index a644e82c..c46ccd46 100644 --- a/docs/mem.md +++ b/docs/mem.md @@ -219,7 +219,7 @@ Name returns the name of the underlying FileSystem #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(volume string, absFilePath string) (vfs.File, error) +func (fs *FileSystem) NewFile(volume string, absFilePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the in-memory implementation of vfs.File. Since this is inside FileSystem, we assume that the caller knows that the CWD is the root. If @@ -322,7 +322,7 @@ empty slice upon nothing found #### func (*Location) NewFile ```go -func (l *Location) NewFile(relFilePath string) (vfs.File, error) +func (l *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile creates a vfs.File given its relative path and tags it onto "l's" path diff --git a/docs/options.md b/docs/options.md index 3f7c9b76..ac6115fd 100644 --- a/docs/options.md +++ b/docs/options.md @@ -5,7 +5,7 @@ Package options provides a means of creating custom options that can be used with operations that are performed on components of filesystem. ## DeleteOption -Currently, we define DeleteOption interface that can be used to implement custom options that can be used for delete operation. One such implementation is the [DeleteAllVersions](./delete_options.md#DeleteAllVersions) option. +Currently, we define DeleteOption interface that can be used to implement custom options that can be used for delete operation. One such implementation is the [delete.AllVersions](./delete_options.md#DeleteAllVersions) option. ## Development @@ -32,8 +32,8 @@ Now, in each implementation of file.Delete(... options.DeleteOptions), implement func (f *File) Delete(opts ...options.DeleteOption) error { for _, o := range opts { switch o.(type) { - case delete.DeleteAllVersions: - deleteAllVersions = true + case delete.AllVersions: + allVersions = true case delete.MyTakeBackupDeleteOption: // do something to take backup default: diff --git a/docs/os.md b/docs/os.md index 81972ac7..2cabfa63 100644 --- a/docs/os.md +++ b/docs/os.md @@ -205,7 +205,7 @@ Name returns "os" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(volume string, name string) (vfs.File, error) +func (fs *FileSystem) NewFile(volume string, name string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the os implementation of [vfs.File](../README.md#type-file). @@ -294,7 +294,7 @@ of the location. #### func (*Location) NewFile ```go -func (l *Location) NewFile(fileName string) (vfs.File, error) +func (l *Location) NewFile(fileName string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile uses the properties of the calling location to generate a [vfs.File](../README.md#type-file) (backed by an [os.File](#type-file)). A string argument is expected to be a relative path to diff --git a/docs/s3.md b/docs/s3.md index f327374e..95a2c277 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -147,7 +147,7 @@ the new file. func (f *File) Delete(opts ...options.DeleteOption) error ``` Delete clears any local temp file, or write buffer from read/writes to the file, -then makes a DeleteObject call to s3 for the file. If opts is of type DeleteAllVersions, DeleteObject call is made to +then makes a DeleteObject call to s3 for the file. If opts is of type delete.AllVersions, DeleteObject call is made to s3 for each version of the file. Returns any error returned by the API. @@ -296,7 +296,7 @@ Name returns "AWS S3" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(volume string, name string) (vfs.File, error) +func (fs *FileSystem) NewFile(volume string, name string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the s3 implementation of [vfs.File](../README.md#type-file). @@ -402,7 +402,7 @@ considerations of [List()](#func-location-list) apply here as well. #### func (*Location) NewFile ```go -func (l *Location) NewFile(filePath string) (vfs.File, error) +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile uses the properties of the calling location to generate a vfs.File (backed by an [s3.File](#type-file)). The filePath argument is expected to be a relative path diff --git a/docs/sftp.md b/docs/sftp.md index f9c49855..9e56ca5c 100644 --- a/docs/sftp.md +++ b/docs/sftp.md @@ -411,7 +411,7 @@ Name returns "Secure File Transfer Protocol" #### func (*FileSystem) NewFile ```go -func (fs *FileSystem) NewFile(authority, filePath string) (vfs.File, error) +func (fs *FileSystem) NewFile(authority, filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile function returns the SFTP implementation of vfs.File. @@ -522,7 +522,7 @@ considerations of [List()](#func-location-list) apply here as well. #### func (*Location) NewFile ```go -func (l *Location) NewFile(filePath string) (vfs.File, error) +func (l *Location) NewFile(filePath string, opts ...options.NewFileOption) (vfs.File, error) ``` NewFile uses the properties of the calling location to generate a [vfs.File](../README.md#type-file) (backed by an sftp.File). The filePath argument is expected to be a relative diff --git a/mocks/File.go b/mocks/File.go index 59a7490c..38d9b341 100644 --- a/mocks/File.go +++ b/mocks/File.go @@ -173,11 +173,11 @@ func (_c *File_CopyToLocation_Call) RunAndReturn(run func(vfs.Location) (vfs.Fil return _c } -// Delete provides a mock function with given fields: deleteOpts -func (_m *File) Delete(deleteOpts ...options.DeleteOption) error { - _va := make([]interface{}, len(deleteOpts)) - for _i := range deleteOpts { - _va[_i] = deleteOpts[_i] +// Delete provides a mock function with given fields: opts +func (_m *File) Delete(opts ...options.DeleteOption) error { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, _va...) @@ -189,7 +189,7 @@ func (_m *File) Delete(deleteOpts ...options.DeleteOption) error { var r0 error if rf, ok := ret.Get(0).(func(...options.DeleteOption) error); ok { - r0 = rf(deleteOpts...) + r0 = rf(opts...) } else { r0 = ret.Error(0) } @@ -203,13 +203,13 @@ type File_Delete_Call struct { } // Delete is a helper method to define mock.On call -// - deleteOpts ...options.DeleteOption -func (_e *File_Expecter) Delete(deleteOpts ...interface{}) *File_Delete_Call { +// - opts ...options.DeleteOption +func (_e *File_Expecter) Delete(opts ...interface{}) *File_Delete_Call { return &File_Delete_Call{Call: _e.mock.On("Delete", - append([]interface{}{}, deleteOpts...)...)} + append([]interface{}{}, opts...)...)} } -func (_c *File_Delete_Call) Run(run func(deleteOpts ...options.DeleteOption)) *File_Delete_Call { +func (_c *File_Delete_Call) Run(run func(opts ...options.DeleteOption)) *File_Delete_Call { _c.Call.Run(func(args mock.Arguments) { variadicArgs := make([]options.DeleteOption, len(args)-0) for i, a := range args[0:] { diff --git a/mocks/FileSystem.go b/mocks/FileSystem.go index fee875fa..c60061b6 100644 --- a/mocks/FileSystem.go +++ b/mocks/FileSystem.go @@ -3,8 +3,10 @@ package mocks import ( - vfs "github.com/c2fo/vfs/v6" + options "github.com/c2fo/vfs/v6/options" mock "github.com/stretchr/testify/mock" + + vfs "github.com/c2fo/vfs/v6" ) // FileSystem is an autogenerated mock type for the FileSystem type @@ -65,9 +67,16 @@ func (_c *FileSystem_Name_Call) RunAndReturn(run func() string) *FileSystem_Name return _c } -// NewFile provides a mock function with given fields: volume, absFilePath -func (_m *FileSystem) NewFile(volume string, absFilePath string) (vfs.File, error) { - ret := _m.Called(volume, absFilePath) +// NewFile provides a mock function with given fields: volume, absFilePath, opts +func (_m *FileSystem) NewFile(volume string, absFilePath string, opts ...options.NewFileOption) (vfs.File, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, volume, absFilePath) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for NewFile") @@ -75,19 +84,19 @@ func (_m *FileSystem) NewFile(volume string, absFilePath string) (vfs.File, erro var r0 vfs.File var r1 error - if rf, ok := ret.Get(0).(func(string, string) (vfs.File, error)); ok { - return rf(volume, absFilePath) + if rf, ok := ret.Get(0).(func(string, string, ...options.NewFileOption) (vfs.File, error)); ok { + return rf(volume, absFilePath, opts...) } - if rf, ok := ret.Get(0).(func(string, string) vfs.File); ok { - r0 = rf(volume, absFilePath) + if rf, ok := ret.Get(0).(func(string, string, ...options.NewFileOption) vfs.File); ok { + r0 = rf(volume, absFilePath, opts...) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(vfs.File) } } - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(volume, absFilePath) + if rf, ok := ret.Get(1).(func(string, string, ...options.NewFileOption) error); ok { + r1 = rf(volume, absFilePath, opts...) } else { r1 = ret.Error(1) } @@ -103,13 +112,21 @@ type FileSystem_NewFile_Call struct { // NewFile is a helper method to define mock.On call // - volume string // - absFilePath string -func (_e *FileSystem_Expecter) NewFile(volume interface{}, absFilePath interface{}) *FileSystem_NewFile_Call { - return &FileSystem_NewFile_Call{Call: _e.mock.On("NewFile", volume, absFilePath)} +// - opts ...options.NewFileOption +func (_e *FileSystem_Expecter) NewFile(volume interface{}, absFilePath interface{}, opts ...interface{}) *FileSystem_NewFile_Call { + return &FileSystem_NewFile_Call{Call: _e.mock.On("NewFile", + append([]interface{}{volume, absFilePath}, opts...)...)} } -func (_c *FileSystem_NewFile_Call) Run(run func(volume string, absFilePath string)) *FileSystem_NewFile_Call { +func (_c *FileSystem_NewFile_Call) Run(run func(volume string, absFilePath string, opts ...options.NewFileOption)) *FileSystem_NewFile_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string), args[1].(string)) + variadicArgs := make([]options.NewFileOption, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(options.NewFileOption) + } + } + run(args[0].(string), args[1].(string), variadicArgs...) }) return _c } @@ -119,7 +136,7 @@ func (_c *FileSystem_NewFile_Call) Return(_a0 vfs.File, _a1 error) *FileSystem_N return _c } -func (_c *FileSystem_NewFile_Call) RunAndReturn(run func(string, string) (vfs.File, error)) *FileSystem_NewFile_Call { +func (_c *FileSystem_NewFile_Call) RunAndReturn(run func(string, string, ...options.NewFileOption) (vfs.File, error)) *FileSystem_NewFile_Call { _c.Call.Return(run) return _c } diff --git a/mocks/Location.go b/mocks/Location.go index 1969193b..67c9f65c 100644 --- a/mocks/Location.go +++ b/mocks/Location.go @@ -70,11 +70,11 @@ func (_c *Location_ChangeDir_Call) RunAndReturn(run func(string) error) *Locatio return _c } -// DeleteFile provides a mock function with given fields: relFilePath, deleteOpts -func (_m *Location) DeleteFile(relFilePath string, deleteOpts ...options.DeleteOption) error { - _va := make([]interface{}, len(deleteOpts)) - for _i := range deleteOpts { - _va[_i] = deleteOpts[_i] +// DeleteFile provides a mock function with given fields: relFilePath, opts +func (_m *Location) DeleteFile(relFilePath string, opts ...options.DeleteOption) error { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] } var _ca []interface{} _ca = append(_ca, relFilePath) @@ -87,7 +87,7 @@ func (_m *Location) DeleteFile(relFilePath string, deleteOpts ...options.DeleteO var r0 error if rf, ok := ret.Get(0).(func(string, ...options.DeleteOption) error); ok { - r0 = rf(relFilePath, deleteOpts...) + r0 = rf(relFilePath, opts...) } else { r0 = ret.Error(0) } @@ -102,13 +102,13 @@ type Location_DeleteFile_Call struct { // DeleteFile is a helper method to define mock.On call // - relFilePath string -// - deleteOpts ...options.DeleteOption -func (_e *Location_Expecter) DeleteFile(relFilePath interface{}, deleteOpts ...interface{}) *Location_DeleteFile_Call { +// - opts ...options.DeleteOption +func (_e *Location_Expecter) DeleteFile(relFilePath interface{}, opts ...interface{}) *Location_DeleteFile_Call { return &Location_DeleteFile_Call{Call: _e.mock.On("DeleteFile", - append([]interface{}{relFilePath}, deleteOpts...)...)} + append([]interface{}{relFilePath}, opts...)...)} } -func (_c *Location_DeleteFile_Call) Run(run func(relFilePath string, deleteOpts ...options.DeleteOption)) *Location_DeleteFile_Call { +func (_c *Location_DeleteFile_Call) Run(run func(relFilePath string, opts ...options.DeleteOption)) *Location_DeleteFile_Call { _c.Call.Run(func(args mock.Arguments) { variadicArgs := make([]options.DeleteOption, len(args)-1) for i, a := range args[1:] { @@ -406,9 +406,16 @@ func (_c *Location_ListByRegex_Call) RunAndReturn(run func(*regexp.Regexp) ([]st return _c } -// NewFile provides a mock function with given fields: relFilePath -func (_m *Location) NewFile(relFilePath string) (vfs.File, error) { - ret := _m.Called(relFilePath) +// NewFile provides a mock function with given fields: relFilePath, opts +func (_m *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (vfs.File, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, relFilePath) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) if len(ret) == 0 { panic("no return value specified for NewFile") @@ -416,19 +423,19 @@ func (_m *Location) NewFile(relFilePath string) (vfs.File, error) { var r0 vfs.File var r1 error - if rf, ok := ret.Get(0).(func(string) (vfs.File, error)); ok { - return rf(relFilePath) + if rf, ok := ret.Get(0).(func(string, ...options.NewFileOption) (vfs.File, error)); ok { + return rf(relFilePath, opts...) } - if rf, ok := ret.Get(0).(func(string) vfs.File); ok { - r0 = rf(relFilePath) + if rf, ok := ret.Get(0).(func(string, ...options.NewFileOption) vfs.File); ok { + r0 = rf(relFilePath, opts...) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(vfs.File) } } - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(relFilePath) + if rf, ok := ret.Get(1).(func(string, ...options.NewFileOption) error); ok { + r1 = rf(relFilePath, opts...) } else { r1 = ret.Error(1) } @@ -443,13 +450,21 @@ type Location_NewFile_Call struct { // NewFile is a helper method to define mock.On call // - relFilePath string -func (_e *Location_Expecter) NewFile(relFilePath interface{}) *Location_NewFile_Call { - return &Location_NewFile_Call{Call: _e.mock.On("NewFile", relFilePath)} +// - opts ...options.NewFileOption +func (_e *Location_Expecter) NewFile(relFilePath interface{}, opts ...interface{}) *Location_NewFile_Call { + return &Location_NewFile_Call{Call: _e.mock.On("NewFile", + append([]interface{}{relFilePath}, opts...)...)} } -func (_c *Location_NewFile_Call) Run(run func(relFilePath string)) *Location_NewFile_Call { +func (_c *Location_NewFile_Call) Run(run func(relFilePath string, opts ...options.NewFileOption)) *Location_NewFile_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + variadicArgs := make([]options.NewFileOption, len(args)-1) + for i, a := range args[1:] { + if a != nil { + variadicArgs[i] = a.(options.NewFileOption) + } + } + run(args[0].(string), variadicArgs...) }) return _c } @@ -459,7 +474,7 @@ func (_c *Location_NewFile_Call) Return(_a0 vfs.File, _a1 error) *Location_NewFi return _c } -func (_c *Location_NewFile_Call) RunAndReturn(run func(string) (vfs.File, error)) *Location_NewFile_Call { +func (_c *Location_NewFile_Call) RunAndReturn(run func(string, ...options.NewFileOption) (vfs.File, error)) *Location_NewFile_Call { _c.Call.Return(run) return _c } diff --git a/options/delete/deleteAllVersions.go b/options/delete/allVersions.go similarity index 51% rename from options/delete/deleteAllVersions.go rename to options/delete/allVersions.go index 2808d262..5925a1f6 100644 --- a/options/delete/deleteAllVersions.go +++ b/options/delete/allVersions.go @@ -4,13 +4,29 @@ import "github.com/c2fo/vfs/v6/options" const optionNameDeleteAllVersions = "deleteAllVersions" -// WithDeleteAllVersions returns DeleteAllVersions implementation of delete.DeleteOption +// WithAllVersions returns AllVersions implementation of DeleteOption +func WithAllVersions() options.DeleteOption { + return AllVersions{} +} + +// AllVersions represents the DeleteOption that is used to remove all versions of files when deleted. +// This will remove all versions of files for the filesystems that support file versioning. +type AllVersions struct{} + +// DeleteOptionName returns the name of AllVersions option +func (w AllVersions) DeleteOptionName() string { + return optionNameDeleteAllVersions +} + +// WithDeleteAllVersions returns DeleteAllVersions implementation of options.DeleteOption +// Deprecated: use WithAllVersions instead func WithDeleteAllVersions() options.DeleteOption { return DeleteAllVersions{} } // DeleteAllVersions represents the DeleteOption that is used to remove all versions of files when deleted. // This will remove all versions of files for the filesystems that support file versioning. +// Deprecated: use AllVersions instead type DeleteAllVersions struct{} // DeleteOptionName returns the name of DeleteAllVersions option diff --git a/options/newfile/contentType.go b/options/newfile/contentType.go new file mode 100644 index 00000000..559361c5 --- /dev/null +++ b/options/newfile/contentType.go @@ -0,0 +1,19 @@ +package newfile + +import "github.com/c2fo/vfs/v6/options" + +const optionNameNewFileContentType = "newFileContentType" + +// WithContentType returns ContentType implementation of NewFileOption +func WithContentType(contentType string) options.NewFileOption { + ct := ContentType(contentType) + return &ct +} + +// ContentType represents the NewFileOption that is used to explicitly specify a content type on created files. +type ContentType string + +// NewFileOptionName returns the name of ContentType option +func (ct *ContentType) NewFileOptionName() string { + return optionNameNewFileContentType +} diff --git a/options/options.go b/options/options.go index 558d1438..d613278f 100644 --- a/options/options.go +++ b/options/options.go @@ -16,3 +16,8 @@ package options type DeleteOption interface { DeleteOptionName() string } + +// NewFileOption interface contains function that should be implemented by any custom option to qualify as a new file option. +type NewFileOption interface { + NewFileOptionName() string +} diff --git a/vfs.go b/vfs.go index 9771ba4b..dae32ddc 100644 --- a/vfs.go +++ b/vfs.go @@ -22,7 +22,7 @@ type FileSystem interface { // s3://mybucket/path/to/file has a volume of "mybucket and name /path/to/file // results in /tmp/dir1/newerdir/file.txt for the final vfs.File path. // * The file may or may not already exist. - NewFile(volume string, absFilePath string) (File, error) + NewFile(volume string, absFilePath string, opts ...options.NewFileOption) (File, error) // NewLocation initializes a Location on the specified volume with the given path. // @@ -123,7 +123,7 @@ type Location interface { // results in /tmp/dir1/newerdir/file.txt for the final vfs.File path. // * Upon success, a vfs.File, representing the file's new path (location path + file relative path), will be returned. // * The file may or may not already exist. - NewFile(relFilePath string) (File, error) + NewFile(relFilePath string, opts ...options.NewFileOption) (File, error) // DeleteFile deletes the file of the given name at the location. // @@ -131,7 +131,7 @@ type Location interface { // error handling overhead. // // * Accepts relative file path. - DeleteFile(relFilePath string, deleteOpts ...options.DeleteOption) error + DeleteFile(relFilePath string, opts ...options.DeleteOption) error // URI returns the fully qualified absolute URI for the Location. IE, s3://bucket/some/path/ // @@ -197,7 +197,7 @@ type File interface { MoveToFile(file File) error // Delete unlinks the File on the file system. - Delete(deleteOpts ...options.DeleteOption) error + Delete(opts ...options.DeleteOption) error // LastModified returns the timestamp the file was last modified (as *time.Time). LastModified() (*time.Time, error)