Skip to content

Commit

Permalink
feat(storage): support autoclass v2.1 (#8721)
Browse files Browse the repository at this point in the history 
* feat: add autoclass v2.1 support

* start updating bucket_test

* update tests

* address comments and update tests var naming
  • Loading branch information
@cojenco
cojenco authored Oct 30, 2023
1 parent 3053c79 commit fe1e195
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 31 deletions.
51 changes: 35 additions & 16 deletions storage/bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,13 @@ type Autoclass struct {
// If Autoclass is enabled when the bucket is created, the ToggleTime
// is set to the bucket creation time. This field is read-only.
ToggleTime time.Time
// TerminalStorageClass: The storage class that objects in the bucket
// eventually transition to if they are not read for a certain length of
// time. Valid values are NEARLINE and ARCHIVE.
TerminalStorageClass string
// TerminalStorageClassUpdateTime represents the time of the most recent
// update to "TerminalStorageClass".
TerminalStorageClassUpdateTime time.Time
}

func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
Expand Down Expand Up @@ -1241,9 +1248,11 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
}
if ua.Autoclass != nil {
rb.Autoclass = &raw.BucketAutoclass{
Enabled: ua.Autoclass.Enabled,
ForceSendFields: []string{"Enabled"},
Enabled: ua.Autoclass.Enabled,
TerminalStorageClass: ua.Autoclass.TerminalStorageClass,
ForceSendFields: []string{"Enabled"},
}
rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
}
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
Expand Down Expand Up @@ -1954,46 +1963,56 @@ func (a *Autoclass) toRawAutoclass() *raw.BucketAutoclass {
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
// Excluding read only fields ToggleTime and TerminalStorageClassUpdateTime.
return &raw.BucketAutoclass{
Enabled: a.Enabled,
Enabled: a.Enabled,
TerminalStorageClass: a.TerminalStorageClass,
}
}

func (a *Autoclass) toProtoAutoclass() *storagepb.Bucket_Autoclass {
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
return &storagepb.Bucket_Autoclass{
// Excluding read only fields ToggleTime and TerminalStorageClassUpdateTime.
ba := &storagepb.Bucket_Autoclass{
Enabled: a.Enabled,
}
if a.TerminalStorageClass != "" {
ba.TerminalStorageClass = &a.TerminalStorageClass
}
return ba
}

func toAutoclassFromRaw(a *raw.BucketAutoclass) *Autoclass {
if a == nil || a.ToggleTime == "" {
return nil
}
// Return Autoclass.ToggleTime only if parsed with a valid value.
ac := &Autoclass{
Enabled: a.Enabled,
TerminalStorageClass: a.TerminalStorageClass,
}
// Return ToggleTime and TSCUpdateTime only if parsed with valid values.
t, err := time.Parse(time.RFC3339, a.ToggleTime)
if err != nil {
return &Autoclass{
Enabled: a.Enabled,
}
if err == nil {
ac.ToggleTime = t
}
return &Autoclass{
Enabled: a.Enabled,
ToggleTime: t,
ut, err := time.Parse(time.RFC3339, a.TerminalStorageClassUpdateTime)
if err == nil {
ac.TerminalStorageClassUpdateTime = ut
}
return ac
}

func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
if a == nil || a.GetToggleTime().AsTime().Unix() == 0 {
return nil
}
return &Autoclass{
Enabled: a.GetEnabled(),
ToggleTime: a.GetToggleTime().AsTime(),
Enabled: a.GetEnabled(),
ToggleTime: a.GetToggleTime().AsTime(),
TerminalStorageClass: a.GetTerminalStorageClass(),
TerminalStorageClassUpdateTime: a.GetTerminalStorageClassUpdateTime().AsTime(),
}
}

Expand Down
37 changes: 24 additions & 13 deletions storage/bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &Autoclass{Enabled: true},
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
Expand Down Expand Up @@ -169,7 +169,7 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &raw.BucketAutoclass{Enabled: true},
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
Lifecycle: &raw.BucketLifecycle{
Rule: []*raw.BucketLifecycleRule{{
Action: &raw.BucketLifecycleRuleAction{
Expand Down Expand Up @@ -398,7 +398,7 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
StorageClass: "NEARLINE",
Autoclass: &Autoclass{Enabled: false},
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
}
au.SetLabel("a", "foo")
au.DeleteLabel("b")
Expand Down Expand Up @@ -442,8 +442,8 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
StorageClass: "NEARLINE",
Autoclass: &raw.BucketAutoclass{Enabled: false, ForceSendFields: []string{"Enabled"}},
ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle"},
Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
}
if msg := testutil.Diff(got, want); msg != "" {
t.Error(msg)
Expand Down Expand Up @@ -648,8 +648,10 @@ func TestNewBucket(t *testing.T) {
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
ProjectNumber: 123231313,
Autoclass: &raw.BucketAutoclass{
Enabled: true,
ToggleTime: "2017-10-23T04:05:06Z",
Enabled: true,
ToggleTime: "2017-10-23T04:05:06Z",
TerminalStorageClass: "NEARLINE",
TerminalStorageClassUpdateTime: "2017-10-23T04:05:06Z",
},
}
want := &BucketAttrs{
Expand Down Expand Up @@ -702,8 +704,10 @@ func TestNewBucket(t *testing.T) {
LocationType: "dual-region",
ProjectNumber: 123231313,
Autoclass: &Autoclass{
Enabled: true,
ToggleTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
Enabled: true,
ToggleTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
TerminalStorageClass: "NEARLINE",
TerminalStorageClassUpdateTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
},
}
got, err := newBucket(rb)
Expand All @@ -716,6 +720,7 @@ func TestNewBucket(t *testing.T) {
}

func TestNewBucketFromProto(t *testing.T) {
autoclassTSC := "NEARLINE"
pb := &storagepb.Bucket{
Name: "name",
Acl: []*storagepb.BucketAccessControl{
Expand Down Expand Up @@ -753,7 +758,12 @@ func TestNewBucketFromProto(t *testing.T) {
Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, ToggleTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))},
Autoclass: &storagepb.Bucket_Autoclass{
Enabled: true,
ToggleTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
TerminalStorageClass: &autoclassTSC,
TerminalStorageClassUpdateTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
},
Lifecycle: &storagepb.Bucket_Lifecycle{
Rule: []*storagepb.Bucket_Lifecycle_Rule{
{
Expand Down Expand Up @@ -794,7 +804,7 @@ func TestNewBucketFromProto(t *testing.T) {
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
Autoclass: &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), TerminalStorageClass: "NEARLINE", TerminalStorageClassUpdateTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
Expand Down Expand Up @@ -842,7 +852,7 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &Autoclass{Enabled: true},
Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
Expand All @@ -858,6 +868,7 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
}
got := attrs.toProtoBucket()
autoclassTSC := "ARCHIVE"
want := &storagepb.Bucket{
Name: "name",
Acl: []*storagepb.BucketAccessControl{
Expand Down Expand Up @@ -891,7 +902,7 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &storagepb.Bucket_Autoclass{Enabled: true},
Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
Lifecycle: &storagepb.Bucket_Lifecycle{
Rule: []*storagepb.Bucket_Lifecycle_Rule{
{
Expand Down
29 changes: 27 additions & 2 deletions storage/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -883,22 +883,47 @@ func TestIntegration_Autoclass(t *testing.T) {
defer h.mustDeleteBucket(bkt)

// Get Autoclass configuration from bucket attrs.
// Autoclass.TerminalStorageClass is defaulted to NEARLINE if not specified.
attrs, err := bkt.Attrs(ctx)
if err != nil {
t.Fatalf("get bucket attrs failed: %v", err)
}
var toggleTime time.Time
var tscUpdateTime time.Time
if attrs != nil && attrs.Autoclass != nil {
if got, want := attrs.Autoclass.Enabled, true; got != want {
t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
}
if toggleTime = attrs.Autoclass.ToggleTime; toggleTime.IsZero() {
t.Error("got a zero time value, want a populated value")
}
if got, want := attrs.Autoclass.TerminalStorageClass, "NEARLINE"; got != want {
t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
}
if tscUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime; tscUpdateTime.IsZero() {
t.Error("got a zero time value, want a populated value")
}
}

// Update TerminalStorageClass on the bucket.
ua := BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}}
attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
if got, want := attrs.Autoclass.Enabled, true; got != want {
t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
}
if got, want := attrs.Autoclass.TerminalStorageClass, "ARCHIVE"; got != want {
t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
}
latestTSCUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime
if latestTSCUpdateTime.IsZero() {
t.Error("got a zero time value, want a populated value")
}
if !latestTSCUpdateTime.After(tscUpdateTime) {
t.Error("latestTSCUpdateTime should be newer than bucket creation tscUpdateTime")
}

// Disable Autoclass on the bucket.
ua := BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: false}}
ua = BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: false}}
attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
if got, want := attrs.Autoclass.Enabled, false; got != want {
t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
Expand All @@ -907,7 +932,7 @@ func TestIntegration_Autoclass(t *testing.T) {
if latestToggleTime.IsZero() {
t.Error("got a zero time value, want a populated value")
}
if latestToggleTime.Before(toggleTime) {
if !latestToggleTime.After(toggleTime) {
t.Error("latestToggleTime should be newer than bucket creation toggleTime")
}
})
Expand Down

0 comments on commit fe1e195

Please sign in to comment.