Update: provide a tool and lib to interact with the ostor service

.envrc-dist

@@ -1,3 +1,9 @@
+# for tenant-usage
 export ACI_CLIENT_ID=
 export ACI_SECRET=
 export ACI_DC_URL=https://eu2-cloud.acronis.com
+
+# for bucket-usage (system user)
+export S3_ENDPOINT=
+export S3_SYSTEM_KEY_ID=
+export S3_SYSTEM_SECRET_KEY=

LICENSE

@@ -1,6 +1,6 @@
 BSD 2-Clause License
 
-Copyright (c) 2022, Luzilla Capital GmbH
+Copyright (c) 2022-2024, Luzilla Capital GmbH
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.md

@@ -1,5 +1,9 @@
 # acronis-s3-usage
 
+This is a playground to explore certain APIs provided by ACI (or VHI) to extract usage for the object storage provided.
+
+## pull overall usage (ACI)
+
 Some code to pull S3 storage usage by tenant.
 
 ```sh
@@ -13,3 +17,6 @@ hci_s3_storage -- 11070.81 GB
 Cyber Infrastructure (Type: hci)
 hci_s3_storage -- 4619.61 GB
 ```
+
+## extract usage for buckets (ACI & VHI)
+

cmd/ostor/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/Luzilla/acronis-s3-usage/internal/cmd"
+	"github.com/Luzilla/acronis-s3-usage/pkg/ostor"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	version = "dev"
+	commit  = "none"
+	date    = "unknown"
+)
+
+func main() {
+	app := &cli.App{
+		Name:     "ostor-client",
+		HelpName: "a program to interact with the s3 management APIs in ACI and VHI",
+		Version:  fmt.Sprintf("%s (%s, date: %s)", version, commit, date),
+		Before: func(cCtx *cli.Context) error {
+			client, err := ostor.New(
+				cCtx.String("s3-endpoint"),
+				cCtx.String("s3-system-key-id"),
+				cCtx.String("s3-system-secret"))
+			if err != nil {
+				return err
+			}
+
+			cCtx.Context = context.WithValue(cCtx.Context, cmd.OstorClient, client)
+			return nil
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "s3-endpoint",
+				EnvVars:  []string{"S3_ENDPOINT"},
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "s3-system-key-id",
+				EnvVars:  []string{"S3_SYSTEM_KEY_ID"},
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "s3-system-secret",
+				EnvVars:  []string{"S3_SYSTEM_SECRET_KEY"},
+				Required: true,
+			},
+		},
+		Commands: []*cli.Command{
+			{
+				Name:    "buckets",
+				Aliases: []string{"b"},
+				Usage:   "list buckets",
+				Action:  cmd.ListBuckets,
+			},
+			{
+				Name:    "stats",
+				Aliases: []string{"s"},
+				Usage:   "list stats",
+				Action:  cmd.List,
+			},
+			{
+				Name:    "users",
+				Aliases: []string{"u"},
+				Usage:   "list users",
+				Action:  cmd.Users,
+				Subcommands: []*cli.Command{
+					{
+						Name: "show",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:     "email",
+								Required: true,
+							},
+						},
+						Action: cmd.ShowUser,
+					},
+					{
+						Name: "create-key",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:     "email",
+								Required: true,
+							},
+						},
+						Action: cmd.CreateKey,
+					},
+					{
+						Name: "revoke-key",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:     "email",
+								Required: true,
+							},
+							&cli.StringFlag{
+								Name:     "key-id",
+								Required: true,
+							},
+						},
+						Action: cmd.RevokeKey,
+					},
+				},
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}

cmd/tenant-usage/main.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"fmt"
+	"math"
 	"os"
 
+	"github.com/Luzilla/acronis-s3-usage/internal/utils"
 	"github.com/Luzilla/acronis-s3-usage/pkg/acronis"
 )
 
@@ -36,12 +38,12 @@ func main() {
 				panic(err)
 			}
 
-			fmt.Printf("%s (Type: %s)\n%s -- %.2f GB\n\n",
+			fmt.Printf("%s (Type: %s)\n%s -- %s\n\n",
 				app.Name,
 				app.Type,
 				usages.Name,
-				// bitshift -> byte to gb
-				(usages.AbsoluteValue / (1 << 30)))
+				utils.PrettyByteSize(int(math.Round(usages.AbsoluteValue))),
+			)
 		}
 	}
 }

go.mod

@@ -1,7 +1,21 @@
 module github.com/Luzilla/acronis-s3-usage
 
-go 1.17
+go 1.21
 
 require github.com/go-resty/resty/v2 v2.7.0
 
-require golang.org/x/net v0.0.0-20211029224645-99673261e6eb // indirect
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+)
+
+require (
+	github.com/fatih/color v1.16.0
+	github.com/rodaine/table v1.1.1
+	github.com/urfave/cli/v2 v2.27.1
+	golang.org/x/net v0.17.0 // indirect
+)

go.sum

@@ -1,9 +1,46 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
 github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
-golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rodaine/table v1.1.1 h1:zBliy3b4Oj6JRmncse2Z85WmoQvDrXOYuy0JXCt8Qz8=
+github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFpfcziA=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
+github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/cmd/buckets.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"github.com/Luzilla/acronis-s3-usage/internal/utils"
+	"github.com/Luzilla/acronis-s3-usage/pkg/ostor"
+	"github.com/rodaine/table"
+	"github.com/urfave/cli/v2"
+)
+
+func ListBuckets(cCtx *cli.Context) error {
+	client := cCtx.Context.Value(OstorClient).(*ostor.Ostor)
+
+	buckets, err := client.GetBuckets("")
+	if err != nil {
+		return err
+	}
+
+	tbl := table.New("Bucket", "Size (current)", "Owner", "Created At")
+	tbl.WithHeaderFormatter(headerFmt()).WithFirstColumnFormatter(columnFmt())
+
+	for _, b := range buckets.Buckets {
+		tbl.AddRow(b.Name, utils.PrettyByteSize(b.Size.Current), b.OwnerID, b.CreatedAt)
+	}
+	tbl.Print()
+
+	return nil
+}

internal/cmd/stats.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Luzilla/acronis-s3-usage/internal/utils"
+	"github.com/Luzilla/acronis-s3-usage/pkg/ostor"
+	"github.com/rodaine/table"
+	"github.com/urfave/cli/v2"
+)
+
+type stat struct {
+	Put        int
+	Get        int
+	List       int
+	Other      int
+	Downloaded int
+	Uploaded   int
+}
+
+func List(cCtx *cli.Context) error {
+	client := cCtx.Context.Value(OstorClient).(*ostor.Ostor)
+
+	items, err := client.List()
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Found %d objects\n", items.Count)
+	fmt.Printf("Truncated: %t\n", items.Truncated)
+
+	if items.Count == 0 {
+		fmt.Println("no items")
+		return nil
+	}
+
+	keep := map[string]stat{}
+
+	for _, obj := range items.Items {
+		// fmt.Println(obj)
+		usage, err := client.ObjectUsage(obj)
+		if err != nil {
+			fmt.Println("usage: " + err.Error())
+			os.Exit(2)
+		}
+
+		for _, item := range usage.Items {
+			var b = item.Key.Bucket
+			if _, ok := keep[b]; !ok {
+				keep[b] = stat{}
+			}
+			keep[b] = addToStruct(keep[b], item.Counters.Operations, item.Counters.Net)
+		}
+	}
+
+	tbl := table.New("Bucket", "Put", "Get", "List", "Other", "Downloaded", "Uploaded")
+	tbl.WithHeaderFormatter(headerFmt()).WithFirstColumnFormatter(columnFmt())
+
+	for bucket, s := range keep {
+		tbl.AddRow(bucket, s.Put, s.Get, s.List, s.Other, utils.PrettyByteSize(s.Downloaded), utils.PrettyByteSize(s.Uploaded))
+	}
+
+	tbl.Print()
+
+	return nil
+}
+
+func addToStruct(k stat, o ostor.ItemCountersOps, n ostor.ItemCountersNet) stat {
+	k.Put = k.Put + o.Put
+	k.Get = k.Get + o.Get
+	k.List = k.List + o.List
+	k.Other = k.Other + o.Other
+	k.Uploaded = k.Uploaded + n.Uploaded
+	k.Downloaded = k.Downloaded + n.Downloaded
+
+	return k
+}

internal/cmd/types.go

@@ -0,0 +1,5 @@
+package cmd
+
+const (
+	OstorClient string = "xxclient"
+)