- Added prune command and a run file.

main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mitchellh/cli"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:]))
+}
+
+func run(args []string) int {
+	// Initialize cli
+	c := &cli.CLI{
+		Name:    "aws-s3",
+		Version: "1.0.0",
+		Commands: map[string]cli.CommandFactory{
+			"prune": pruneCommandFactory,
+		},
+		Args: args,
+	}
+
+	// Run cli
+	exitCode, err := c.Run()
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	return exitCode
+}

objects.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"time"
+)
+
+// A Object provides details of an S3 object
+type Object struct {
+	Bucket       string
+	Key          string
+	Folder       string
+	File         string
+	LastModified time.Time
+	Age          time.Duration
+}
+
+// SortalbeObjects provides sorting of S3 objects
+type SortalbeObjects []Object
+
+func (s SortalbeObjects) Len() int      { return len(s) }
+func (s SortalbeObjects) Swap(a, b int) { s[a], s[b] = s[b], s[a] }
+func (s SortalbeObjects) Less(a, b int) bool {
+	if s[a].Key < s[b].Key {
+		return true
+	}
+
+	return false
+}
+
+func sortObjects(objs []Object) {
+	s := SortalbeObjects(objs)
+	sort.Sort(s)
+}
+
+func printObjects(objs []Object, includeFiles bool, includeDirectories bool) {
+	folder := ""
+	for _, obj := range objs {
+		if obj.Folder != folder && includeDirectories {
+			folder = obj.Folder
+			fmt.Printf("  %s\n", folder)
+		}
+
+		if includeFiles {
+			fmt.Printf("    %s (modified=%v age=%vh)\n", obj.File, obj.LastModified, math.Round(obj.Age.Hours()))
+		}
+	}
+}
+
+func countFolders(objs []Object) int {
+	folder := ""
+	folders := 0
+	for _, obj := range objs {
+		if obj.Folder != folder {
+			folder = obj.Folder
+			folders++
+		}
+	}
+	return folders
+}

prune_command.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mitchellh/cli"
+)
+
+func pruneCommandFactory() (cli.Command, error) {
+	c := &pruneCommand{}
+	c.Flags = flag.NewFlagSet("prune", flag.ExitOnError)
+	c.Bucket = c.Flags.String("bucket", "", "Bucket name")
+	c.Region = c.Flags.String("region", "eu-central-1", "Region")
+	c.Prefix = c.Flags.String("prefix", "", "Key prefix")
+	c.MaxAge = c.Flags.Duration("max-age", 24*time.Hour, "Max age")
+	c.DryRun = c.Flags.Bool("dry-run", false, "Dry run")
+	c.ListFiles = c.Flags.Bool("list-files", false, "List files")
+	c.ListFolders = c.Flags.Bool("list-folders", true, "List folders")
+	return c, nil
+}
+
+// PruneCommand removes old s3 objects
+type pruneCommand struct {
+	Flags       *flag.FlagSet
+	Bucket      *string
+	Region      *string
+	Prefix      *string
+	MaxAge      *time.Duration
+	DryRun      *bool
+	ListFiles   *bool
+	ListFolders *bool
+}
+
+func (c *pruneCommand) Run(args []string) int {
+	c.Flags.Parse(args)
+
+	bucket := *c.Bucket
+	region := *c.Region
+	prefix := *c.Prefix
+	maxAge := *c.MaxAge
+	dryrun := *c.DryRun
+
+	if bucket == "" || region == "" || prefix == "" {
+		c.Flags.PrintDefaults()
+		os.Exit(1)
+	}
+
+	start := time.Now()
+	fmt.Printf("searching for matching s3 objects (bucket=%s region=%s prefix=%s max-age=%s dry-run=%v)\n", bucket, region, prefix, maxAge, dryrun)
+
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region: aws.String(region),
+	}))
+
+	svc := s3.New(sess)
+
+	removeObjs, keepObjs, err := searchObjects(svc, bucket, region, prefix, start, maxAge)
+	if err == nil {
+		removeObjects(svc, bucket, removeObjs, dryrun, *c.ListFiles, *c.ListFolders)
+		keepObjects(keepObjs, dryrun, *c.ListFiles, *c.ListFolders)
+	} else {
+		fmt.Println(err)
+	}
+
+	return 0
+}
+
+func (c *pruneCommand) Help() string {
+	fmt.Println("Available flags are:")
+	c.Flags.PrintDefaults()
+	return ""
+}
+
+func (c *pruneCommand) Synopsis() string {
+	return "Runs cleanup of s3 objects"
+}
+
+func removeObjects(svc *s3.S3, bucket string, objs []Object, dryrun bool, listFiles bool, listDirectories bool) {
+	batchSize := 1000
+
+	sortObjects(objs)
+	folders := countFolders(objs)
+
+	if dryrun {
+		fmt.Printf("would remove %v s3 objects from %v folders\n", len(objs), folders)
+		printObjects(objs, listFiles, listDirectories)
+		return
+	}
+
+	fmt.Printf("removing %v s3 objects from %v folders in batches of %v\n", len(objs), folders, batchSize)
+	printObjects(objs, listFiles, listDirectories)
+
+	if len(objs) == 0 {
+		return
+	}
+
+	ids := make([]*s3.ObjectIdentifier, 0)
+	for _, obj := range objs {
+		ids = append(ids, &s3.ObjectIdentifier{
+			Key: aws.String(obj.Key),
+		})
+	}
+
+	batches := make([][]*s3.ObjectIdentifier, 0)
+	for i := 0; i < len(ids); i += batchSize {
+		end := i + batchSize
+
+		if end > len(ids) {
+			end = len(ids)
+		}
+
+		batches = append(batches, ids[i:end])
+	}
+
+	for _, batch := range batches {
+		removeObjectsBatch(svc, bucket, batch)
+	}
+}
+
+func removeObjectsBatch(svc *s3.S3, bucket string, ids []*s3.ObjectIdentifier) {
+	input := &s3.DeleteObjectsInput{
+		Bucket: aws.String(bucket),
+		Delete: &s3.Delete{
+			Objects: ids,
+			Quiet:   aws.Bool(false),
+		},
+	}
+
+	result, err := svc.DeleteObjects(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				fmt.Println(aerr.Error())
+			}
+		} else {
+			fmt.Println(err.Error())
+		}
+		return
+	}
+
+	fmt.Printf("successfully deleted %v s3 objects, %v error(s)\n", len(result.Deleted), len(result.Errors))
+}
+
+func keepObjects(objs []Object, dryrun bool, listFiles bool, listDirectories bool) {
+	sortObjects(objs)
+	folders := countFolders(objs)
+
+	if dryrun {
+		fmt.Printf("would keep %v s3 objects in %v folders\n", len(objs), folders)
+	} else {
+		fmt.Printf("keeping %v s3 objects in %v folders in\n", len(objs), folders)
+	}
+
+	printObjects(objs, listFiles, listDirectories)
+}
+
+func searchObjects(svc *s3.S3, bucket string, region string, prefix string, start time.Time, maxAge time.Duration) ([]Object, []Object, error) {
+	allObjs := make([]Object, 0)
+	removeObjs := make([]Object, 0)
+	keepObjs := make([]Object, 0)
+
+	i := 0
+	err := svc.ListObjectsPages(&s3.ListObjectsInput{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(prefix),
+	}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) {
+		i++
+		for _, obj := range p.Contents {
+			allObjs = append(allObjs, Object{
+				Bucket: bucket,
+				Key:    *obj.Key,
+			})
+		}
+		return true
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, obj := range allObjs {
+		objData, err := svc.HeadObject(&s3.HeadObjectInput{
+			Bucket: &bucket,
+			Key:    &obj.Key,
+		})
+
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if objData.LastModified != nil {
+			obj.LastModified = *objData.LastModified
+		}
+
+		parts := strings.Split(obj.Key, "/")
+		obj.File = parts[len(parts)-1]
+		obj.Folder = strings.Join(parts[:len(parts)-1], "/")
+
+		obj.Age = start.Sub(obj.LastModified)
+		if obj.Age > maxAge {
+			removeObjs = append(removeObjs, obj)
+		} else {
+			keepObjs = append(keepObjs, obj)
+		}
+	}
+
+	return removeObjs, keepObjs, nil
+}

run

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+if [[ -f ./.env ]]; then
+  # shellcheck disable=SC1091
+  source ./.env
+fi
+
+go build -o ./aws-s3
+./aws-s3 "${@}"