cmd.GetBuckets - Code Metrics - clivewalkden/go-wasabi-cleanup - Measure and Improve Code Quality continuously with Scrutinizer

cmd.GetBuckets A
last analyzed 2025-02-18 11:56 UTC

↳ Parent: cmd/clean.go

Complexity

Conditions

Size

Total Lines	6
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	5
nop	1
dl	0
loc	6
rs	10
c	0
b	0
f	0

/*
 * Copyright (c) 2023 Clive Walkden <[email protected]>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

package cmd

import (
	"context"
	"fmt"
	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/spf13/viper"
	"log"
	"time"
	"wasabi-cleanup/internal/client/wasabi"
	"wasabi-cleanup/internal/config"
	"wasabi-cleanup/internal/reporting"
	"wasabi-cleanup/internal/utils"

	"github.com/spf13/cobra"
)

var (
	dryRun bool

	// cleanCmd represents the clean command
	cleanCmd = &cobra.Command{
		Use:   "clean",
		Short: "Clean up the outdated files.",
		Run: func(cmd *cobra.Command, args []string) {
			clean(cmd)
		},
	}
)

// S3Client is an interface that includes the methods we need from s3.Client.
type S3Client interface {
	ListBuckets(ctx context.Context, params *s3.ListBucketsInput, optFns ...func(*s3.Options)) (*s3.ListBucketsOutput, error)
	DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
}

// S3Object represents an object in an S3 bucket.
type S3Object struct {
	Key          string
	LastModified time.Time
	Size         int64
}

// S3Objects represents a list of objects in an S3 bucket.
type S3Objects struct {
	Items []types.ObjectIdentifier
	Size  int64
}

// init initializes the clean command and its flags.
func init() {
	cleanCmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Show what will be deleted but don't delete it")
}

// GetBuckets is a function that retrieves a list of all buckets from the provided S3 client.
// It returns a slice of Bucket objects and an error. If the operation is successful, the error is nil.
// If there is an error during the operation, the function returns nil and the error.
//
// Parameters:
// client: An instance of an S3 client.
//
// Returns:
// []types.Bucket: A slice of Bucket objects representing all the buckets retrieved from the S3 client.
// error: An error that will be nil if the operation is successful, and an error object if the operation fails.
func GetBuckets(client S3Client) ([]types.Bucket, error) {
	buckets, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
	if err != nil {
		return nil, err
	}
	return buckets.Buckets, nil
}

// ProcessBucket processes a single bucket. It checks if the bucket is in the config and if it needs to be cleaned.
func ProcessBucket(bucket types.Bucket, client S3Client, dryRun bool, verbose bool) (reporting.Result, error) {
	if verbose {
		fmt.Printf("Checking Bucket %s\n", *bucket.Name)
	}

	if config.AppConfig().Buckets[*bucket.Name] == 0 {
		if viper.GetBool("verbose") {
			fmt.Printf("\t- Bucket not in config, skipping\n")
		}
		return reporting.Result{}, nil
	}

	// The date we need to delete items prior to
	comparisonDate := time.Now().AddDate(0, 0, -config.AppConfig().Buckets[*bucket.Name]-1)
	if verbose {
		fmt.Printf("\t- Checking files date is before %s\n", comparisonDate)
	}

	objectList, safeList, err := DeleteOldObjects(bucket, client, comparisonDate, dryRun, verbose)
	if err != nil {
		return reporting.Result{}, err
	}

	result := reporting.Result{
		Name:        *bucket.Name,
		Kept:        len(safeList.Items),
		KeptSize:    utils.ByteCountSI(safeList.Size),
		Deleted:     len(objectList.Items),
		DeletedSize: utils.ByteCountSI(objectList.Size),
	}

	return result, nil
}

// GetObjects retrieves objects from the bucket.
func GetObjects(bucket types.Bucket, client S3Client) ([]types.Object, error) {
	params := &s3.ListObjectsV2Input{Bucket: bucket.Name}
	p := s3.NewListObjectsV2Paginator(client, params)

	var objects []types.Object
	for p.HasMorePages() {
		page, err := p.NextPage(context.TODO())
		if err != nil {
			return nil, err
		}
		objects = append(objects, page.Contents...)
	}

	return objects, nil
}

// DeleteObject deletes an object from the bucket.
func DeleteObject(bucket types.Bucket, client S3Client, object types.Object) error {
	_, err := client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
		Bucket: bucket.Name,
		Key:    object.Key,
	})

	return err
}

// DeleteOldObjects deletes objects in a bucket that are older than the comparison date.
func DeleteOldObjects(bucket types.Bucket, client S3Client, comparisonDate time.Time, dryRun bool, verbose bool) (S3Objects, S3Objects, error) {
	objectList := S3Objects{}
	safeList := S3Objects{}

	objects, err := GetObjects(bucket, client)
	if err != nil {
		return S3Objects{}, S3Objects{}, err
	}

	for _, obj := range objects {
		if obj.LastModified.Before(comparisonDate) {
			objectList.Items = append(objectList.Items, types.ObjectIdentifier{
				Key: obj.Key,
			})
			objectList.Size += aws.ToInt64(obj.Size)

			if dryRun {
				if verbose {
					fmt.Printf("\t\t- Deleting object %s\n", *obj.Key)
				} else {
					fmt.Printf("\t- Deleting object %s\n", *obj.Key)
				}
			} else {
				if verbose {
					fmt.Printf("\t\t- Deleting object %s\n", *obj.Key)
				}
				err = DeleteObject(bucket, client, obj)
				if err != nil {
					return S3Objects{}, S3Objects{}, err
				}
			}
		} else {
			safeList.Items = append(safeList.Items, types.ObjectIdentifier{
				Key: obj.Key,
			})
			safeList.Size += aws.ToInt64(obj.Size)
		}
	}

	return objectList, safeList, nil
}

// CreateReport creates a report based on the results of the cleaning process.
func CreateReport(results []reporting.Result) reporting.Report {
	report := reporting.Report{Result: results}
	return report
}

// clean is the main function for the clean command. It retrieves the list of buckets, processes each bucket, and outputs a report.
func clean(cmd *cobra.Command) {
	dryRun, _ := cmd.Flags().GetBool("dry-run")
	verbose, _ := cmd.Flags().GetBool("verbose")

	client := wasabi.Client()

	buckets, err := GetBuckets(client)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Working...")
	var results []reporting.Result
	for _, bucket := range buckets {
		result, err := ProcessBucket(bucket, client, dryRun, verbose)
		if err != nil {
			log.Fatal(err)
		}
		results = append(results, result)
	}

	report := CreateReport(results)
	reporting.Output(report)
}


1			/*
2			* Copyright (c) 2023 Clive Walkden <[email protected]>
3			*
4			* Permission is hereby granted, free of charge, to any person obtaining a copy
5			* of this software and associated documentation files (the "Software"), to deal
6			* in the Software without restriction, including without limitation the rights
7			* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8			* copies of the Software, and to permit persons to whom the Software is
9			* furnished to do so, subject to the following conditions:
10			*
11			* The above copyright notice and this permission notice shall be included in all
12			* copies or substantial portions of the Software.
13			*
14			* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15			* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16			* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17			* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
18			* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
19			* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20			* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21			* OTHER DEALINGS IN THE SOFTWARE.
22			*/
23
24			package cmd
25
26			import (
27			"context"
28			"fmt"
29			"github.com/aws/aws-sdk-go-v2/aws"
30			"github.com/aws/aws-sdk-go-v2/service/s3"
31			"github.com/aws/aws-sdk-go-v2/service/s3/types"
32			"github.com/spf13/viper"
33			"log"
34			"time"
35			"wasabi-cleanup/internal/client/wasabi"
36			"wasabi-cleanup/internal/config"
37			"wasabi-cleanup/internal/reporting"
38			"wasabi-cleanup/internal/utils"
39
40			"github.com/spf13/cobra"
41			)
42
43			var (
44			dryRun bool
45
46			// cleanCmd represents the clean command
47			cleanCmd = &cobra.Command{
48			Use: "clean",
49			Short: "Clean up the outdated files.",
50			Run: func(cmd *cobra.Command, args []string) {
51			clean(cmd)
52			},
53			}
54			)
55
56			// S3Client is an interface that includes the methods we need from s3.Client.
57			type S3Client interface {
58			ListBuckets(ctx context.Context, params s3.ListBucketsInput, optFns ...func(s3.Options)) (*s3.ListBucketsOutput, error)
59			DeleteObject(ctx context.Context, params s3.DeleteObjectInput, optFns ...func(s3.Options)) (*s3.DeleteObjectOutput, error)
60			ListObjectsV2(ctx context.Context, params s3.ListObjectsV2Input, optFns ...func(s3.Options)) (*s3.ListObjectsV2Output, error)
61			}
62
63			// S3Object represents an object in an S3 bucket.
64			type S3Object struct {
65			Key string
66			LastModified time.Time
67			Size int64
68			}
69
70			// S3Objects represents a list of objects in an S3 bucket.
71			type S3Objects struct {
72			Items []types.ObjectIdentifier
73			Size int64
74			}
75
76			// init initializes the clean command and its flags.
77			func init() {
78			cleanCmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Show what will be deleted but don't delete it")
79			}
80
81			// GetBuckets is a function that retrieves a list of all buckets from the provided S3 client.
82			// It returns a slice of Bucket objects and an error. If the operation is successful, the error is nil.
83			// If there is an error during the operation, the function returns nil and the error.
84			//
85			// Parameters:
86			// client: An instance of an S3 client.
87			//
88			// Returns:
89			// []types.Bucket: A slice of Bucket objects representing all the buckets retrieved from the S3 client.
90			// error: An error that will be nil if the operation is successful, and an error object if the operation fails.
91			func GetBuckets(client S3Client) ([]types.Bucket, error) {
92			buckets, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
93			if err != nil {
94			return nil, err
95			}
96			return buckets.Buckets, nil
97			}
98
99			// ProcessBucket processes a single bucket. It checks if the bucket is in the config and if it needs to be cleaned.
100			func ProcessBucket(bucket types.Bucket, client S3Client, dryRun bool, verbose bool) (reporting.Result, error) {
101			if verbose {
102			fmt.Printf("Checking Bucket %s\n", *bucket.Name)
103			}
104
105			if config.AppConfig().Buckets[*bucket.Name] == 0 {
106			if viper.GetBool("verbose") {
107			fmt.Printf("\t- Bucket not in config, skipping\n")
108			}
109			return reporting.Result{}, nil
110			}
111
112			// The date we need to delete items prior to
113			comparisonDate := time.Now().AddDate(0, 0, -config.AppConfig().Buckets[*bucket.Name]-1)
114			if verbose {
115			fmt.Printf("\t- Checking files date is before %s\n", comparisonDate)
116			}
117
118			objectList, safeList, err := DeleteOldObjects(bucket, client, comparisonDate, dryRun, verbose)
119			if err != nil {
120			return reporting.Result{}, err
121			}
122
123			result := reporting.Result{
124			Name: *bucket.Name,
125			Kept: len(safeList.Items),
126			KeptSize: utils.ByteCountSI(safeList.Size),
127			Deleted: len(objectList.Items),
128			DeletedSize: utils.ByteCountSI(objectList.Size),
129			}
130
131			return result, nil
132			}
133
134			// GetObjects retrieves objects from the bucket.
135			func GetObjects(bucket types.Bucket, client S3Client) ([]types.Object, error) {
136			params := &s3.ListObjectsV2Input{Bucket: bucket.Name}
137			p := s3.NewListObjectsV2Paginator(client, params)
138
139			var objects []types.Object
140			for p.HasMorePages() {
141			page, err := p.NextPage(context.TODO())
142			if err != nil {
143			return nil, err
144			}
145			objects = append(objects, page.Contents...)
146			}
147
148			return objects, nil
149			}
150
151			// DeleteObject deletes an object from the bucket.
152			func DeleteObject(bucket types.Bucket, client S3Client, object types.Object) error {
153			_, err := client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
154			Bucket: bucket.Name,
155			Key: object.Key,
156			})
157
158			return err
159			}
160
161			// DeleteOldObjects deletes objects in a bucket that are older than the comparison date.
162			func DeleteOldObjects(bucket types.Bucket, client S3Client, comparisonDate time.Time, dryRun bool, verbose bool) (S3Objects, S3Objects, error) {
163			objectList := S3Objects{}
164			safeList := S3Objects{}
165
166			objects, err := GetObjects(bucket, client)
167			if err != nil {
168			return S3Objects{}, S3Objects{}, err
169			}
170
171			for _, obj := range objects {
172			if obj.LastModified.Before(comparisonDate) {
173			objectList.Items = append(objectList.Items, types.ObjectIdentifier{
174			Key: obj.Key,
175			})
176			objectList.Size += aws.ToInt64(obj.Size)
177
178			if dryRun {
179			if verbose {
180			fmt.Printf("\t\t- Deleting object %s\n", *obj.Key)
181			} else {
182			fmt.Printf("\t- Deleting object %s\n", *obj.Key)
183			}
184			} else {
185			if verbose {
186			fmt.Printf("\t\t- Deleting object %s\n", *obj.Key)
187			}
188			err = DeleteObject(bucket, client, obj)
189			if err != nil {
190			return S3Objects{}, S3Objects{}, err
191			}
192			}
193			} else {
194			safeList.Items = append(safeList.Items, types.ObjectIdentifier{
195			Key: obj.Key,
196			})
197			safeList.Size += aws.ToInt64(obj.Size)
198			}
199			}
200
201			return objectList, safeList, nil
202			}
203
204			// CreateReport creates a report based on the results of the cleaning process.
205			func CreateReport(results []reporting.Result) reporting.Report {
206			report := reporting.Report{Result: results}
207			return report
208			}
209
210			// clean is the main function for the clean command. It retrieves the list of buckets, processes each bucket, and outputs a report.
211			func clean(cmd *cobra.Command) {
212			dryRun, _ := cmd.Flags().GetBool("dry-run")
213			verbose, _ := cmd.Flags().GetBool("verbose")
214
215			client := wasabi.Client()
216
217			buckets, err := GetBuckets(client)
218			if err != nil {
219			log.Fatal(err)
220			}
221
222			log.Println("Working...")
223			var results []reporting.Result
224			for _, bucket := range buckets {
225			result, err := ProcessBucket(bucket, client, dryRun, verbose)
226			if err != nil {
227			log.Fatal(err)
228			}
229			results = append(results, result)
230			}
231
232			report := CreateReport(results)
233			reporting.Output(report)
234			}
235

clivewalkden / go-wasabi-cleanup

cmd.GetBuckets A last analyzed 2025-02-18 11:56 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

cmd.GetBuckets A
last analyzed 2025-02-18 11:56 UTC