|
1
|
|
|
/* |
|
2
|
|
|
* Copyright (c) 2023 Clive Walkden <[email protected]> |
|
3
|
|
|
* |
|
4
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy |
|
5
|
|
|
* of this software and associated documentation files (the "Software"), to deal |
|
6
|
|
|
* in the Software without restriction, including without limitation the rights |
|
7
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
8
|
|
|
* copies of the Software, and to permit persons to whom the Software is |
|
9
|
|
|
* furnished to do so, subject to the following conditions: |
|
10
|
|
|
* |
|
11
|
|
|
* The above copyright notice and this permission notice shall be included in all |
|
12
|
|
|
* copies or substantial portions of the Software. |
|
13
|
|
|
* |
|
14
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
|
15
|
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES |
|
16
|
|
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
|
17
|
|
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT |
|
18
|
|
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
|
19
|
|
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
|
20
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
|
21
|
|
|
* OTHER DEALINGS IN THE SOFTWARE. |
|
22
|
|
|
*/ |
|
23
|
|
|
|
|
24
|
|
|
package cmd |
|
25
|
|
|
|
|
26
|
|
|
import ( |
|
27
|
|
|
"context" |
|
28
|
|
|
"fmt" |
|
29
|
|
|
"github.com/aws/aws-sdk-go-v2/aws" |
|
30
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3" |
|
31
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types" |
|
32
|
|
|
"github.com/spf13/viper" |
|
33
|
|
|
"log" |
|
34
|
|
|
"time" |
|
35
|
|
|
"wasabiCleanup/internal/client/wasabi" |
|
36
|
|
|
"wasabiCleanup/internal/config" |
|
37
|
|
|
"wasabiCleanup/internal/reporting" |
|
38
|
|
|
"wasabiCleanup/internal/utils" |
|
39
|
|
|
|
|
40
|
|
|
"github.com/spf13/cobra" |
|
41
|
|
|
) |
|
42
|
|
|
|
|
43
|
|
|
var ( |
|
44
|
|
|
dryRun bool |
|
45
|
|
|
|
|
46
|
|
|
// cleanCmd represents the clean command |
|
47
|
|
|
cleanCmd = &cobra.Command{ |
|
48
|
|
|
Use: "clean", |
|
49
|
|
|
Short: "Clean up the outdated files.", |
|
50
|
|
|
Run: func(cmd *cobra.Command, args []string) { |
|
51
|
|
|
clean(cmd) |
|
52
|
|
|
}, |
|
53
|
|
|
} |
|
54
|
|
|
) |
|
55
|
|
|
|
|
56
|
|
|
// S3Client is an interface that includes the methods we need from s3.Client. |
|
57
|
|
|
type S3Client interface { |
|
58
|
|
|
ListBuckets(ctx context.Context, params *s3.ListBucketsInput, optFns ...func(*s3.Options)) (*s3.ListBucketsOutput, error) |
|
59
|
|
|
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) |
|
60
|
|
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) |
|
61
|
|
|
} |
|
62
|
|
|
|
|
63
|
|
|
// S3Object represents an object in an S3 bucket. |
|
64
|
|
|
type S3Object struct { |
|
65
|
|
|
Key string |
|
66
|
|
|
LastModified time.Time |
|
67
|
|
|
Size int64 |
|
68
|
|
|
} |
|
69
|
|
|
|
|
70
|
|
|
// S3Objects represents a list of objects in an S3 bucket. |
|
71
|
|
|
type S3Objects struct { |
|
72
|
|
|
Items []types.ObjectIdentifier |
|
73
|
|
|
Size int64 |
|
74
|
|
|
} |
|
75
|
|
|
|
|
76
|
|
|
// init initializes the clean command and its flags. |
|
77
|
|
|
func init() { |
|
78
|
|
|
cleanCmd.Flags().BoolVarP(&dryRun, "dryrun", "n", false, "Show what will be deleted but don't delete it") |
|
79
|
|
|
} |
|
80
|
|
|
|
|
81
|
|
|
// GetBuckets is a function that retrieves a list of all buckets from the provided S3 client. |
|
82
|
|
|
// It returns a slice of Bucket objects and an error. If the operation is successful, the error is nil. |
|
83
|
|
|
// If there is an error during the operation, the function returns nil and the error. |
|
84
|
|
|
// |
|
85
|
|
|
// Parameters: |
|
86
|
|
|
// client: An instance of an S3 client. |
|
87
|
|
|
// |
|
88
|
|
|
// Returns: |
|
89
|
|
|
// []types.Bucket: A slice of Bucket objects representing all the buckets retrieved from the S3 client. |
|
90
|
|
|
// error: An error that will be nil if the operation is successful, and an error object if the operation fails. |
|
91
|
|
|
func GetBuckets(client S3Client) ([]types.Bucket, error) { |
|
92
|
|
|
buckets, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) |
|
93
|
|
|
if err != nil { |
|
94
|
|
|
return nil, err |
|
95
|
|
|
} |
|
96
|
|
|
return buckets.Buckets, nil |
|
97
|
|
|
} |
|
98
|
|
|
|
|
99
|
|
|
// ProcessBucket processes a single bucket. It checks if the bucket is in the config and if it needs to be cleaned. |
|
100
|
|
|
func ProcessBucket(bucket types.Bucket, client S3Client, dryRun bool, verbose bool) (reporting.Result, error) { |
|
101
|
|
|
if verbose { |
|
102
|
|
|
fmt.Printf("Checking Bucket %s\n", *bucket.Name) |
|
103
|
|
|
} |
|
104
|
|
|
|
|
105
|
|
|
if config.AppConfig().Buckets[*bucket.Name] == 0 { |
|
106
|
|
|
if viper.GetBool("verbose") { |
|
107
|
|
|
fmt.Printf("\t- Bucket not in config, skipping\n") |
|
108
|
|
|
} |
|
109
|
|
|
return reporting.Result{}, nil |
|
110
|
|
|
} |
|
111
|
|
|
|
|
112
|
|
|
// The date we need to delete items prior to |
|
113
|
|
|
comparisonDate := time.Now().AddDate(0, 0, -config.AppConfig().Buckets[*bucket.Name]-1) |
|
114
|
|
|
if verbose { |
|
115
|
|
|
fmt.Printf("\t- Checking files date is before %s\n", comparisonDate) |
|
116
|
|
|
} |
|
117
|
|
|
|
|
118
|
|
|
objectList, safeList, err := DeleteOldObjects(bucket, client, comparisonDate, dryRun, verbose) |
|
119
|
|
|
if err != nil { |
|
120
|
|
|
return reporting.Result{}, err |
|
121
|
|
|
} |
|
122
|
|
|
|
|
123
|
|
|
result := reporting.Result{ |
|
124
|
|
|
Name: *bucket.Name, |
|
125
|
|
|
Kept: len(safeList.Items), |
|
126
|
|
|
KeptSize: utils.ByteCountSI(safeList.Size), |
|
127
|
|
|
Deleted: len(objectList.Items), |
|
128
|
|
|
DeletedSize: utils.ByteCountSI(objectList.Size), |
|
129
|
|
|
} |
|
130
|
|
|
|
|
131
|
|
|
return result, nil |
|
132
|
|
|
} |
|
133
|
|
|
|
|
134
|
|
|
// DeleteOldObjects deletes objects in a bucket that are older than the comparison date. |
|
135
|
|
|
func DeleteOldObjects(bucket types.Bucket, client S3Client, comparisonDate time.Time, dryRun bool, verbose bool) (S3Objects, S3Objects, error) { |
|
136
|
|
|
objectList := S3Objects{} |
|
137
|
|
|
safeList := S3Objects{} |
|
138
|
|
|
|
|
139
|
|
|
params := &s3.ListObjectsV2Input{Bucket: bucket.Name} |
|
140
|
|
|
p := s3.NewListObjectsV2Paginator(client, params) |
|
141
|
|
|
|
|
142
|
|
|
// Loop through the objects and delete any that are older than the comparison date |
|
143
|
|
|
for p.HasMorePages() { |
|
144
|
|
|
page, err := p.NextPage(context.TODO()) |
|
145
|
|
|
if err != nil { |
|
146
|
|
|
return S3Objects{}, S3Objects{}, err |
|
147
|
|
|
} |
|
148
|
|
|
|
|
149
|
|
|
for _, obj := range page.Contents { |
|
150
|
|
|
if obj.LastModified.Before(comparisonDate) { |
|
151
|
|
|
objectList.Items = append(objectList.Items, types.ObjectIdentifier{ |
|
152
|
|
|
Key: obj.Key, |
|
153
|
|
|
}) |
|
154
|
|
|
objectList.Size += aws.ToInt64(obj.Size) |
|
155
|
|
|
|
|
156
|
|
|
if dryRun { |
|
157
|
|
|
if verbose { |
|
158
|
|
|
fmt.Printf("\t\t\t- Deleting object %s\n", *obj.Key) |
|
159
|
|
|
} else { |
|
160
|
|
|
fmt.Printf("\t- Deleting object %s\n", *obj.Key) |
|
161
|
|
|
} |
|
162
|
|
|
} else { |
|
163
|
|
|
if verbose { |
|
164
|
|
|
fmt.Printf("\t\t\t- Deleting object %s\n", *obj.Key) |
|
165
|
|
|
} |
|
166
|
|
|
_, err = client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ |
|
167
|
|
|
Bucket: bucket.Name, |
|
168
|
|
|
Key: obj.Key, |
|
169
|
|
|
}) |
|
170
|
|
|
|
|
171
|
|
|
if err != nil { |
|
172
|
|
|
return S3Objects{}, S3Objects{}, err |
|
173
|
|
|
} |
|
174
|
|
|
} |
|
175
|
|
|
} else { |
|
176
|
|
|
safeList.Items = append(safeList.Items, types.ObjectIdentifier{ |
|
177
|
|
|
Key: obj.Key, |
|
178
|
|
|
}) |
|
179
|
|
|
safeList.Size += aws.ToInt64(obj.Size) |
|
180
|
|
|
} |
|
181
|
|
|
} |
|
182
|
|
|
} |
|
183
|
|
|
|
|
184
|
|
|
return objectList, safeList, nil |
|
185
|
|
|
} |
|
186
|
|
|
|
|
187
|
|
|
// CreateReport creates a report based on the results of the cleaning process. |
|
188
|
|
|
func CreateReport(results []reporting.Result) reporting.Report { |
|
189
|
|
|
report := reporting.Report{Result: results} |
|
190
|
|
|
return report |
|
191
|
|
|
} |
|
192
|
|
|
|
|
193
|
|
|
// clean is the main function for the clean command. It retrieves the list of buckets, processes each bucket, and outputs a report. |
|
194
|
|
|
func clean(cmd *cobra.Command) { |
|
195
|
|
|
dryRun, _ := cmd.Flags().GetBool("dryrun") |
|
196
|
|
|
verbose, _ := cmd.Flags().GetBool("verbose") |
|
197
|
|
|
|
|
198
|
|
|
client := wasabi.Client() |
|
199
|
|
|
|
|
200
|
|
|
buckets, err := GetBuckets(client) |
|
201
|
|
|
if err != nil { |
|
202
|
|
|
log.Fatal(err) |
|
203
|
|
|
} |
|
204
|
|
|
|
|
205
|
|
|
log.Println("Working...") |
|
206
|
|
|
var results []reporting.Result |
|
207
|
|
|
for _, bucket := range buckets { |
|
208
|
|
|
result, err := ProcessBucket(bucket, client, dryRun, verbose) |
|
209
|
|
|
if err != nil { |
|
210
|
|
|
log.Fatal(err) |
|
211
|
|
|
} |
|
212
|
|
|
results = append(results, result) |
|
213
|
|
|
} |
|
214
|
|
|
|
|
215
|
|
|
report := CreateReport(results) |
|
216
|
|
|
reporting.Output(report) |
|
217
|
|
|
} |
|
218
|
|
|
|
clivewalkden /
go-wasabi-cleanup
