|
1
|
|
|
/* |
|
2
|
|
|
* Copyright (c) 2023 Clive Walkden <[email protected]> |
|
3
|
|
|
* |
|
4
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy |
|
5
|
|
|
* of this software and associated documentation files (the "Software"), to deal |
|
6
|
|
|
* in the Software without restriction, including without limitation the rights |
|
7
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
8
|
|
|
* copies of the Software, and to permit persons to whom the Software is |
|
9
|
|
|
* furnished to do so, subject to the following conditions: |
|
10
|
|
|
* |
|
11
|
|
|
* The above copyright notice and this permission notice shall be included in all |
|
12
|
|
|
* copies or substantial portions of the Software. |
|
13
|
|
|
* |
|
14
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
|
15
|
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES |
|
16
|
|
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
|
17
|
|
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT |
|
18
|
|
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
|
19
|
|
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
|
20
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
|
21
|
|
|
* OTHER DEALINGS IN THE SOFTWARE. |
|
22
|
|
|
*/ |
|
23
|
|
|
|
|
24
|
|
|
package cmd |
|
25
|
|
|
|
|
26
|
|
|
import ( |
|
27
|
|
|
"context" |
|
28
|
|
|
"fmt" |
|
29
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3" |
|
30
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types" |
|
31
|
|
|
"github.com/spf13/viper" |
|
32
|
|
|
"log" |
|
33
|
|
|
"time" |
|
34
|
|
|
"wasabiCleanup/internal/client/wasabi" |
|
35
|
|
|
"wasabiCleanup/internal/config" |
|
36
|
|
|
"wasabiCleanup/internal/reporting" |
|
37
|
|
|
"wasabiCleanup/internal/utils" |
|
38
|
|
|
|
|
39
|
|
|
"github.com/spf13/cobra" |
|
40
|
|
|
) |
|
41
|
|
|
|
|
42
|
|
|
var ( |
|
43
|
|
|
dryRun bool |
|
44
|
|
|
|
|
45
|
|
|
// cleanCmd represents the clean command |
|
46
|
|
|
cleanCmd = &cobra.Command{ |
|
47
|
|
|
Use: "clean", |
|
48
|
|
|
Short: "Clean up the outdated files.", |
|
49
|
|
|
Run: func(cmd *cobra.Command, args []string) { |
|
50
|
|
|
clean(cmd) |
|
51
|
|
|
}, |
|
52
|
|
|
} |
|
53
|
|
|
) |
|
54
|
|
|
|
|
55
|
|
|
type S3Object struct { |
|
56
|
|
|
Key string |
|
57
|
|
|
LastModified time.Time |
|
58
|
|
|
Size int64 |
|
59
|
|
|
} |
|
60
|
|
|
|
|
61
|
|
|
type S3Objects struct { |
|
62
|
|
|
Items []types.ObjectIdentifier |
|
63
|
|
|
Size int64 |
|
64
|
|
|
} |
|
65
|
|
|
|
|
66
|
|
|
func init() { |
|
67
|
|
|
cleanCmd.Flags().BoolVarP(&dryRun, "dryrun", "n", false, "Show what will be deleted but don't delete it") |
|
68
|
|
|
} |
|
69
|
|
|
|
|
70
|
|
|
func clean(cmd *cobra.Command) { |
|
71
|
|
|
dryRun, _ := cmd.Flags().GetBool("dryrun") |
|
72
|
|
|
verbose, _ := cmd.Flags().GetBool("verbose") |
|
73
|
|
|
|
|
74
|
|
|
client := wasabi.Client() |
|
75
|
|
|
|
|
76
|
|
|
report := reporting.Report{DryRun: dryRun} |
|
77
|
|
|
|
|
78
|
|
|
buckets, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) |
|
79
|
|
|
if err != nil { |
|
80
|
|
|
log.Fatal(err) |
|
81
|
|
|
} |
|
82
|
|
|
|
|
83
|
|
|
log.Println("Working...") |
|
84
|
|
|
for _, object := range buckets.Buckets { |
|
85
|
|
|
if verbose { |
|
86
|
|
|
fmt.Printf("Checking Bucket %s\n", *object.Name) |
|
87
|
|
|
} |
|
88
|
|
|
|
|
89
|
|
|
if config.AppConfig().Buckets[*object.Name] == 0 { |
|
90
|
|
|
if viper.GetBool("verbose") { |
|
91
|
|
|
fmt.Printf("\t- Bucket not in config, skipping\n") |
|
92
|
|
|
} |
|
93
|
|
|
continue |
|
94
|
|
|
} |
|
95
|
|
|
|
|
96
|
|
|
// Return files that need deleting from this bucket based on the Retention Policy |
|
97
|
|
|
objectList := S3Objects{} |
|
98
|
|
|
safeList := S3Objects{} |
|
99
|
|
|
maxKeys := 0 |
|
100
|
|
|
|
|
101
|
|
|
params := &s3.ListObjectsV2Input{Bucket: object.Name} |
|
102
|
|
|
|
|
103
|
|
|
// Create the Paginator for the ListObjectsV2 operation. |
|
104
|
|
|
p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) { |
|
105
|
|
|
if v := int32(maxKeys); v != 0 { |
|
106
|
|
|
o.Limit = v |
|
107
|
|
|
} |
|
108
|
|
|
}) |
|
109
|
|
|
|
|
110
|
|
|
// The date we need to delete items prior to |
|
111
|
|
|
comparisonDate := time.Now().AddDate(0, 0, -config.AppConfig().Buckets[*object.Name]-1) |
|
112
|
|
|
if verbose { |
|
113
|
|
|
fmt.Printf("\t- Checking files date is before %s\n", comparisonDate) |
|
114
|
|
|
} |
|
115
|
|
|
|
|
116
|
|
|
// Iterate through the S3 object pages, printing each object returned. |
|
117
|
|
|
var i int |
|
118
|
|
|
for p.HasMorePages() { |
|
119
|
|
|
i++ |
|
120
|
|
|
|
|
121
|
|
|
// Next Page takes a new context for each page retrieval. This is where |
|
122
|
|
|
// you could add timeouts or deadlines. |
|
123
|
|
|
page, err := p.NextPage(context.TODO()) |
|
124
|
|
|
if err != nil { |
|
125
|
|
|
log.Fatalf("\t\tfailed to get page %v, %v", i, err) |
|
126
|
|
|
} |
|
127
|
|
|
|
|
128
|
|
|
if verbose { |
|
129
|
|
|
fmt.Printf("\t\t- Next page (%d)\n", i) |
|
130
|
|
|
} |
|
131
|
|
|
|
|
132
|
|
|
// Log the objects found |
|
133
|
|
|
for _, obj := range page.Contents { |
|
134
|
|
|
if obj.LastModified.Before(comparisonDate) { |
|
135
|
|
|
objectList.Items = append(objectList.Items, types.ObjectIdentifier{ |
|
136
|
|
|
Key: obj.Key, |
|
137
|
|
|
}) |
|
138
|
|
|
objectList.Size += obj.Size |
|
139
|
|
|
|
|
140
|
|
|
if dryRun { |
|
141
|
|
|
if verbose { |
|
142
|
|
|
fmt.Printf("\t\t\t- Deleting object %s\n", *obj.Key) |
|
143
|
|
|
} else { |
|
144
|
|
|
fmt.Printf("\t- Deleting object %s\n", *obj.Key) |
|
145
|
|
|
} |
|
146
|
|
|
} else { |
|
147
|
|
|
if verbose { |
|
148
|
|
|
fmt.Printf("\t\t\t- Deleting object %s\n", *obj.Key) |
|
149
|
|
|
} |
|
150
|
|
|
_, err = client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ |
|
151
|
|
|
Bucket: object.Name, |
|
152
|
|
|
Key: obj.Key, |
|
153
|
|
|
}) |
|
154
|
|
|
|
|
155
|
|
|
if err != nil { |
|
156
|
|
|
panic("Couldn't delete items") |
|
157
|
|
|
} |
|
158
|
|
|
} |
|
159
|
|
|
} else { |
|
160
|
|
|
safeList.Items = append(safeList.Items, types.ObjectIdentifier{ |
|
161
|
|
|
Key: obj.Key, |
|
162
|
|
|
}) |
|
163
|
|
|
safeList.Size += obj.Size |
|
164
|
|
|
} |
|
165
|
|
|
} |
|
166
|
|
|
} |
|
167
|
|
|
|
|
168
|
|
|
result := reporting.Result{ |
|
169
|
|
|
Name: *object.Name, |
|
170
|
|
|
Kept: len(safeList.Items), |
|
171
|
|
|
KeptSize: utils.ByteCountSI(safeList.Size), |
|
172
|
|
|
Deleted: len(objectList.Items), |
|
173
|
|
|
DeletedSize: utils.ByteCountSI(objectList.Size), |
|
174
|
|
|
} |
|
175
|
|
|
|
|
176
|
|
|
report.Result = append(report.Result, result) |
|
177
|
|
|
} |
|
178
|
|
|
|
|
179
|
|
|
reporting.Output(report) |
|
180
|
|
|
} |
|
181
|
|
|
|
clivewalkden /
go-wasabi-cleanup
