feat(app): add Directory Mode for clearing Directory Buckets for S3 E…

…xpress One Zone (#189)

* wip: without readme and tests and refactor

directoryBucketsMode

* use copy

* Revert "use copy"

This reverts commit e80645a.

* refactor

* deploy sh

* fix deploy.sh

* comments

* comments

* comment

* comments

* sort by directory bucket name

* TestS3_ListObjectsByPage

* TestS3_ListDirectoryBuckets

* TestS3_CheckBucketExists

* TestS3Wrapper_ListBucketNamesFilteredByKeyword

* TestS3Wrapper_ClearS3Objects

* fix TestS3Wrapper_ClearS3Objects

* test for sort

* test sh

* readme and action.yml

* readme

* readme

* readme

* testdata deploy sh

* warn

* change

* comments

* comments

* change comments

README.md

@@ -28,9 +28,11 @@ As described below ([Interactive Mode](#interactive-mode)), you can **search for
 
 ### Cross-region
 
-In deleting multiple buckets, you can delete them all at once, even if they are in **multiple regions**.
+In deleting multiple buckets, you can list and delete them all at once, even if they are in **multiple regions**.
 
-### Versioning
+(In the **Directory Buckets** Mode for **S3 Express One Zone** (`-d` option), operation across regions is not possible, but only in **one region**. You can specify the region with the `-r` option.)
+
+### Deletion of buckets with versioning enabled
 
 **Even if versioning is turned on**, you can empty it just as if it were turned off. Therefore, you can use it **without** being aware of the versioning settings.
 
@@ -42,6 +44,12 @@ The `-o | --oldVersionsOnly` option allows you to **delete only old versions** a
 
 This option cannot be specified with the `-f | --force` option.
 
+### Deletion of Directory Buckets for S3 Express One Zone
+
+The `-d | --directoryBucketsMode` option allows you to delete **the Directory Buckets** for **S3 Express One Zone**.
+
+In this mode, operation across regions is not possible, but only in **one region**. You can specify the region with the `-r` option.
+
 ### Number of objects that can be deleted
 
 The delete-objects API provided by the CLI and SDK has a limit of **"the number of objects that can be deleted in one command is limited to 1000"**, but **This tool has no limit on the number**.
@@ -91,7 +99,7 @@ When this occurs, cls3 responds by adding a mechanism that waits a few seconds a
 ## How to use
 
   ```bash
-  cls3 -b <bucketName> [-b <bucketName>] [-p <profile>] [-r <region>] [-f|--force] [-i|--interactive] [-o|--oldVersionsOnly] [-q|--quietMode]
+  cls3 -b <bucketName> [-b <bucketName>] [-p <profile>] [-r <region>] [-f|--force] [-i|--interactive] [-o|--oldVersionsOnly] [-q|--quietMode] [-d|--directoryBucketsMode]
   ```
 
 - -b, --bucketName: optional
@@ -104,7 +112,9 @@ When this occurs, cls3 responds by adding a mechanism that waits a few seconds a
   - AWS profile name
 - -r, --region: optional(default: `ap-northeast-1`)
   - AWS Region
+    - If this option is not specified and your AWS profile is tied to a region, the region is used instead of the default region.
   - It is not necessary to be aware of this as it can be used **across regions**.
+    - But in the **Directory Buckets** Mode for **S3 Express One Zone** (with `-d` option), you should specify the region. The mode is not available across regions.
 - -f, --force: optional
   - ForceMode (Delete the bucket together)
 - -i, --interactive: optional
@@ -115,6 +125,10 @@ When this occurs, cls3 responds by adding a mechanism that waits a few seconds a
 - -q, --quietMode: optional
   - Hide live display of number of deletions
   - It would be good to use in CI
+- -d, --directoryBucketsMode: optional
+  - **Directory Buckets** Mode for **S3 Express One Zone**
+  - Operation across regions is not possible, but only in **one region**.
+    - You can specify the region with the `-r` option.
 
 ## Interactive Mode
 
@@ -143,10 +157,13 @@ Then you select bucket names in the UI.
 
 ## GitHub Actions
 
-You can use cls3 with parameters **"bucket-name", "force", "quiet" and "region"** (actually no need to specify "region"
-as it runs across regions) in GitHub Actions Workflow.
+You can use cls3 in GitHub Actions Workflow.
+
+The `quiet` allows you to hive live display of number of deletions (**default: true in GitHub Actions ONLY**).
 
-The "quiet" allows you to hive live display of number of deletions (**default: true in GitHub Actions ONLY**).
+Basically, you do not need to specify a `region` parameter, since you can delete buckets across regions. However,
+in Directory Buckets mode (`directory-buckets-mode`) for S3 Express One Zone, the region must be specified. This mode cannot
+be used across regions.
 
 To delete multiple buckets, specify bucket names separated by commas.
 
@@ -172,7 +189,9 @@ jobs:
           # bucket-name: YourBucket1, YourBucket2, YourBucket3 # To delete multiple buckets
           force: true # Whether to delete the bucket itself, not just the object (default: false)
           quiet: false # Hide live display of number of deletions (default: true in GitHub Actions ONLY.)
-          region: us-east-1 # Actually, no need to specify as it runs across regions
+          old-versions-only: false # Delete old version objects only (including all delete-markers) (default: false)
+          directory-buckets-mode: false # Directory Buckets Mode for S3 Express One Zone (default: false)
+          region: us-east-1 # Specify the region in the Directory Buckets Mode for S3 Express One Zone
 ```
 
 You can also run raw commands after installing the cls3 binary.

action.yml

@@ -16,6 +16,14 @@ inputs:
     description: "Hide live display of number of deletions (default: true in GitHub Actions ONLY.)"
     default: true
     required: false
+  old-versions-only:
+    description: "Delete old version objects only (including all delete-markers)"
+    default: false
+    required: false
+  directory-buckets-mode:
+    description: "Clear Directory Buckets for S3 Express One Zone"
+    default: false
+    required: false
   region:
     description: "AWS Region"
     default: "us-east-1"
@@ -49,9 +57,17 @@ runs:
           if [ "${{ inputs.quiet }}" = "true" ]; then
             quiet="-q"
           fi
+          old_versions_only=""
+          if [ "${{ inputs.old-versions-only }}" = "true" ]; then
+            old_versions_only="-o"
+          fi
+          directory_buckets_mode=""
+          if [ "${{ inputs.directory-buckets-mode }}" = "true" ]; then
+            directory_buckets_mode="-d"
+          fi
           region=""
           if [ -n "${{ inputs.region }}" ]; then
             region="-r ${{ inputs.region }}"
           fi
-          cls3 $buckets $force $quiet $region
+          cls3 $buckets $force $quiet $old_versions_only $directory_buckets_mode $region
         fi

internal/app/app.go

@@ -16,14 +16,15 @@ import (
 const SDKRetryMaxAttempts = 3
 
 type App struct {
-	Cli             *cli.App
-	BucketNames     *cli.StringSlice
-	Profile         string
-	Region          string
-	ForceMode       bool
-	InteractiveMode bool
-	OldVersionsOnly bool
-	QuietMode       bool
+	Cli                  *cli.App
+	BucketNames          *cli.StringSlice
+	Profile              string
+	Region               string
+	ForceMode            bool
+	InteractiveMode      bool
+	OldVersionsOnly      bool
+	QuietMode            bool
+	DirectoryBucketsMode bool
 }
 
 func NewApp(version string) *App {
@@ -81,6 +82,13 @@ func NewApp(version string) *App {
 				Usage:       "Hide live display of number of deletions",
 				Destination: &app.QuietMode,
 			},
+			&cli.BoolFlag{
+				Name:        "directoryBucketsMode",
+				Aliases:     []string{"d"},
+				Value:       false,
+				Usage:       "Clear Directory Buckets for S3 Express One Zone",
+				Destination: &app.DirectoryBucketsMode,
+			},
 		},
 	}
 
@@ -111,6 +119,13 @@ func (a *App) getAction() func(c *cli.Context) error {
 			errMsg := fmt.Sprintln("When specifying -o, do not specify the -f option.")
 			return fmt.Errorf("InvalidOptionError: %v", errMsg)
 		}
+		if a.DirectoryBucketsMode && a.OldVersionsOnly {
+			errMsg := fmt.Sprintln("When specifying -d, do not specify the -o option.")
+			return fmt.Errorf("InvalidOptionError: %v", errMsg)
+		}
+		if a.DirectoryBucketsMode && a.Region == "" {
+			io.Logger.Warn().Msg("You are in the Directory Buckets Mode `-d` to clear the Directory Buckets. In this mode, operation across regions is not possible, but only in one region. You can specify the region with the `-r` option.")
+		}
 
 		config, err := client.LoadAWSConfig(c.Context, a.Region, a.Profile)
 		if err != nil {
@@ -126,7 +141,7 @@ func (a *App) getAction() func(c *cli.Context) error {
 		s3Wrapper := wrapper.NewS3Wrapper(client)
 
 		if a.InteractiveMode {
-			buckets, continuation, err := a.doInteractiveMode(c.Context, s3Wrapper)
+			buckets, continuation, err := a.doInteractiveMode(c.Context, s3Wrapper, a.DirectoryBucketsMode)
 			if err != nil {
 				return err
 			}
@@ -140,7 +155,7 @@ func (a *App) getAction() func(c *cli.Context) error {
 		}
 
 		for _, bucket := range a.BucketNames.Value() {
-			if err := s3Wrapper.ClearS3Objects(c.Context, bucket, a.ForceMode, a.OldVersionsOnly, a.QuietMode); err != nil {
+			if err := s3Wrapper.ClearS3Objects(c.Context, bucket, a.ForceMode, a.OldVersionsOnly, a.QuietMode, a.DirectoryBucketsMode); err != nil {
 				return err
 			}
 		}
@@ -149,15 +164,15 @@ func (a *App) getAction() func(c *cli.Context) error {
 	}
 }
 
-func (a *App) doInteractiveMode(ctx context.Context, s3Wrapper *wrapper.S3Wrapper) ([]string, bool, error) {
+func (a *App) doInteractiveMode(ctx context.Context, s3Wrapper *wrapper.S3Wrapper, directoryBucketsMode bool) ([]string, bool, error) {
 	var checkboxes []string
 	var keyword string
 
 	BucketNameLabel := "Filter a keyword of bucket names: "
 	keyword = io.InputKeywordForFilter(BucketNameLabel)
 
 	label := "Select buckets." + "\n"
-	bucketNames, err := s3Wrapper.ListBucketNamesFilteredByKeyword(ctx, aws.String(keyword))
+	bucketNames, err := s3Wrapper.ListBucketNamesFilteredByKeyword(ctx, aws.String(keyword), directoryBucketsMode)
 	if err != nil {
 		return checkboxes, false, err
 	}

internal/wrapper/s3_wrapper.go

@@ -30,8 +30,9 @@ func (s *S3Wrapper) ClearS3Objects(
 	forceMode bool,
 	oldVersionsOnly bool,
 	quietMode bool,
+	directoryBucketsMode bool,
 ) error {
-	exists, err := s.client.CheckBucketExists(ctx, aws.String(bucketName))
+	exists, err := s.client.CheckBucketExists(ctx, aws.String(bucketName), directoryBucketsMode)
 	if err != nil {
 		return err
 	}
@@ -40,17 +41,23 @@ func (s *S3Wrapper) ClearS3Objects(
 		return nil
 	}
 
-	region, err := s.client.GetBucketLocation(ctx, aws.String(bucketName))
-	if err != nil {
-		return err
+	// This is so that buckets in other regions than the specified one can also be deleted.
+	// If directoryBucketsMode is true, this variable is unnecessary because only one region's
+	// buckets can be operated on.
+	var bucketRegion string
+	if !directoryBucketsMode {
+		bucketRegion, err = s.client.GetBucketLocation(ctx, aws.String(bucketName))
+		if err != nil {
+			return err
+		}
 	}
 
 	eg := errgroup.Group{}
 	errorStr := ""
 	errorsCount := 0
 	errorsMtx := sync.Mutex{}
-	deletedVersionsCount := 0
-	deletedVersionsCountMtx := sync.Mutex{}
+	deletedObjectsCount := 0
+	deletedObjectsCountMtx := sync.Mutex{}
 
 	var writer *uilive.Writer
 	if !quietMode {
@@ -64,38 +71,48 @@ func (s *S3Wrapper) ClearS3Objects(
 	var keyMarker *string
 	var versionIdMarker *string
 	for {
-		var versions []types.ObjectIdentifier
-
-		// ListObjectVersions API can only retrieve up to 1000 items, so it is good to pass it
-		// directly to DeleteObjects, which can only delete up to 1000 items.
-		versions, keyMarker, versionIdMarker, err = s.client.ListObjectVersionsByPage(
-			ctx,
-			aws.String(bucketName),
-			region,
-			oldVersionsOnly,
-			keyMarker,
-			versionIdMarker,
-		)
-		if err != nil {
-			return err
+		var objects []types.ObjectIdentifier
+
+		if directoryBucketsMode {
+			// ListObjects API can only retrieve up to 1000 items, so it is good to pass it
+			// directly to DeleteObjects, which can only delete up to 1000 items.
+			objects, keyMarker, err = s.client.ListObjectsByPage(ctx, aws.String(bucketName), bucketRegion, keyMarker)
+			if err != nil {
+				return err
+			}
+		} else {
+			// ListObjectVersions API can only retrieve up to 1000 items, so it is good to pass it
+			// directly to DeleteObjects, which can only delete up to 1000 items.
+			objects, keyMarker, versionIdMarker, err = s.client.ListObjectVersionsByPage(
+				ctx,
+				aws.String(bucketName),
+				bucketRegion,
+				oldVersionsOnly,
+				keyMarker,
+				versionIdMarker,
+			)
+			if err != nil {
+				return err
+			}
 		}
-		if len(versions) == 0 {
+
+		if len(objects) == 0 {
 			break
 		}
 
 		eg.Go(func() error {
-			deletedVersionsCountMtx.Lock()
-			deletedVersionsCount += len(versions)
+			deletedObjectsCountMtx.Lock()
+			deletedObjectsCount += len(objects)
 			if !quietMode {
-				fmt.Fprintf(writer, "Clearing... %d objects\n", deletedVersionsCount)
+				fmt.Fprintf(writer, "Clearing... %d objects\n", deletedObjectsCount)
 			}
-			deletedVersionsCountMtx.Unlock()
+			deletedObjectsCountMtx.Unlock()
 
 			// One DeleteObjects is executed for each loop of the List, and it usually ends during
 			// the next loop. Therefore, there seems to be no throttling concern, so the number of
 			// parallels is not limited by semaphore. (Throttling occurs at about 3500 deletions
 			// per second.)
-			gotErrors, err := s.client.DeleteObjects(ctx, aws.String(bucketName), versions, region)
+			gotErrors, err := s.client.DeleteObjects(ctx, aws.String(bucketName), objects, bucketRegion)
 			if err != nil {
 				return err
 			}
@@ -139,14 +156,14 @@ func (s *S3Wrapper) ClearS3Objects(
 		}
 	}
 
-	if deletedVersionsCount == 0 {
+	if deletedObjectsCount == 0 {
 		io.Logger.Info().Msgf("%v No objects.", bucketName)
 	} else {
-		io.Logger.Info().Msgf("%v Cleared!!: %v objects.", bucketName, deletedVersionsCount)
+		io.Logger.Info().Msgf("%v Cleared!!: %v objects.", bucketName, deletedObjectsCount)
 	}
 
 	if forceMode {
-		if err := s.client.DeleteBucket(ctx, aws.String(bucketName), region); err != nil {
+		if err := s.client.DeleteBucket(ctx, aws.String(bucketName), bucketRegion); err != nil {
 			return err
 		}
 		io.Logger.Info().Msgf("%v Deleted!!", bucketName)
@@ -155,10 +172,17 @@ func (s *S3Wrapper) ClearS3Objects(
 	return nil
 }
 
-func (s *S3Wrapper) ListBucketNamesFilteredByKeyword(ctx context.Context, keyword *string) ([]string, error) {
+func (s *S3Wrapper) ListBucketNamesFilteredByKeyword(ctx context.Context, keyword *string, directoryBucketsMode bool) ([]string, error) {
 	filteredBucketNames := []string{}
 
-	buckets, err := s.client.ListBuckets(ctx)
+	var listBucketsFunc func(ctx context.Context) ([]types.Bucket, error)
+	if directoryBucketsMode {
+		listBucketsFunc = s.client.ListDirectoryBuckets
+	} else {
+		listBucketsFunc = s.client.ListBuckets
+	}
+
+	buckets, err := listBucketsFunc(ctx)
 	if err != nil {
 		return filteredBucketNames, err
 	}