test(storage): add several-msg + resumable uploads to objects.insert (#…

…9636)

* test(storage): add several-msg + resumable uploads to objects.insert

Not all error code paths in grpc uploads are tested without adding a larger object to the storage.objects.insert test.

Additionally, reduces the 9mb object to 3mb where possible to reduce test time.

* expand comment

storage/retry_conformance_test.go

@@ -34,16 +34,27 @@ import (
 	"google.golang.org/api/iterator"
 )
 
-var (
-	// Resource vars for retry tests
-	bucketIDs           = uid.NewSpace("bucket", nil)
-	objectIDs           = uid.NewSpace("object", nil)
-	notificationIDs     = uid.NewSpace("notification", nil)
+const (
 	projectID           = "my-project-id"
 	serviceAccountEmail = "my-sevice-account@my-project-id.iam.gserviceaccount.com"
-	randomBytesToWrite  = []byte("abcdef")
-	randomBytes9MB      = generateRandomBytes(size9MB)
-	size9MB             = 9437184 // 9 MiB
+	MiB                 = 1 << 10 << 10
+)
+
+var (
+	// Resource vars for retry tests
+	bucketIDs       = uid.NewSpace("bucket", nil)
+	objectIDs       = uid.NewSpace("object", nil)
+	notificationIDs = uid.NewSpace("notification", nil)
+
+	size9MiB           = 9 * MiB
+	randomBytesToWrite = []byte("abcdef")
+	// A 3 MiB object is large enough to span several messages and trigger a
+	// resumable upload in the Go library (with chunksize set to 2MiB). We use
+	// this in tests that require a larger object that can be less than 9MiB.
+	randomBytes3MiB = generateRandomBytes(3 * MiB)
+	// A 9 MiB object is required for "storage.resumable.upload"-specific tests,
+	// because there is a test that test errors after the first 8MiB.
+	randomBytes9MiB = generateRandomBytes(size9MiB)
 )
 
 type retryFunc func(ctx context.Context, c *Client, fs *resources, preconditions bool) error
@@ -231,10 +242,10 @@ var methods = map[string][]retryFunc{
 	},
 	"storage.objects.download": {
 		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
-			// Before running the test method, populate a large test object of 9 MiB.
+			// Before running the test method, populate a large test object.
 			objName := objectIDs.New()
-			if err := uploadTestObject(fs.bucket.Name, objName, randomBytes9MB); err != nil {
-				return fmt.Errorf("failed to create 9 MiB large object pre test, err: %v", err)
+			if err := uploadTestObject(fs.bucket.Name, objName, randomBytes3MiB); err != nil {
+				return fmt.Errorf("failed to create large object pre test, err: %v", err)
 			}
 			// Download the large test object for the S8 download method group.
 			r, err := c.Bucket(fs.bucket.Name).Object(objName).NewReader(ctx)
@@ -246,20 +257,20 @@ var methods = map[string][]retryFunc{
 			if err != nil {
 				return fmt.Errorf("failed to ReadAll, err: %v", err)
 			}
-			if got, want := len(data), size9MB; got != want {
+			if got, want := len(data), 3*MiB; got != want {
 				return fmt.Errorf("body length mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
 			}
-			if got, want := data, randomBytes9MB; !bytes.Equal(got, want) {
+			if got, want := data, randomBytes3MiB; !bytes.Equal(got, want) {
 				return fmt.Errorf("body mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
 			}
 			return nil
 		},
 		func(ctx context.Context, c *Client, fs *resources, _ bool) error {
 			// Test JSON reads.
-			// Before running the test method, populate a large test object of 9 MiB.
+			// Before running the test method, populate a large test object.
 			objName := objectIDs.New()
-			if err := uploadTestObject(fs.bucket.Name, objName, randomBytes9MB); err != nil {
-				return fmt.Errorf("failed to create 9 MiB large object pre test, err: %v", err)
+			if err := uploadTestObject(fs.bucket.Name, objName, randomBytes3MiB); err != nil {
+				return fmt.Errorf("failed to create large object pre test, err: %v", err)
 			}
 
 			client, ok := c.tc.(*httpStorageClient)
@@ -282,10 +293,10 @@ var methods = map[string][]retryFunc{
 			if err != nil {
 				return fmt.Errorf("failed to ReadAll, err: %v", err)
 			}
-			if got, want := len(data), size9MB; got != want {
+			if got, want := len(data), 3*MiB; got != want {
 				return fmt.Errorf("body length mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
 			}
-			if got, want := data, randomBytes9MB; !bytes.Equal(got, want) {
+			if got, want := data, randomBytes3MiB; !bytes.Equal(got, want) {
 				return fmt.Errorf("body mismatch\ngot:\n%v\n\nwant:\n%v", got, want)
 			}
 			return nil
@@ -376,6 +387,7 @@ var methods = map[string][]retryFunc{
 		},
 	},
 	"storage.objects.insert": {
+		// Single-shot upload that is sent in a single message.
 		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
 			obj := c.Bucket(fs.bucket.Name).Object("new-object.txt")
 
@@ -392,6 +404,44 @@ var methods = map[string][]retryFunc{
 			}
 			return nil
 		},
+		// Single-shot upload that spans several GRPC message sends (we send
+		// storagepb.ServiceConstants_MAX_WRITE_CHUNK_BYTES=2MiB per msg).
+		// This is needed to cover all error code paths.
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			obj := c.Bucket(fs.bucket.Name).Object("new-object.txt")
+
+			if preconditions {
+				obj = obj.If(Conditions{DoesNotExist: true})
+			}
+
+			objW := obj.NewWriter(ctx)
+			if _, err := objW.Write(randomBytes3MiB); err != nil {
+				return fmt.Errorf("io.Copy: %v", err)
+			}
+			if err := objW.Close(); err != nil {
+				return fmt.Errorf("Writer.Close: %v", err)
+			}
+			return nil
+		},
+		// Resumable upload.
+		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+			obj := c.Bucket(fs.bucket.Name).Object("new-object.txt")
+
+			if preconditions {
+				obj = obj.If(Conditions{DoesNotExist: true})
+			}
+
+			objW := obj.NewWriter(ctx)
+			objW.ChunkSize = 2 * MiB
+
+			if _, err := objW.Write(randomBytes3MiB); err != nil {
+				return fmt.Errorf("io.Copy: %v", err)
+			}
+			if err := objW.Close(); err != nil {
+				return fmt.Errorf("Writer.Close: %v", err)
+			}
+			return nil
+		},
 	},
 	"storage.resumable.upload": {
 		func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
@@ -403,7 +453,7 @@ var methods = map[string][]retryFunc{
 			// Set Writer.ChunkSize to 2 MiB to perform resumable uploads.
 			w.ChunkSize = 2097152
 
-			if _, err := w.Write(randomBytes9MB); err != nil {
+			if _, err := w.Write(randomBytes9MiB); err != nil {
 				return fmt.Errorf("writing object: %v", err)
 			}
 			if err := w.Close(); err != nil {