Support to Docker Registry V2 API

Makefile

@@ -4,7 +4,7 @@ export GO111MODULE := on
 
 .PHONY: test binary install clean
 
-cmd/ecspresso/ecspresso: *.go cmd/ecspresso/*.go go.* appspec/*.go
+cmd/ecspresso/ecspresso: *.go cmd/ecspresso/*.go go.* */*.go
 	cd cmd/ecspresso && go build -ldflags "-s -w -X main.Version=${GIT_VER} -X main.buildDate=${DATE}" -gcflags="-trimpath=${PWD}"
 
 install: cmd/ecspresso/ecspresso

go.sum

@@ -112,6 +112,7 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

registry/client.go

@@ -0,0 +1,143 @@
+package registry
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+const dockerHubHost = "registry.hub.docker.com"
+
+// Repositry represents a repositry using Docker Registry API v2.
+type Repositry struct {
+	client   *http.Client
+	host     string
+	repo     string
+	user     string
+	password string
+	token    string
+}
+
+// New creates a client for a repositry.
+func New(image, user, password string) *Repositry {
+	c := &Repositry{
+		client:   &http.Client{},
+		user:     user,
+		password: password,
+	}
+	p := strings.SplitN(image, "/", 2)
+	if strings.Contains(p[0], ".") && len(p) >= 2 {
+		// Docker registry v2 API
+		c.host = p[0]
+		c.repo = p[1]
+	} else {
+		// DockerHub
+		if !strings.Contains(image, "/") {
+			image = "library/" + image
+		}
+		c.host = dockerHubHost
+		c.repo = image
+	}
+	return c
+}
+
+func (c *Repositry) login(endpoint, service, scope string) error {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return err
+	}
+	u.RawQuery = strings.Join([]string{
+		"service=" + url.QueryEscape(service),
+		"scope=" + url.QueryEscape(scope),
+	}, "&")
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return err
+	}
+	if c.user != "" && c.password != "" {
+		req.SetBasicAuth(c.user, c.password)
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("login failed %s", resp.Status)
+	}
+	dec := json.NewDecoder(resp.Body)
+	var body struct {
+		Token string `json:"Token"`
+	}
+	if err := dec.Decode(&body); err != nil {
+		return err
+	}
+	if body.Token == "" {
+		return errors.New("response does not contains token")
+	}
+	c.token = body.Token
+	return nil
+}
+
+func (c *Repositry) getManifests(tag string) (*http.Response, error) {
+	u := fmt.Sprintf("https://%s/v2/%s/manifests/%s", c.host, c.repo, tag)
+	req, _ := http.NewRequest(http.MethodHead, u, nil)
+	req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	} else if c.user == "AWS" && c.password != "" {
+		// ECR
+		req.Header.Set("Authorization", "Basic "+c.password)
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return resp, err
+}
+
+// HasImage returns an image tag exists or not in the repository.
+func (c *Repositry) HasImage(tag string) (bool, error) {
+	tries := 2
+	for tries > 0 {
+		tries--
+		resp, err := c.getManifests(tag)
+		if err != nil {
+			return false, err
+		}
+		switch resp.StatusCode {
+		case http.StatusUnauthorized:
+			h := resp.Header.Get("Www-Authenticate")
+			if strings.HasPrefix(h, "Bearer ") {
+				auth := strings.SplitN(h, " ", 2)[1]
+				if err := c.login(parseAuthHeader(auth)); err != nil {
+					return false, err
+				}
+			}
+		case http.StatusOK:
+			return true, nil
+		default:
+			return false, errors.New(resp.Status)
+		}
+	}
+	return false, errors.New("aborted")
+}
+
+var (
+	partRegexp = regexp.MustCompile(`[a-zA-Z0-9_]+="[^"]*"`)
+)
+
+func parseAuthHeader(bearer string) (endpoint, service, scope string) {
+	parsed := make(map[string]string, 3)
+	for _, part := range partRegexp.FindAllString(bearer, -1) {
+		kv := strings.SplitN(part, "=", 2)
+		parsed[kv[0]] = kv[1][1 : len(kv[1])-1]
+	}
+	return parsed["realm"], parsed["service"], parsed["scope"]
+}

registry/client_test.go

@@ -0,0 +1,30 @@
+package registry_test
+
+import (
+	"testing"
+
+	"github.com/kayac/ecspresso/registry"
+)
+
+var testImages = []struct {
+	image string
+	tag   string
+}{
+	{image: "debian", tag: "latest"},
+	{image: "katsubushi/katsubushi", tag: "v1.6.0"},
+	{image: "public.ecr.aws/mackerel/mackerel-container-agent", tag: "plugins"},
+	{image: "gcr.io/kaniko-project/executor", tag: "v0.10.0"},
+	{image: "ghcr.io/github/super-linter", tag: "v3"},
+}
+
+func TestImages(t *testing.T) {
+	for _, c := range testImages {
+		t.Logf("testing %s:%s", c.image, c.tag)
+		client := registry.New(c.image, "", "")
+		if ok, err := client.HasImage(c.tag); err != nil {
+			t.Errorf("%s:%s error %s", c.image, c.tag, err)
+		} else if !ok {
+			t.Errorf("%s:%s not found", c.image, c.tag)
+		}
+	}
+}

verify.go

@@ -22,7 +22,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/fatih/color"
-	"github.com/kayac/ecspresso/dockerhub"
+	"github.com/kayac/ecspresso/registry"
 	"github.com/pkg/errors"
 )
 
@@ -281,57 +281,42 @@ func (d *App) verifyTaskDefinition(ctx context.Context) error {
 }
 
 var (
-	ecrImageURLRegex    = regexp.MustCompile(`dkr\.ecr\..+.amazonaws\.com/.*:.*`)
-	dockerHubImageRegex = regexp.MustCompile(`([a-zA-Z0-9_-]+/)?[a-zA-Z0-9_-]+:.*`)
+	ecrImageURLRegex = regexp.MustCompile(`dkr\.ecr\..+.amazonaws\.com/.*:.*`)
 )
 
 func (d *App) verifyECRImage(ctx context.Context, image string) error {
-	rr := strings.Split(strings.SplitN(image, "/", 2)[1], ":")
-	repo, tag := rr[0], rr[1]
-	var nextToken *string
-	for {
-		out, err := d.verifier.ecr.ListImagesWithContext(ctx, &ecr.ListImagesInput{
-			RepositoryName: aws.String(repo),
-			Filter:         &ecr.ListImagesFilter{TagStatus: aws.String("TAGGED")},
-			NextToken:      nextToken,
-		})
-		if err != nil {
-			return err
-		}
-		nextToken = out.NextToken
-		for _, img := range out.ImageIds {
-			d.DebugLog(*img.ImageTag)
-			if aws.StringValue(img.ImageTag) == tag {
-				return nil
-			}
-		}
-		if nextToken == nil {
-			break
-		}
+	d.DebugLog("VERIFY ECR Image")
+	out, err := d.verifier.ecr.GetAuthorizationTokenWithContext(
+		ctx,
+		&ecr.GetAuthorizationTokenInput{},
+	)
+	if err != nil {
+		return err
 	}
-	return errors.Errorf("%s:%s not found", repo, tag)
+	token := out.AuthorizationData[0].AuthorizationToken
+	return d.verifyRegistryImage(ctx, image, "AWS", aws.StringValue(token))
 }
 
-func (d *App) verifyDockerHubImage(ctx context.Context, image string) error {
-	rr := strings.Split(image, ":")
-	repoName, tag := rr[0], rr[1]
-	if !strings.Contains(repoName, "/") {
-		repoName = "library/" + repoName
+func (d *App) verifyRegistryImage(ctx context.Context, image, user, password string) error {
+	rr := strings.SplitN(image, ":", 2)
+	image = rr[0]
+	var tag string
+	if len(rr) == 1 {
+		tag = "latest"
+	} else {
+		tag = rr[1]
 	}
-	d.DebugLog(fmt.Sprintf("dockerhub repo=%s tag=%s", repoName, tag))
+	d.DebugLog(fmt.Sprintf("image=%s tag=%s", image, tag))
 
-	repo, err := dockerhub.New(repoName)
-	if err != nil {
-		return err
-	}
+	repo := registry.New(image, user, password)
 	ok, err := repo.HasImage(tag)
 	if err != nil {
 		return err
 	}
 	if ok {
 		return nil
 	}
-	return errors.Errorf("%s:%s is not found in DockerHub", repoName, tag)
+	return errors.Errorf("%s:%s is not found in Registry", image, tag)
 }
 
 func (d *App) verifyImage(ctx context.Context, image string) error {
@@ -340,10 +325,8 @@ func (d *App) verifyImage(ctx context.Context, image string) error {
 	}
 	if ecrImageURLRegex.MatchString(image) {
 		return d.verifyECRImage(ctx, image)
-	} else if dockerHubImageRegex.MatchString(image) {
-		return d.verifyDockerHubImage(ctx, image)
 	}
-	return verifySkipErr("not supported URL (patches are welcome!)")
+	return d.verifyRegistryImage(ctx, image, "", "")
 }
 
 func (d *App) verifyContainer(ctx context.Context, c *ecs.ContainerDefinition) error {