Merge pull request #3402 from nalind/platforms

bud: teach --platform to take a list

buildah.go

@@ -397,7 +397,7 @@ func OpenBuilder(store storage.Store, container string) (*Builder, error) {
 		return nil, errors.Errorf("container %q is not a %s container (is a %q container)", container, define.Package, b.Type)
 	}
 	b.store = store
-	b.fixupConfig()
+	b.fixupConfig(nil)
 	b.setupLogger()
 	return b, nil
 }
@@ -433,7 +433,7 @@ func OpenBuilderByPath(store storage.Store, path string) (*Builder, error) {
 		err = json.Unmarshal(buildstate, &b)
 		if err == nil && b.Type == containerType && builderMatchesPath(b, abs) {
 			b.store = store
-			b.fixupConfig()
+			b.fixupConfig(nil)
 			b.setupLogger()
 			return b, nil
 		}
@@ -471,7 +471,7 @@ func OpenAllBuilders(store storage.Store) (builders []*Builder, err error) {
 		if err == nil && b.Type == containerType {
 			b.store = store
 			b.setupLogger()
-			b.fixupConfig()
+			b.fixupConfig(nil)
 			builders = append(builders, b)
 			continue
 		}

cmd/buildah/bud.go

@@ -13,10 +13,16 @@ import (
 	buildahcli "github.com/containers/buildah/pkg/cli"
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/util"
+	"github.com/containers/common/libimage"
+	"github.com/containers/common/libimage/manifests"
 	"github.com/containers/common/pkg/auth"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/storage"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/semaphore"
 )
 
 type budOptions struct {
@@ -299,7 +305,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
 	}
 	namespaceOptions.AddOrReplace(usernsOption...)
 
-	imageOS, arch, err := parse.PlatformFromOptions(c)
+	platforms, err := parse.PlatformsFromOptions(c)
 	if err != nil {
 		return err
 	}
@@ -309,75 +315,119 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
 		return errors.Wrapf(err, "unable to obtain decrypt config")
 	}
 
-	options := define.BuildOptions{
-		AddCapabilities:         iopts.CapAdd,
-		AdditionalTags:          tags,
-		Annotations:             iopts.Annotation,
-		Architecture:            arch,
-		Args:                    args,
-		BlobDirectory:           iopts.BlobCache,
-		CNIConfigDir:            iopts.CNIConfigDir,
-		CNIPluginPath:           iopts.CNIPlugInPath,
-		CommonBuildOpts:         commonOpts,
-		Compression:             compression,
-		ConfigureNetwork:        networkPolicy,
-		ContextDirectory:        contextDir,
-		DefaultMountsFilePath:   globalFlagResults.DefaultMountsFile,
-		Devices:                 iopts.Devices,
-		DropCapabilities:        iopts.CapDrop,
-		Err:                     stderr,
-		ForceRmIntermediateCtrs: iopts.ForceRm,
-		From:                    iopts.From,
-		IDMappingOptions:        idmappingOptions,
-		IIDFile:                 iopts.Iidfile,
-		In:                      stdin,
-		Isolation:               isolation,
-		Labels:                  iopts.Label,
-		Layers:                  layers,
-		LogRusage:               iopts.LogRusage,
-		Manifest:                iopts.Manifest,
-		MaxPullPushRetries:      maxPullPushRetries,
-		NamespaceOptions:        namespaceOptions,
-		NoCache:                 iopts.NoCache,
-		OS:                      imageOS,
-		Out:                     stdout,
-		Output:                  output,
-		OutputFormat:            format,
-		PullPolicy:              pullPolicy,
-		PullPushRetryDelay:      pullPushRetryDelay,
-		Quiet:                   iopts.Quiet,
-		RemoveIntermediateCtrs:  iopts.Rm,
-		ReportWriter:            reporter,
-		Runtime:                 iopts.Runtime,
-		RuntimeArgs:             runtimeFlags,
-		RusageLogFile:           iopts.RusageLogFile,
-		SignBy:                  iopts.SignBy,
-		SignaturePolicyPath:     iopts.SignaturePolicy,
-		Squash:                  iopts.Squash,
-		SystemContext:           systemContext,
-		Target:                  iopts.Target,
-		TransientMounts:         iopts.Volumes,
-		OciDecryptConfig:        decConfig,
-		Jobs:                    &iopts.Jobs,
+	var (
+		jobSemaphore *semaphore.Weighted
+		builds       multierror.Group
+	)
+	if iopts.Jobs > 0 {
+		jobSemaphore = semaphore.NewWeighted(int64(iopts.Jobs))
+		iopts.Jobs = 0
+	}
+	if iopts.Manifest != "" {
+		// Ensure that the list's ID is known before we spawn off any
+		// goroutines that'll want to modify it, so that they don't
+		// race and create two lists, one of which will rapidly become
+		// ignored.
+		rt, err := libimage.RuntimeFromStore(store, nil)
+		if err != nil {
+			return err
+		}
+		_, err = rt.LookupManifestList(iopts.Manifest)
+		if err != nil && errors.Cause(err) == storage.ErrImageUnknown {
+			list := manifests.Create()
+			_, err = list.SaveToImage(store, "", []string{iopts.Manifest}, manifest.DockerV2ListMediaType)
+		}
+		if err != nil {
+			return err
+		}
 	}
+	var excludes []string
 	if iopts.IgnoreFile != "" {
-		excludes, err := parseDockerignore(iopts.IgnoreFile)
-		if err != nil {
+		if excludes, err = parseDockerignore(iopts.IgnoreFile); err != nil {
 			return err
 		}
-		options.Excludes = excludes
 	}
+	var timestamp *time.Time
 	if c.Flag("timestamp").Changed {
-		timestamp := time.Unix(iopts.Timestamp, 0).UTC()
-		options.Timestamp = &timestamp
-	}
+		t := time.Unix(iopts.Timestamp, 0).UTC()
+		timestamp = &t
+	}
+	for _, platform := range platforms {
+		platform := platform
+		builds.Go(func() error {
+			platformContext := *systemContext
+			platformContext.OSChoice = platform.OS
+			platformContext.ArchitectureChoice = platform.Arch
+			platformContext.VariantChoice = platform.Variant
+			options := define.BuildOptions{
+				AddCapabilities:         iopts.CapAdd,
+				AdditionalTags:          tags,
+				Annotations:             iopts.Annotation,
+				Architecture:            platform.Arch,
+				Args:                    args,
+				BlobDirectory:           iopts.BlobCache,
+				CNIConfigDir:            iopts.CNIConfigDir,
+				CNIPluginPath:           iopts.CNIPlugInPath,
+				CommonBuildOpts:         commonOpts,
+				Compression:             compression,
+				ConfigureNetwork:        networkPolicy,
+				ContextDirectory:        contextDir,
+				DefaultMountsFilePath:   globalFlagResults.DefaultMountsFile,
+				Devices:                 iopts.Devices,
+				DropCapabilities:        iopts.CapDrop,
+				Err:                     stderr,
+				ForceRmIntermediateCtrs: iopts.ForceRm,
+				From:                    iopts.From,
+				IDMappingOptions:        idmappingOptions,
+				IIDFile:                 iopts.Iidfile,
+				In:                      stdin,
+				Isolation:               isolation,
+				Labels:                  iopts.Label,
+				Layers:                  layers,
+				LogRusage:               iopts.LogRusage,
+				Manifest:                iopts.Manifest,
+				MaxPullPushRetries:      maxPullPushRetries,
+				NamespaceOptions:        namespaceOptions,
+				NoCache:                 iopts.NoCache,
+				OS:                      platform.OS,
+				Out:                     stdout,
+				Output:                  output,
+				OutputFormat:            format,
+				PullPolicy:              pullPolicy,
+				PullPushRetryDelay:      pullPushRetryDelay,
+				Quiet:                   iopts.Quiet,
+				RemoveIntermediateCtrs:  iopts.Rm,
+				ReportWriter:            reporter,
+				Runtime:                 iopts.Runtime,
+				RuntimeArgs:             runtimeFlags,
+				RusageLogFile:           iopts.RusageLogFile,
+				SignBy:                  iopts.SignBy,
+				SignaturePolicyPath:     iopts.SignaturePolicy,
+				Squash:                  iopts.Squash,
+				SystemContext:           &platformContext,
+				Target:                  iopts.Target,
+				TransientMounts:         iopts.Volumes,
+				OciDecryptConfig:        decConfig,
+				Jobs:                    &iopts.Jobs,
+				JobSemaphore:            jobSemaphore,
+				Excludes:                excludes,
+				Timestamp:               timestamp,
+			}
+			if iopts.Quiet {
+				options.ReportWriter = ioutil.Discard
+			}
 
-	if iopts.Quiet {
-		options.ReportWriter = ioutil.Discard
+			_, _, err = imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...)
+			return err
+		})
 	}
-
-	_, _, err = imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...)
-	return err
+	if merr := builds.Wait(); merr != nil {
+		if merr.Len() == 1 {
+			return merr.Errors[0]
+		}
+		return merr.ErrorOrNil()
+	}
+	return nil
 }
 
 // discoverContainerfile tries to find a Containerfile or a Dockerfile within the provided `path`.

cmd/buildah/from.go

@@ -195,6 +195,13 @@ func fromCmd(c *cobra.Command, args []string, iopts fromReply) error {
 	if err != nil {
 		return errors.Wrapf(err, "error building system context")
 	}
+	platforms, err := parse.PlatformsFromOptions(c)
+	if err != nil {
+		return err
+	}
+	if len(platforms) > 1 {
+		logrus.Warnf("ignoring platforms other than %+v: %+v", platforms[0], platforms[1:])
+	}
 
 	pullFlagsCount := 0
 	if c.Flag("pull").Changed {

cmd/buildah/pull.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/common/pkg/auth"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -67,6 +68,7 @@ func init() {
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "don't output progress information when pulling images")
 	flags.String("os", runtime.GOOS, "prefer `OS` instead of the running OS for choosing images")
 	flags.String("arch", runtime.GOARCH, "prefer `ARCH` instead of the architecture of the machine for choosing images")
+	flags.StringSlice("platform", []string{parse.DefaultPlatform()}, "prefer OS/ARCH instead of the current operating system and architecture for choosing images")
 	flags.String("variant", "", "override the `variant` of the specified image")
 	flags.BoolVar(&opts.tlsVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry. TLS verification cannot be used when talking to an insecure registry.")
 	if err := flags.MarkHidden("blob-cache"); err != nil {
@@ -94,6 +96,13 @@ func pullCmd(c *cobra.Command, args []string, iopts pullOptions) error {
 	if err != nil {
 		return errors.Wrapf(err, "error building system context")
 	}
+	platforms, err := parse.PlatformsFromOptions(c)
+	if err != nil {
+		return err
+	}
+	if len(platforms) > 1 {
+		logrus.Warnf("ignoring platforms other than %+v: %+v", platforms[0], platforms[1:])
+	}
 
 	store, err := getStore(c)
 	if err != nil {

cmd/buildah/unshare.go

@@ -33,7 +33,7 @@ func init() {
 	unshareCommand.SetUsageTemplate(UsageTemplate())
 	flags := unshareCommand.Flags()
 	flags.SetInterspersed(false)
-	flags.StringSliceVar(&unshareMounts, "mount", []string{}, "mount the specified containers (default [])")
+	flags.StringSliceVarP(&unshareMounts, "mount", "m", []string{}, "mount the specified containers (default [])")
 	rootCmd.AddCommand(unshareCommand)
 }
 

cmd/buildah/version.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/containerd/containerd/platforms"
 	cniversion "github.com/containernetworking/cni/pkg/version"
 	"github.com/containers/buildah/define"
 	iversion "github.com/containers/image/v5/version"
@@ -33,6 +34,7 @@ type versionInfo struct {
 	GitCommit     string `json:"gitCommit"`
 	Built         string `json:"built"`
 	OsArch        string `json:"osArch"`
+	BuildPlatform string `json:"buildPlatform"`
 }
 
 type versionOptions struct {
@@ -83,6 +85,7 @@ func versionCmd(opts versionOptions) error {
 		GitCommit:     GitCommit,
 		Built:         time.Unix(buildTime, 0).Format(time.ANSIC),
 		OsArch:        runtime.GOOS + "/" + runtime.GOARCH,
+		BuildPlatform: platforms.DefaultString(),
 	}
 
 	if opts.json {
@@ -106,6 +109,7 @@ func versionCmd(opts versionOptions) error {
 	//Prints out the build time in readable format
 	fmt.Println("Built:          ", version.Built)
 	fmt.Println("OS/Arch:        ", version.OsArch)
+	fmt.Println("BuildPlatform:  ", version.BuildPlatform)
 
 	return nil
 }

commit.go

@@ -183,6 +183,12 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe
 		create = true
 		list = manifests.Create()
 	} else {
+		locker, err := manifests.LockerForImage(b.store, manifestList.ID())
+		if err != nil {
+			return "", err
+		}
+		locker.Lock()
+		defer locker.Unlock()
 		_, list, err = manifests.LoadFromImage(b.store, manifestList.ID())
 		if err != nil {
 			return "", err

config.go

@@ -51,7 +51,7 @@ func unmarshalConvertedConfig(ctx context.Context, dest interface{}, img types.I
 	return nil
 }
 
-func (b *Builder) initConfig(ctx context.Context, img types.Image) error {
+func (b *Builder) initConfig(ctx context.Context, img types.Image, sys *types.SystemContext) error {
 	if img != nil { // A pre-existing image, as opposed to a "FROM scratch" new one.
 		rawManifest, manifestMIMEType, err := img.Manifest(ctx)
 		if err != nil {
@@ -95,11 +95,11 @@ func (b *Builder) initConfig(ctx context.Context, img types.Image) error {
 	}
 
 	b.setupLogger()
-	b.fixupConfig()
+	b.fixupConfig(sys)
 	return nil
 }
 
-func (b *Builder) fixupConfig() {
+func (b *Builder) fixupConfig(sys *types.SystemContext) {
 	if b.Docker.Config != nil {
 		// Prefer image-level settings over those from the container it was built from.
 		b.Docker.ContainerConfig = *b.Docker.Config
@@ -114,10 +114,18 @@ func (b *Builder) fixupConfig() {
 		b.OCIv1.Created = &now
 	}
 	if b.OS() == "" {
-		b.SetOS(runtime.GOOS)
+		if sys != nil && sys.OSChoice != "" {
+			b.SetOS(sys.OSChoice)
+		} else {
+			b.SetOS(runtime.GOOS)
+		}
 	}
 	if b.Architecture() == "" {
-		b.SetArchitecture(runtime.GOARCH)
+		if sys != nil && sys.ArchitectureChoice != "" {
+			b.SetArchitecture(sys.ArchitectureChoice)
+		} else {
+			b.SetArchitecture(runtime.GOARCH)
+		}
 	}
 	if b.Format == define.Dockerv2ImageManifest && b.Hostname() == "" {
 		b.SetHostname(stringid.TruncateID(stringid.GenerateRandomID()))

contrib/completions/bash/buildah

@@ -434,6 +434,7 @@ return 1
      --os
      --pid
      --platform
+     --platforms
      --runtime
      --runtime-flag
      --security-opt