From b8b86d6dc0b07d3384d4b164f63bb46e92d13594 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 13 Aug 2019 11:52:22 +0200 Subject: [PATCH 1/5] chore(ci/cd): build containers using drone.io Because of the native arm capabilities, this now builds the containers using drone.io as well. --- .circleci/config.yml | 2 +- .drone/docker-manifest.tmpl | 22 +++++ .drone/drone.jsonnet | 74 +++++++++++++++++ .drone/drone.yml | 134 +++++++++++++++++++++++++++++++ Makefile | 16 +++- cmd/loki-canary/Dockerfile | 13 +-- cmd/loki-canary/Dockerfile.cross | 18 +++++ cmd/loki/Dockerfile | 13 +-- cmd/loki/Dockerfile.cross | 21 +++++ cmd/promtail/Dockerfile | 21 ++--- cmd/promtail/Dockerfile.cross | 26 ++++++ 11 files changed, 319 insertions(+), 41 deletions(-) create mode 100644 .drone/docker-manifest.tmpl create mode 100644 .drone/drone.jsonnet create mode 100644 .drone/drone.yml create mode 100644 cmd/loki-canary/Dockerfile.cross create mode 100644 cmd/loki/Dockerfile.cross create mode 100644 cmd/promtail/Dockerfile.cross diff --git a/.circleci/config.yml b/.circleci/config.yml index e0882e5ca3dd..bd623920c2bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ workflows: - run: name: container command: | - make $APP-image + make $APP-image-cross # builds and pushes a container .publish: &publish diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl new file mode 100644 index 000000000000..b940a695224c --- /dev/null +++ b/.drone/docker-manifest.tmpl @@ -0,0 +1,22 @@ +image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64 + platform: + architecture: amd64 + os: linux + - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet new file mode 100644 index 000000000000..ef4f140f7d12 --- /dev/null +++ b/.drone/drone.jsonnet @@ -0,0 +1,74 @@ +local pipeline(name) = { + kind: 'pipeline', + name: name, + steps: [], +}; + +local docker(arch, app) = { + name: '%s-image' % app, + image: 'plugins/docker', + settings: { + repo: 'shorez/%s' % app, + dockerfile: 'cmd/%s/Dockerfile' % app, + // auto_tag: true, + // auto_tag_suffix: arch, + username: { from_secret: 'docker_username' }, + password: { from_secret: 'docker_password' }, + }, +}; + +local multiarch_image(arch) = pipeline('docker-' + arch) { + platform: { + os: 'linux', + arch: arch, + }, + steps: [{ + name: 'image-tag', + image: 'alpine', + commands: [ + 'apk add --no-cache bash git', + 'echo $(./tools/image-tag)-%s > .tags' % arch, + ], + }] + [ + docker(arch, app) { depends_on: ['image-tag'] } + for app in ['promtail', 'loki', 'loki-canary'] + ], +}; + +local manifest(apps) = pipeline('manifest') { + steps: [ + { + name: 'manifest-' + app, + image: 'plugins/manifest', + settings: { + // the target parameter is abused for the app's name, + // as it is unused in spec mode. See docker-manifest.tmpl + target: app, + spec: '.drone/docker-manifest.tmpl', + auto_tag: true, + ignore_missing: true, + username: { from_secret: 'docker_username' }, + password: { from_secret: 'docker_password' }, + }, + } + for app in apps + ], +} + { + depends_on: [ + 'docker-amd64', + 'docker-arm', + 'docker-arm64', + ], +}; + +local drone = [ + multiarch_image('amd64'), + multiarch_image('arm64'), + multiarch_image('arm'), + + // manifest(['promtail', 'loki', 'loki-canary']), +]; + +{ + drone: std.manifestYamlStream(drone), +} diff --git a/.drone/drone.yml b/.drone/drone.yml new file mode 100644 index 000000000000..c31ce937975b --- /dev/null +++ b/.drone/drone.yml @@ -0,0 +1,134 @@ +kind: pipeline +name: docker-amd64 +platform: + arch: amd64 + os: linux +steps: +- commands: + - apk add --no-cache bash git + - echo $(./tools/image-tag)-amd64 > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + password: + from_secret: docker_password + repo: shorez/promtail + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-image + settings: + dockerfile: cmd/loki/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki-canary + username: + from_secret: docker_username +--- +kind: pipeline +name: docker-arm64 +platform: + arch: arm64 + os: linux +steps: +- commands: + - apk add --no-cache bash git + - echo $(./tools/image-tag)-arm64 > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + password: + from_secret: docker_password + repo: shorez/promtail + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-image + settings: + dockerfile: cmd/loki/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki-canary + username: + from_secret: docker_username +--- +kind: pipeline +name: docker-arm +platform: + arch: arm + os: linux +steps: +- commands: + - apk add --no-cache bash git + - echo $(./tools/image-tag)-arm > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + password: + from_secret: docker_password + repo: shorez/promtail + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-image + settings: + dockerfile: cmd/loki/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki + username: + from_secret: docker_username +- depends_on: + - image-tag + image: plugins/docker + name: loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + password: + from_secret: docker_password + repo: shorez/loki-canary + username: + from_secret: docker_username diff --git a/Makefile b/Makefile index 1b5c5bbf82ca..37c2855dd579 100644 --- a/Makefile +++ b/Makefile @@ -394,7 +394,9 @@ endef # promtail promtail-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile . +promtail-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile.cross . promtail-debug-image: OCI_PLATFORMS= promtail-debug-image: @@ -405,7 +407,9 @@ promtail-push: promtail-image # loki loki-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . +loki-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross . loki-debug-image: OCI_PLATFORMS= loki-debug-image: @@ -416,7 +420,9 @@ loki-push: loki-image # loki-canary loki-canary-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . +loki-canary-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . loki-canary-push: loki-canary-image $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) @@ -432,3 +438,7 @@ build-image: benchmark-store: go run ./pkg/storage/hack/main.go go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out + +# regenerate drone yaml +drone: + jsonnet .drone/drone.jsonnet | jq .drone -r | yq -y . > .drone/drone.yml diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile index f769def8895e..37ec2ac7dace 100644 --- a/cmd/loki-canary/Dockerfile +++ b/cmd/loki-canary/Dockerfile @@ -1,16 +1,7 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary +RUN make clean && make BUILD_IN_CONTAINER=false loki-canary FROM alpine:3.9 RUN apk add --update --no-cache ca-certificates diff --git a/cmd/loki-canary/Dockerfile.cross b/cmd/loki-canary/Dockerfile.cross new file mode 100644 index 000000000000..f769def8895e --- /dev/null +++ b/cmd/loki-canary/Dockerfile.cross @@ -0,0 +1,18 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary +ENTRYPOINT [ "/usr/bin/loki-canary" ] diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile index 213275263cb5..7f556e3f70f6 100644 --- a/cmd/loki/Dockerfile +++ b/cmd/loki/Dockerfile @@ -1,16 +1,7 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/loki -f cmd/loki/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki +RUN make clean && make BUILD_IN_CONTAINER=false loki FROM alpine:3.9 RUN apk add --update --no-cache ca-certificates diff --git a/cmd/loki/Dockerfile.cross b/cmd/loki/Dockerfile.cross new file mode 100644 index 000000000000..213275263cb5 --- /dev/null +++ b/cmd/loki/Dockerfile.cross @@ -0,0 +1,21 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/loki -f cmd/loki/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki/loki /usr/bin/loki +COPY cmd/loki/loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 +ENTRYPOINT [ "/usr/bin/loki" ] +CMD ["-config.file=/etc/loki/local-config.yaml"] diff --git a/cmd/promtail/Dockerfile b/cmd/promtail/Dockerfile index 606343265558..c939a3221d9d 100644 --- a/cmd/promtail/Dockerfile +++ b/cmd/promtail/Dockerfile @@ -1,26 +1,17 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false promtail +RUN apt-get update && apt-get install -qy libsystemd-dev +RUN make clean && make BUILD_IN_CONTAINER=false promtail # Promtail requires debian as the base image to support systemd journal reading FROM debian:stretch-slim # tzdata required for the timestamp stage to work RUN apt-get update && \ - apt-get install -qy \ - tzdata ca-certificates libsystemd-dev && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get install -qy \ + tzdata ca-certificates libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail /usr/bin/promtail COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml ENTRYPOINT ["/usr/bin/promtail"] - diff --git a/cmd/promtail/Dockerfile.cross b/cmd/promtail/Dockerfile.cross new file mode 100644 index 000000000000..606343265558 --- /dev/null +++ b/cmd/promtail/Dockerfile.cross @@ -0,0 +1,26 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false promtail + +# Promtail requires debian as the base image to support systemd journal reading +FROM debian:stretch-slim +# tzdata required for the timestamp stage to work +RUN apt-get update && \ + apt-get install -qy \ + tzdata ca-certificates libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail /usr/bin/promtail +COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml +ENTRYPOINT ["/usr/bin/promtail"] + From 3d5c45d16f22dec0b0760d0106a424d018e6754d Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 13 Aug 2019 15:55:23 +0200 Subject: [PATCH 2/5] chore(packaging): tag rules --- .drone/docker-manifest.tmpl | 8 +++---- .drone/drone.jsonnet | 7 +++--- .drone/drone.yml | 47 +++++++++++++++++++++++++++++++++++++ tools/image-tag | 2 +- 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl index b940a695224c..6b02e7809d85 100644 --- a/.drone/docker-manifest.tmpl +++ b/.drone/docker-manifest.tmpl @@ -1,4 +1,4 @@ -image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} {{#if build.tags}} tags: {{#each build.tags}} @@ -6,16 +6,16 @@ tags: {{/each}} {{/if}} manifests: - - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64 + - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 platform: architecture: amd64 os: linux - - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64 + - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 platform: architecture: arm64 os: linux variant: v8 - - image: shorez/{{ config.target }}:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm + - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm platform: architecture: arm os: linux diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index ef4f140f7d12..c0879f6d5427 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -10,8 +10,6 @@ local docker(arch, app) = { settings: { repo: 'shorez/%s' % app, dockerfile: 'cmd/%s/Dockerfile' % app, - // auto_tag: true, - // auto_tag_suffix: arch, username: { from_secret: 'docker_username' }, password: { from_secret: 'docker_password' }, }, @@ -27,6 +25,7 @@ local multiarch_image(arch) = pipeline('docker-' + arch) { image: 'alpine', commands: [ 'apk add --no-cache bash git', + 'git fetch origin --tags', 'echo $(./tools/image-tag)-%s > .tags' % arch, ], }] + [ @@ -45,11 +44,11 @@ local manifest(apps) = pipeline('manifest') { // as it is unused in spec mode. See docker-manifest.tmpl target: app, spec: '.drone/docker-manifest.tmpl', - auto_tag: true, ignore_missing: true, username: { from_secret: 'docker_username' }, password: { from_secret: 'docker_password' }, }, + depends_on: ['clone'], } for app in apps ], @@ -66,7 +65,7 @@ local drone = [ multiarch_image('arm64'), multiarch_image('arm'), - // manifest(['promtail', 'loki', 'loki-canary']), + manifest(['promtail', 'loki', 'loki-canary']), ]; { diff --git a/.drone/drone.yml b/.drone/drone.yml index c31ce937975b..7a7fe081ad1b 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -6,6 +6,7 @@ platform: steps: - commands: - apk add --no-cache bash git + - git fetch origin --tags - echo $(./tools/image-tag)-amd64 > .tags image: alpine name: image-tag @@ -51,6 +52,7 @@ platform: steps: - commands: - apk add --no-cache bash git + - git fetch origin --tags - echo $(./tools/image-tag)-arm64 > .tags image: alpine name: image-tag @@ -96,6 +98,7 @@ platform: steps: - commands: - apk add --no-cache bash git + - git fetch origin --tags - echo $(./tools/image-tag)-arm > .tags image: alpine name: image-tag @@ -132,3 +135,47 @@ steps: repo: shorez/loki-canary username: from_secret: docker_username +--- +depends_on: +- docker-amd64 +- docker-arm +- docker-arm64 +kind: pipeline +name: manifest +steps: +- depends_on: + - clone + image: plugins/manifest + name: manifest-promtail + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: promtail + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: loki + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki-canary + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: loki-canary + username: + from_secret: docker_username diff --git a/tools/image-tag b/tools/image-tag index c8db5f11d553..40a63b94a292 100755 --- a/tools/image-tag +++ b/tools/image-tag @@ -8,6 +8,6 @@ WIP=$(git diff --quiet || echo '-WIP') BRANCH=$(git rev-parse --abbrev-ref HEAD | sed s#/#-#g) SHA=$(git rev-parse --short HEAD) -# If this is a tag, use it, otherwise branch-hash +# If this is a tag, use it, otherwise branch-hash TAG=$(git describe --exact-match 2> /dev/null || echo "${BRANCH}-${SHA}") echo ${TAG}${WIP} From d073c70d965f62770bec7fa266d1a8ff8dd6a61f Mon Sep 17 00:00:00 2001 From: sh0rez Date: Wed, 14 Aug 2019 13:55:53 +0200 Subject: [PATCH 3/5] chore(ci/cd): drone dry run Prevents drone from actually publishing anything, but rather just do as-if, until we are certain that it should replace CircleCI --- .drone/drone.jsonnet | 6 ++++- .drone/drone.yml | 53 ++++++++------------------------------------ 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index c0879f6d5427..66da74bafd3c 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -12,6 +12,8 @@ local docker(arch, app) = { dockerfile: 'cmd/%s/Dockerfile' % app, username: { from_secret: 'docker_username' }, password: { from_secret: 'docker_password' }, + //TODO: disable once considered stable + dry_run: true, }, }; @@ -65,7 +67,9 @@ local drone = [ multiarch_image('arm64'), multiarch_image('arm'), - manifest(['promtail', 'loki', 'loki-canary']), + // tie it all up as a fast manifest + // TODO: enable once considered stable + // manifest(['promtail', 'loki', 'loki-canary']), ]; { diff --git a/.drone/drone.yml b/.drone/drone.yml index 7a7fe081ad1b..8ce7fdf23563 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -16,6 +16,7 @@ steps: name: promtail-image settings: dockerfile: cmd/promtail/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/promtail @@ -27,6 +28,7 @@ steps: name: loki-image settings: dockerfile: cmd/loki/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki @@ -38,6 +40,7 @@ steps: name: loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki-canary @@ -62,6 +65,7 @@ steps: name: promtail-image settings: dockerfile: cmd/promtail/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/promtail @@ -73,6 +77,7 @@ steps: name: loki-image settings: dockerfile: cmd/loki/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki @@ -84,6 +89,7 @@ steps: name: loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki-canary @@ -108,6 +114,7 @@ steps: name: promtail-image settings: dockerfile: cmd/promtail/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/promtail @@ -119,6 +126,7 @@ steps: name: loki-image settings: dockerfile: cmd/loki/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki @@ -130,52 +138,9 @@ steps: name: loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile + dry_run: true password: from_secret: docker_password repo: shorez/loki-canary username: from_secret: docker_username ---- -depends_on: -- docker-amd64 -- docker-arm -- docker-arm64 -kind: pipeline -name: manifest -steps: -- depends_on: - - clone - image: plugins/manifest - name: manifest-promtail - settings: - ignore_missing: true - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: promtail - username: - from_secret: docker_username -- depends_on: - - clone - image: plugins/manifest - name: manifest-loki - settings: - ignore_missing: true - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: loki - username: - from_secret: docker_username -- depends_on: - - clone - image: plugins/manifest - name: manifest-loki-canary - settings: - ignore_missing: true - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: loki-canary - username: - from_secret: docker_username From f253f6d985a34488223dd8dc74405160763bf183 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Wed, 14 Aug 2019 14:36:44 +0200 Subject: [PATCH 4/5] chore(ci/cd): s/shorez/grafana/g --- .drone/docker-manifest.tmpl | 8 ++++---- .drone/drone.jsonnet | 2 +- .drone/drone.yml | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl index 6b02e7809d85..54f26489e1a7 100644 --- a/.drone/docker-manifest.tmpl +++ b/.drone/docker-manifest.tmpl @@ -1,4 +1,4 @@ -image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} +image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} {{#if build.tags}} tags: {{#each build.tags}} @@ -6,16 +6,16 @@ tags: {{/each}} {{/if}} manifests: - - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 + - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 platform: architecture: amd64 os: linux - - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 + - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 platform: architecture: arm64 os: linux variant: v8 - - image: shorez/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm + - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm platform: architecture: arm os: linux diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index 66da74bafd3c..6bf24ad660e3 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -8,7 +8,7 @@ local docker(arch, app) = { name: '%s-image' % app, image: 'plugins/docker', settings: { - repo: 'shorez/%s' % app, + repo: 'grafana/%s' % app, dockerfile: 'cmd/%s/Dockerfile' % app, username: { from_secret: 'docker_username' }, password: { from_secret: 'docker_password' }, diff --git a/.drone/drone.yml b/.drone/drone.yml index 8ce7fdf23563..34022bdc1802 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -19,7 +19,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/promtail + repo: grafana/promtail username: from_secret: docker_username - depends_on: @@ -31,7 +31,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki + repo: grafana/loki username: from_secret: docker_username - depends_on: @@ -43,7 +43,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki-canary + repo: grafana/loki-canary username: from_secret: docker_username --- @@ -68,7 +68,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/promtail + repo: grafana/promtail username: from_secret: docker_username - depends_on: @@ -80,7 +80,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki + repo: grafana/loki username: from_secret: docker_username - depends_on: @@ -92,7 +92,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki-canary + repo: grafana/loki-canary username: from_secret: docker_username --- @@ -117,7 +117,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/promtail + repo: grafana/promtail username: from_secret: docker_username - depends_on: @@ -129,7 +129,7 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki + repo: grafana/loki username: from_secret: docker_username - depends_on: @@ -141,6 +141,6 @@ steps: dry_run: true password: from_secret: docker_password - repo: shorez/loki-canary + repo: grafana/loki-canary username: from_secret: docker_username From b2e9d40140849b36f27da4413ca8df0713798586 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Aug 2019 19:37:28 +0200 Subject: [PATCH 5/5] chore(ci/cd): build images on PR as well --- .drone/docker-manifest.tmpl | 8 +- .drone/drone.jsonnet | 60 +++++-- .drone/drone.yml | 327 +++++++++++++++++++++++++++++++----- 3 files changed, 333 insertions(+), 62 deletions(-) diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl index 54f26489e1a7..7ebb8c942407 100644 --- a/.drone/docker-manifest.tmpl +++ b/.drone/docker-manifest.tmpl @@ -1,4 +1,4 @@ -image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} +image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} {{#if build.tags}} tags: {{#each build.tags}} @@ -6,16 +6,16 @@ tags: {{/each}} {{/if}} manifests: - - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 platform: architecture: amd64 os: linux - - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 platform: architecture: arm64 os: linux variant: v8 - - image: grafana/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm platform: architecture: arm os: linux diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index 6bf24ad660e3..366d1f6c4706 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -1,3 +1,18 @@ +local apps = ['loki', 'loki-canary', 'promtail']; +local archs = ['amd64', 'arm64', 'arm']; + +local condition(verb) = { + tagMaster: { + ref: { + [verb]: + [ + 'refs/heads/master', + 'refs/tags/v*', + ], + }, + }, +}; + local pipeline(name) = { kind: 'pipeline', name: name, @@ -5,15 +20,14 @@ local pipeline(name) = { }; local docker(arch, app) = { - name: '%s-image' % app, + name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, image: 'plugins/docker', settings: { - repo: 'grafana/%s' % app, + repo: 'grafanasaur/%s' % app, dockerfile: 'cmd/%s/Dockerfile' % app, - username: { from_secret: 'docker_username' }, - password: { from_secret: 'docker_password' }, - //TODO: disable once considered stable - dry_run: true, + username: { from_secret: 'saur_username' }, + password: { from_secret: 'saur_password' }, + dry_run: false, }, }; @@ -31,8 +45,20 @@ local multiarch_image(arch) = pipeline('docker-' + arch) { 'echo $(./tools/image-tag)-%s > .tags' % arch, ], }] + [ - docker(arch, app) { depends_on: ['image-tag'] } - for app in ['promtail', 'loki', 'loki-canary'] + // dry run for everything that is not tag or master + docker(arch, app) { + depends_on: ['image-tag'], + when: condition('exclude').tagMaster, + settings+: { dry_run: true }, + } + for app in apps + ] + [ + // publish for tag or master + docker(arch, app) { + depends_on: ['image-tag'], + when: condition('include').tagMaster, + } + for app in apps ], }; @@ -56,20 +82,18 @@ local manifest(apps) = pipeline('manifest') { ], } + { depends_on: [ - 'docker-amd64', - 'docker-arm', - 'docker-arm64', + 'docker-%s' % arch + for arch in archs ], }; local drone = [ - multiarch_image('amd64'), - multiarch_image('arm64'), - multiarch_image('arm'), - - // tie it all up as a fast manifest - // TODO: enable once considered stable - // manifest(['promtail', 'loki', 'loki-canary']), + multiarch_image(arch) + for arch in archs +] + [ + manifest(['promtail', 'loki', 'loki-canary']) { + trigger: condition('include').tagMaster, + }, ]; { diff --git a/.drone/drone.yml b/.drone/drone.yml index 34022bdc1802..10c77f023066 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -13,39 +13,105 @@ steps: - depends_on: - image-tag image: plugins/docker - name: promtail-image + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image settings: dockerfile: cmd/promtail/Dockerfile dry_run: true password: - from_secret: docker_password - repo: grafana/promtail + from_secret: saur_password + repo: grafanasaur/promtail username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-image + name: publish-loki-image settings: dockerfile: cmd/loki/Dockerfile - dry_run: true + dry_run: false password: - from_secret: docker_password - repo: grafana/loki + from_secret: saur_password + repo: grafanasaur/loki username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-canary-image + name: publish-loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile - dry_run: true + dry_run: false password: - from_secret: docker_password - repo: grafana/loki-canary + from_secret: saur_password + repo: grafanasaur/loki-canary username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* --- kind: pipeline name: docker-arm64 @@ -62,39 +128,105 @@ steps: - depends_on: - image-tag image: plugins/docker - name: promtail-image + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image settings: dockerfile: cmd/promtail/Dockerfile dry_run: true password: - from_secret: docker_password - repo: grafana/promtail + from_secret: saur_password + repo: grafanasaur/promtail username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-image + name: publish-loki-image settings: dockerfile: cmd/loki/Dockerfile - dry_run: true + dry_run: false password: - from_secret: docker_password - repo: grafana/loki + from_secret: saur_password + repo: grafanasaur/loki username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-canary-image + name: publish-loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile - dry_run: true + dry_run: false password: - from_secret: docker_password - repo: grafana/loki-canary + from_secret: saur_password + repo: grafanasaur/loki-canary username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* --- kind: pipeline name: docker-arm @@ -111,36 +243,151 @@ steps: - depends_on: - image-tag image: plugins/docker - name: promtail-image + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image settings: dockerfile: cmd/promtail/Dockerfile dry_run: true password: - from_secret: docker_password - repo: grafana/promtail + from_secret: saur_password + repo: grafanasaur/promtail username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-image + name: publish-loki-image settings: dockerfile: cmd/loki/Dockerfile - dry_run: true + dry_run: false password: - from_secret: docker_password - repo: grafana/loki + from_secret: saur_password + repo: grafanasaur/loki username: - from_secret: docker_username + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* - depends_on: - image-tag image: plugins/docker - name: loki-canary-image + name: publish-loki-canary-image settings: dockerfile: cmd/loki-canary/Dockerfile - dry_run: true + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +--- +depends_on: +- docker-amd64 +- docker-arm64 +- docker-arm +kind: pipeline +name: manifest +steps: +- depends_on: + - clone + image: plugins/manifest + name: manifest-promtail + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: promtail + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: loki + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki-canary + settings: + ignore_missing: true password: from_secret: docker_password - repo: grafana/loki-canary + spec: .drone/docker-manifest.tmpl + target: loki-canary username: from_secret: docker_username +trigger: + ref: + include: + - refs/heads/master + - refs/tags/v*