diff --git a/.circleci/config.yml b/.circleci/config.yml index e0882e5ca3dd..bd623920c2bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ workflows: - run: name: container command: | - make $APP-image + make $APP-image-cross # builds and pushes a container .publish: &publish diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl new file mode 100644 index 000000000000..7ebb8c942407 --- /dev/null +++ b/.drone/docker-manifest.tmpl @@ -0,0 +1,22 @@ +image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 + platform: + architecture: amd64 + os: linux + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - image: grafanasaur/{{config.target}}:{{#if build.tag}}{{build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet new file mode 100644 index 000000000000..366d1f6c4706 --- /dev/null +++ b/.drone/drone.jsonnet @@ -0,0 +1,101 @@ +local apps = ['loki', 'loki-canary', 'promtail']; +local archs = ['amd64', 'arm64', 'arm']; + +local condition(verb) = { + tagMaster: { + ref: { + [verb]: + [ + 'refs/heads/master', + 'refs/tags/v*', + ], + }, + }, +}; + +local pipeline(name) = { + kind: 'pipeline', + name: name, + steps: [], +}; + +local docker(arch, app) = { + name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, + image: 'plugins/docker', + settings: { + repo: 'grafanasaur/%s' % app, + dockerfile: 'cmd/%s/Dockerfile' % app, + username: { from_secret: 'saur_username' }, + password: { from_secret: 'saur_password' }, + dry_run: false, + }, +}; + +local multiarch_image(arch) = pipeline('docker-' + arch) { + platform: { + os: 'linux', + arch: arch, + }, + steps: [{ + name: 'image-tag', + image: 'alpine', + commands: [ + 'apk add --no-cache bash git', + 'git fetch origin --tags', + 'echo $(./tools/image-tag)-%s > .tags' % arch, + ], + }] + [ + // dry run for everything that is not tag or master + docker(arch, app) { + depends_on: ['image-tag'], + when: condition('exclude').tagMaster, + settings+: { dry_run: true }, + } + for app in apps + ] + [ + // publish for tag or master + docker(arch, app) { + depends_on: ['image-tag'], + when: condition('include').tagMaster, + } + for app in apps + ], +}; + +local manifest(apps) = pipeline('manifest') { + steps: [ + { + name: 'manifest-' + app, + image: 'plugins/manifest', + settings: { + // the target parameter is abused for the app's name, + // as it is unused in spec mode. See docker-manifest.tmpl + target: app, + spec: '.drone/docker-manifest.tmpl', + ignore_missing: true, + username: { from_secret: 'docker_username' }, + password: { from_secret: 'docker_password' }, + }, + depends_on: ['clone'], + } + for app in apps + ], +} + { + depends_on: [ + 'docker-%s' % arch + for arch in archs + ], +}; + +local drone = [ + multiarch_image(arch) + for arch in archs +] + [ + manifest(['promtail', 'loki', 'loki-canary']) { + trigger: condition('include').tagMaster, + }, +]; + +{ + drone: std.manifestYamlStream(drone), +} diff --git a/.drone/drone.yml b/.drone/drone.yml new file mode 100644 index 000000000000..10c77f023066 --- /dev/null +++ b/.drone/drone.yml @@ -0,0 +1,393 @@ +kind: pipeline +name: docker-amd64 +platform: + arch: amd64 + os: linux +steps: +- commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-amd64 > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +--- +kind: pipeline +name: docker-arm64 +platform: + arch: arm64 + os: linux +steps: +- commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-arm64 > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +--- +kind: pipeline +name: docker-arm +platform: + arch: arm + os: linux +steps: +- commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-arm > .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: build-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: true + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + exclude: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-image + settings: + dockerfile: cmd/loki/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-loki-canary-image + settings: + dockerfile: cmd/loki-canary/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/loki-canary + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +- depends_on: + - image-tag + image: plugins/docker + name: publish-promtail-image + settings: + dockerfile: cmd/promtail/Dockerfile + dry_run: false + password: + from_secret: saur_password + repo: grafanasaur/promtail + username: + from_secret: saur_username + when: + ref: + include: + - refs/heads/master + - refs/tags/v* +--- +depends_on: +- docker-amd64 +- docker-arm64 +- docker-arm +kind: pipeline +name: manifest +steps: +- depends_on: + - clone + image: plugins/manifest + name: manifest-promtail + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: promtail + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: loki + username: + from_secret: docker_username +- depends_on: + - clone + image: plugins/manifest + name: manifest-loki-canary + settings: + ignore_missing: true + password: + from_secret: docker_password + spec: .drone/docker-manifest.tmpl + target: loki-canary + username: + from_secret: docker_username +trigger: + ref: + include: + - refs/heads/master + - refs/tags/v* diff --git a/Makefile b/Makefile index 1b5c5bbf82ca..37c2855dd579 100644 --- a/Makefile +++ b/Makefile @@ -394,7 +394,9 @@ endef # promtail promtail-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile . +promtail-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) -f cmd/promtail/Dockerfile.cross . promtail-debug-image: OCI_PLATFORMS= promtail-debug-image: @@ -405,7 +407,9 @@ promtail-push: promtail-image # loki loki-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile . +loki-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) -f cmd/loki/Dockerfile.cross . loki-debug-image: OCI_PLATFORMS= loki-debug-image: @@ -416,7 +420,9 @@ loki-push: loki-image # loki-canary loki-canary-image: - $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile . +loki-canary-image-cross: + $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . loki-canary-push: loki-canary-image $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) @@ -432,3 +438,7 @@ build-image: benchmark-store: go run ./pkg/storage/hack/main.go go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out + +# regenerate drone yaml +drone: + jsonnet .drone/drone.jsonnet | jq .drone -r | yq -y . > .drone/drone.yml diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile index f769def8895e..37ec2ac7dace 100644 --- a/cmd/loki-canary/Dockerfile +++ b/cmd/loki-canary/Dockerfile @@ -1,16 +1,7 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary +RUN make clean && make BUILD_IN_CONTAINER=false loki-canary FROM alpine:3.9 RUN apk add --update --no-cache ca-certificates diff --git a/cmd/loki-canary/Dockerfile.cross b/cmd/loki-canary/Dockerfile.cross new file mode 100644 index 000000000000..f769def8895e --- /dev/null +++ b/cmd/loki-canary/Dockerfile.cross @@ -0,0 +1,18 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary +ENTRYPOINT [ "/usr/bin/loki-canary" ] diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile index 213275263cb5..7f556e3f70f6 100644 --- a/cmd/loki/Dockerfile +++ b/cmd/loki/Dockerfile @@ -1,16 +1,7 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/loki -f cmd/loki/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki +RUN make clean && make BUILD_IN_CONTAINER=false loki FROM alpine:3.9 RUN apk add --update --no-cache ca-certificates diff --git a/cmd/loki/Dockerfile.cross b/cmd/loki/Dockerfile.cross new file mode 100644 index 000000000000..213275263cb5 --- /dev/null +++ b/cmd/loki/Dockerfile.cross @@ -0,0 +1,21 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/loki -f cmd/loki/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki/loki /usr/bin/loki +COPY cmd/loki/loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 +ENTRYPOINT [ "/usr/bin/loki" ] +CMD ["-config.file=/etc/loki/local-config.yaml"] diff --git a/cmd/promtail/Dockerfile b/cmd/promtail/Dockerfile index 606343265558..c939a3221d9d 100644 --- a/cmd/promtail/Dockerfile +++ b/cmd/promtail/Dockerfile @@ -1,26 +1,17 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:latest -# Directories in this file are referenced from the root of the project not this folder -# This file is intented to be called from the root like so: -# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . -FROM golang:1.11.4-alpine as goenv -RUN go env GOARCH > /goarch && \ - go env GOARM > /goarm - -FROM --platform=linux/amd64 $BUILD_IMAGE as build -COPY --from=goenv /goarch /goarm / +FROM golang:1.12 as build COPY . /go/src/github.com/grafana/loki WORKDIR /go/src/github.com/grafana/loki -RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false promtail +RUN apt-get update && apt-get install -qy libsystemd-dev +RUN make clean && make BUILD_IN_CONTAINER=false promtail # Promtail requires debian as the base image to support systemd journal reading FROM debian:stretch-slim # tzdata required for the timestamp stage to work RUN apt-get update && \ - apt-get install -qy \ - tzdata ca-certificates libsystemd-dev && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get install -qy \ + tzdata ca-certificates libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail /usr/bin/promtail COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml ENTRYPOINT ["/usr/bin/promtail"] - diff --git a/cmd/promtail/Dockerfile.cross b/cmd/promtail/Dockerfile.cross new file mode 100644 index 000000000000..606343265558 --- /dev/null +++ b/cmd/promtail/Dockerfile.cross @@ -0,0 +1,26 @@ +ARG BUILD_IMAGE=grafana/loki-build-image:latest +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . +FROM golang:1.11.4-alpine as goenv +RUN go env GOARCH > /goarch && \ + go env GOARM > /goarm + +FROM --platform=linux/amd64 $BUILD_IMAGE as build +COPY --from=goenv /goarch /goarm / +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false promtail + +# Promtail requires debian as the base image to support systemd journal reading +FROM debian:stretch-slim +# tzdata required for the timestamp stage to work +RUN apt-get update && \ + apt-get install -qy \ + tzdata ca-certificates libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail /usr/bin/promtail +COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml +ENTRYPOINT ["/usr/bin/promtail"] + diff --git a/tools/image-tag b/tools/image-tag index c8db5f11d553..40a63b94a292 100755 --- a/tools/image-tag +++ b/tools/image-tag @@ -8,6 +8,6 @@ WIP=$(git diff --quiet || echo '-WIP') BRANCH=$(git rev-parse --abbrev-ref HEAD | sed s#/#-#g) SHA=$(git rev-parse --short HEAD) -# If this is a tag, use it, otherwise branch-hash +# If this is a tag, use it, otherwise branch-hash TAG=$(git describe --exact-match 2> /dev/null || echo "${BRANCH}-${SHA}") echo ${TAG}${WIP}