Merge pull request #39 from paritytech/as-new-runners-ci

[ci] small refactor and add runners for deploy

.gitlab-ci.yml

@@ -3,18 +3,20 @@ stages:
   - lint
   - test
   - build
-  - deploy
+  - staging
+  - production
 
 variables:
   CI_IMAGE:                        "node:16.10"
   BUILDAH_IMAGE:                   "quay.io/buildah/stable:v1.27"
   APP_PORT:                        3000
-  DEPLOYMENT_IMAGE_REGISTRY:       docker.io/paritytech/github-issue-sync
-  DEPLOYMENT_TAG:                  $CI_COMMIT_SHORT_SHA-$CI_PIPELINE_ID
+  IMAGE_NAME:                      "docker.io/paritytech/github-issue-sync"
+  DOCKER_TAG:                      "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
   # Vault Section
-  VAULT_ADDR:                      https://vault.parity-mgmt-vault.parity.io
-  VAULT_AUTH_PATH:                 gitlab-parity-io-jwt
-  VAULT_AUTH_ROLE:                 cicd_gitlab_parity_${CI_PROJECT_NAME}
+  VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
+  VAULT_ADDR:                      "${VAULT_SERVER_URL}"
+  VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
+  VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
   HELM_SECRETS_DRIVER:             vals
 
 default:
@@ -29,36 +31,49 @@ default:
     - kubernetes-parity-build
   image:                           $CI_IMAGE
 
-.common-refs:                      &common-refs
-  rules:
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                 # PRs
-
 .test-refs:                        &test-refs
   rules:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                 # PRs
 
-.staging-refs:                     &staging-refs
+.publish-refs:                     &publish-refs
   rules:
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^stg-v[0-9]+\.[0-9]+.*$/  # e.g. stg-v0.1
+
+.common-refs:                      &common-refs
+  rules:
+    - !reference [.test-refs, rules]
+    - !reference [.publish-refs, rules]
 
 .production-refs:                  &production-refs
   rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/      # e.g. v0.1
+    - if: $CI_COMMIT_REF_NAME == "master"
+      when: manual
 
 lint:
   <<:                              *common-refs
   stage:                           lint
   script:
-    # TODO: Use pre-commit run --all
     - yarn --immutable
     - yarn lint
 
-# template task for building and pushing an image
-.build-push-docker-image:          &build-push-docker-image
+build-docker-only:
   stage:                           build
   image:                           $BUILDAH_IMAGE
+  <<:                              *test-refs
+  script:
+    - buildah bud
+        --format=docker
+        --build-arg VCS_REF="$CI_COMMIT_SHA"
+        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg VERSION="$DEPLOYMENT_TAG"
+        --build-arg PORT="$APP_PORT"
+        --tag "$IMAGE_NAME:$DOCKER_TAG"
+        --file src/server/Dockerfile .
+
+build-push-docker-image:
+  stage:                           build
+  <<:                              *publish-refs
+  image:                           $BUILDAH_IMAGE
   script:
     - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
         ( echo "no docker credentials provided"; exit 1 )
@@ -70,15 +85,15 @@ lint:
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
         --build-arg VERSION="$DEPLOYMENT_TAG"
         --build-arg PORT="$APP_PORT"
-        --tag "$DEPLOYMENT_IMAGE_REGISTRY:$DEPLOYMENT_TAG"
+        --tag "$IMAGE_NAME:$DOCKER_TAG"
         --file src/server/Dockerfile .
     - buildah bud
         --format=docker
         --build-arg VCS_REF="$CI_COMMIT_SHA"
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
         --build-arg VERSION="$DEPLOYMENT_TAG"
         --build-arg PORT="$APP_PORT"
-        --tag "$DEPLOYMENT_IMAGE_REGISTRY:$DEPLOYMENT_TAG"
+        --tag "$IMAGE_NAME:$DOCKER_TAG"
         --file src/server/Dockerfile .
     - echo "$Docker_Hub_Pass_Parity" |
         buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
@@ -87,37 +102,21 @@ lint:
   after_script:
     - buildah logout --all
 
-build-docker-only:
-  stage:                           build
-  image:                           $BUILDAH_IMAGE
-  <<:                              *test-refs
-  script:
-    - buildah bud
-        --format=docker
-        --build-arg VCS_REF="$CI_COMMIT_SHA"
-        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
-        --build-arg VERSION="$DEPLOYMENT_TAG"
-        --build-arg PORT="$APP_PORT"
-        --tag "$DEPLOYMENT_IMAGE_REGISTRY:$DEPLOYMENT_TAG"
-        --file src/server/Dockerfile .
-
-build-staging:
-  <<:                              *staging-refs
-  <<:                              *build-push-docker-image
-  environment:
-    name:                          parity-prod
-
-build-production:
-  <<:                              *production-refs
-  <<:                              *build-push-docker-image
-  environment:
-    name:                          parity-stg
 
 # Deploy to Kubernetes
 .deploy-k8s:                       &deploy-k8s
-  stage:                           deploy
   image:                           "paritytech/kubetools:3.5.3"
   script:
+    # Change the app version during build so that Kubernetes is able to update
+    # the deployment even if the images' tags did not change
+    # $CI_PIPELINE_ID is guaranteed to be unique
+    - export KUBERNETES_VERSION_TAG="$CI_PIPELINE_ID"
+    # Those values are embedded in the annotations and that's how the change is
+    # noticed
+    - |-
+      sed -i "s/version:.*/version: $KUBERNETES_VERSION_TAG/" helm/Chart.yaml
+    - |-
+      sed -i "s/appVersion:.*/appVersion: $KUBERNETES_VERSION_TAG/" helm/Chart.yaml
     # https://docs.gitlab.com/ee/ci/examples/authenticating-with-hashicorp-vault/#example
     - export VAULT_TOKEN="$(vault write -field=token auth/$VAULT_AUTH_PATH/login role=$VAULT_AUTH_ROLE jwt=$CI_JOB_JWT)"
     - helm dependency update helm/
@@ -128,8 +127,9 @@ build-production:
         --atomic
         --timeout 300s
         --namespace github-issue-sync
-        --values helm/$VALUES_FILE
-        --set common.image.tag="$DEPLOYMENT_TAG"
+        --values helm/values.yaml
+        --values helm/values-${ENVIRONMENT}.yaml
+        --set common.image.tag="$DOCKER_TAG"
         --set common.containerPort="$APP_PORT"
         --set common.livenessProbe.httpGet.port="$APP_PORT"
         --set common.readinessProbe.httpGet.port="$APP_PORT"
@@ -139,25 +139,28 @@ build-production:
         --atomic
         --timeout 300s
         --namespace github-issue-sync
-        --values helm/$VALUES_FILE
-        --set common.image.tag="$DEPLOYMENT_TAG"
+        --values helm/values.yaml
+        --values helm/values-${ENVIRONMENT}.yaml
+        --set common.image.tag="$DOCKER_TAG"
         --set common.containerPort="$APP_PORT"
         --set common.livenessProbe.httpGet.port="$APP_PORT"
         --set common.readinessProbe.httpGet.port="$APP_PORT"
         github-issue-sync helm/
 
 deploy-staging:
-  <<:                              *staging-refs
+  stage:                           staging
+  <<:                              *publish-refs
   <<:                              *deploy-k8s
   variables:
-    VALUES_FILE:                   values-stg.yaml
-  environment:
-    name:                          parity-stg
+    ENVIRONMENT:                   parity-stg
+  tags:
+    - github-issue-sync-stg
 
 deploy-production:
+  stage:                           production
   <<:                              *production-refs
   <<:                              *deploy-k8s
   variables:
-    VALUES_FILE:                   values-prod.yaml
-  environment:
-    name:                          parity-prod
+    ENVIRONMENT:                   parity-prod
+  tags:
+    - github-issue-sync-prod

CODEOWNERS

@@ -20,4 +20,4 @@
 
 # CI
 /.gitlab-ci.yml @paritytech/ci @paritytech/opstooling
-/helm/ @paritytech/devops @paritytech/opstooling
+/helm/ @paritytech/ci @paritytech/opstooling

helm/values-prod.yaml → helm/values-parity-prod.yaml

@@ -2,9 +2,9 @@ common:
   serviceAccount:
     create: true
     annotations:
-      iam.gke.io/gcp-service-account: 'github-issue-sync@parity-prod.iam.gserviceaccount.com'
+      iam.gke.io/gcp-service-account: "github-issue-sync@parity-prod.iam.gserviceaccount.com"
     name: "github-issue-sync"
-  
+
   env:
     DB_PASSWORD: ref+vault://kv/cicd/gitlab/parity/opstooling/github-issue-sync/parity-prod#DB_PASSWORD
     API_CONTROL_TOKEN: ref+vault://kv/cicd/gitlab/parity/opstooling/github-issue-sync/parity-prod#API_CONTROL_TOKEN
@@ -39,7 +39,7 @@ common:
       - secretName: github-issue-sync.parity-prod.parity.io
         hosts:
           - github-issue-sync.parity-prod.parity.io
-  
+
   cloudsql:
     commandline:
       args: "-instances=parity-prod:europe-west3:github-issue-sync-db-pgsql=tcp:0.0.0.0:5432"

helm/values-stg.yaml → helm/values-parity-stg.yaml

@@ -2,9 +2,9 @@ common:
   serviceAccount:
     create: true
     annotations:
-      iam.gke.io/gcp-service-account: 'github-issue-sync@parity-stg.iam.gserviceaccount.com'
+      iam.gke.io/gcp-service-account: "github-issue-sync@parity-stg.iam.gserviceaccount.com"
     name: "github-issue-sync"
-  
+
   env:
     DB_PASSWORD: ref+vault://kv/cicd/gitlab/parity/opstooling/github-issue-sync/parity-stg#DB_PASSWORD
     API_CONTROL_TOKEN: ref+vault://kv/cicd/gitlab/parity/opstooling/github-issue-sync/parity-stg#API_CONTROL_TOKEN
@@ -41,7 +41,7 @@ common:
       - secretName: github-issue-sync.parity-stg.parity.io
         hosts:
           - github-issue-sync.parity-stg.parity.io
-  
+
   cloudsql:
     commandline:
       args: "-instances=parity-stg:europe-west3:github-issue-sync-db-pgsql=tcp:0.0.0.0:5432"