Release 2020-10-06

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 2020-10-06
+
+## Internal
+
+* Ansible & Terraform for bootstrapping Kubernetes (#343)
+* Ansible & Terraform SFT improvements (#344, #346, #348)
+
 # 2020-09-28
 
 ## Features

ansible/Makefile

@@ -42,6 +42,16 @@ bootstrap: check-env
 		--private-key ${ENV_DIR}/operator-ssh.dec \
 		-vv
 
+# FUTUREWORK: https://github.com/zinfra/backend-issues/issues/1763
+.PHONY: kube-minio-static-files
+kube-minio-static-files: check-env
+	poetry run ansible-playbook ${ANSIBLE_DIR}/kube-minio-static-files.yml \
+		-i ${ENV_DIR}/gen/terraform-inventory.yml \
+		-i ${ENV_DIR}/inventory.yml \
+		--private-key ${ENV_DIR}/operator-ssh.dec \
+		--extra-vars "service_cluster_ip=$$(KUBECONFIG=${ENV_DIR}/gen/artifacts/admin.conf kubectl get service fake-aws-s3 -o json | jq -r .spec.clusterIP)" \
+		-vv
+
 .PHONY: check-env
 check-env:
 ifndef ENV_DIR

ansible/README.md

@@ -22,9 +22,9 @@ soon.
    the terraform folder](../terraform/README.md)
 1. Ensure `$ENV_DIR/operator-ssh.dec` exists and contains an ssh key for the
    environment.
-1. Ensure that `make apply` has been run for the environment. Please refer to
-   the [docs in the terraform folder](../terraform/README.md) for details about
-   how to run this.
+1. Ensure that `make apply` and `make create-inventory` have been run for the
+   environment. Please refer to the [docs in the terraform
+   folder](../terraform/README.md) for details about how to run this.
 1. Ensure all required variables are set in `$ENV_DIR/inventory.yml`
 1. Running `make bootstrap` from this directory will bootstrap the
    environment.

ansible/bootstrap.yml

@@ -1 +1,2 @@
 - import_playbook: ./provision-sft.yml
+- import_playbook: ./kubernetes.yml

ansible/kube-minio-static-files.yml

@@ -0,0 +1,54 @@
+# WARNING: This is not recommended for production use.
+#
+# FUTUREWORK: https://github.com/zinfra/backend-issues/issues/1763
+- hosts: minio
+  any_errors_fatal: true
+  become: true
+  gather_facts: true
+  vars:
+    minio_access_key: 'dummykey'
+    minio_secret_key: 'dummysecret'
+  tasks:
+    - name: "install minio client CLI"
+      import_role:
+        name: ansible-minio
+        tasks_from: install-client
+
+    - name: "add 'local' mc config alias with correct credentials"
+      shell: "mc config host add local http://{{ service_cluster_ip }}:9000 {{ minio_access_key }} {{ minio_secret_key }}"
+
+    - name: "create 'public' bucket"
+      shell: "mc mb --ignore-existing local/public"
+
+    - name: "make the 'public' bucket world-accessible"
+      shell: "mc policy set public local/public"
+      run_once: true
+
+    - name: "remove unneeded config aliases added by default"
+      shell: "mc config host rm {{ item }}"
+      with_items:
+        - gcs
+        - s3
+        - play
+
+    - name: "add some file to minio"
+      import_role:
+        name: minio-static-files
+      vars:
+        prefix: ""
+        domain: "${environment_name}.${root_domain}"
+        deeplink_title: "${environment_name}.${root_domain}"
+
+- hosts: minio
+  any_errors_fatal: true
+  become: true
+  gather_facts: true
+  tags: static-files
+  roles:
+    - role: minio-static-files
+      # Override these variables!
+      # FUTUREWORK: parse them from a configuration file shared with helm
+      # (as the domain needs to be known in helm override values.yaml)
+      prefix: "{{ minio_deeplink_prefix | default('example-') }}"
+      domain: "{{ minio_deeplink_domain | default('example.com') }}"
+      deeplink_title: "{{ minio_deeplink_domain | default('example.com environment') }}"

ansible/provision-sft.yml

@@ -10,3 +10,17 @@
   roles:
     - role: sft-server
     - role: srv-announcer
+  tasks:
+    # The Ubuntu images provided by hetzner have systemd-resolved enabled,
+    # but don't use the nss module, and direct all traffic through the
+    # 127.0.0.53 stub resolver
+    # This one seems to be flaky.
+    # Instead, configure it to use /run/systemd/resolve/resolv.conf, which points to
+    # the DNS servers retrieved via DHCP directly
+    - name: Workaround systemd-resolved being flaky
+      file:
+        src: /run/systemd/resolve/resolv.conf
+        dest: /etc/resolv.conf
+        owner: root
+        group: root
+        state: link

ansible/requirements.yml

@@ -66,4 +66,4 @@
 
 - src: git+https://github.com/wireapp/ansible-sft.git
   name: sft
-  version: "369db98e2e1ddf156ea0c9dbd2801c9d83e10a5a" # master (2020-09-25)
+  version: "e587a0d323c6a30d944d9c0e66203dfb7169026b" # develop (2020-09-30)

ansible/roles/minio-static-files/tasks/main.yml

@@ -1,3 +1,5 @@
+# FUTUREWORK: https://github.com/zinfra/backend-issues/issues/1763
+#
 - name: "create deeplink template files"
   template:
     src: "{{ item }}.j2"

terraform/README.md

@@ -58,6 +58,10 @@ Run all commands from `terraform/environment` directory.
    ```
    make apply
    ```
+1. Create inventory
+   ```
+   make create-inventory
+   ```
 1. To bootstrap the nodes, please refer to the [ansible README](../ansible/README.md)
 
 <sup>[1]</sup>For wire employees: Encrypt this file using `sops`, it will not

terraform/environment/Makefile

@@ -3,22 +3,32 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 CAILLEACH_DIR:=$(abspath ${ROOT_DIR}/../../../cailleach)
 SHELL:=/usr/bin/env bash
 
+export TF_DATA_DIR=${ENV_DIR}/.terraform
+
 .PHONY: init
 init: check-env
-	cd ${ENV_DIR} && \
-		terraform init -backend-config=${ENV_DIR}/backend.tfvars ${ROOT_DIR}
+	terraform init -backend-config=${ENV_DIR}/backend.tfvars
+
+.PHONY: output
+output: check-env
+	terraform output -json
+
+.PHONY: force-unlock
+force-unlock: check-env
+ifndef LOCK_ID
+	$(error please define LOCK_ID)
+endif
+	terraform force-unlock ${LOCK_ID} ${ROOT_DIR}
 
-.PHONY: apply
-apply: check-env
-	cd ${ENV_DIR} && \
-		source hcloud-token.dec && \
-		terraform apply -var 'inventory_file=./gen/terraform-inventory.yml' ${ROOT_DIR}
+.PHONY: create-inventory
+create-inventory: check-env
+	mkdir -p ${ENV_DIR}/gen && \
+		terraform output -json inventory > ${ENV_DIR}/gen/terraform-inventory.yml
 
-.PHONY: destroy
-destroy: check-env
-	cd ${ENV_DIR} && \
-		source hcloud-token.dec && \
-		terraform destroy -var 'inventory_file=./gen/terraform-inventory.yml' ${ROOT_DIR}
+.PHONY: apply plan console destroy
+apply plan console destroy:
+	source ${ENV_DIR}/hcloud-token.dec && \
+		terraform $@ -var-file=${ENV_DIR}/terraform.tfvars
 
 .PHONY: check-env
 check-env:

terraform/environment/hcloud.tf

@@ -6,15 +6,33 @@ variable "hcloud_location" {
   default = "nbg1"
 }
 
-variable "operator_ssh_public_key" {
-  type = string
+variable "operator_ssh_public_keys" {
+  type = object({
+    terraform_managed = map(string) # Map of key name to the public key content
+    preuploaded_key_names = set(string)
+  })
+  validation {
+    condition = (
+      length(var.operator_ssh_public_keys.terraform_managed) > 0 ||
+      length(var.operator_ssh_public_keys.preuploaded_key_names) > 0
+      )
+    error_message = "At least one key must be provided."
+  }
 }
 
 provider "hcloud" {
   # NOTE: You must have a HCLOUD_TOKEN environment variable set!
 }
 
 resource "hcloud_ssh_key" "operator_ssh" {
-  name = "${var.environment}-operator"
-  public_key = var.operator_ssh_public_key
+  for_each = var.operator_ssh_public_keys.terraform_managed
+  name = each.key
+  public_key = each.value
+}
+
+locals {
+  hcloud_ssh_keys = concat(
+    [for key in hcloud_ssh_key.operator_ssh: key.name],
+    tolist(var.operator_ssh_public_keys.preuploaded_key_names)
+    )
 }

terraform/environment/inventory.tf

@@ -2,13 +2,19 @@
 # this outside terraform using outputs, but it is not possible to use 'terraform
 # output' when the init directory is different from the root code directory.
 # Terraform Issue: https://github.com/hashicorp/terraform/issues/17300
-resource "local_file" "inventory" {
-  filename = var.inventory_file
-  content = jsonencode({
+locals {
+  kubernetes_nodes = flatten(module.hetzner_kubernetes[*].nodes)
+  kubernetes_hosts = {for node in local.kubernetes_nodes : node.hostname => {}}
+  sft_instances = flatten(module.sft[*].sft.instances)
+}
+
+output "inventory" {
+  value = {
     "sft_servers" = {
-      "hosts" = { for instance in module.sft[0].sft.instances :  instance.hostname => {
+      "hosts" = { for instance in local.sft_instances :  instance.hostname => {
         "ansible_host" = instance.ipaddress
         "ansible_ssh_user" = "root"
+        # NOTE: Maybe this is not required for ansible 2.9
         "ansible_python_interpreter" = "/usr/bin/python3"
         "sft_fqdn" = instance.fqdn
 
@@ -20,5 +26,27 @@ resource "local_file" "inventory" {
         "srv_announcer_record_target": instance.fqdn
       }}
     }
-  })
+    "kube-master"  = {"hosts" = local.kubernetes_hosts}
+    "kube-node" = {"hosts" = local.kubernetes_hosts}
+    "etcd" = {"hosts" = local.kubernetes_hosts}
+    "minio" = {"hosts" = local.kubernetes_hosts}
+    "k8s-cluster" = {
+      "hosts" = {for node in local.kubernetes_nodes :
+        node.hostname => {
+          "ansible_host" = node.ipaddress
+          "etcd_member_name" = node.etcd_member_name
+        }
+      }
+      "vars" = {
+        "ansible_ssh_user" = "root"
+        # NOTE: Maybe this is not required for ansible 2.9
+        "ansible_python_interpreter" = "/usr/bin/python3"
+
+        "helm_enabled" = true
+        "kubeconfig_localhost" = true
+        "bootstrap_os" = "ubuntu"
+        "docker_dns_servers_strict" = false
+      }
+    }
+  }
 }

terraform/environment/kubernetes.tf

@@ -0,0 +1,32 @@
+variable "kubernetes_node_count" {
+  type = number
+  default = 0
+  validation {
+    condition = (
+      var.kubernetes_node_count == 0 ||
+      var.kubernetes_node_count % 2 == 1
+    )
+    error_message = "The kubernetes_node_count must be 0 or an odd number. ETCD does not like even numbers."
+  }
+}
+
+variable "kubernetes_server_type" {
+  default = "cx51"
+}
+
+variable "kubernetes_ssh_keys" {
+  type = set(string)
+  default = []
+}
+
+module "hetzner_kubernetes" {
+  count =  min(1, var.kubernetes_node_count)
+
+  source = "../modules/hetzner-kubernetes"
+
+  environment = var.environment
+  root_domain = var.root_domain
+  server_type = var.kubernetes_server_type
+  ssh_keys = local.hcloud_ssh_keys
+  node_count = var.kubernetes_node_count
+}

terraform/environment/main.tf

@@ -5,7 +5,3 @@ variable "environment" {
 variable "root_domain" {
   type = string
 }
-
-variable "inventory_file" {
-  type = string
-}

terraform/environment/sft.tf

@@ -22,5 +22,5 @@ module "sft" {
   server_type = var.sft_server_type
   image = var.hcloud_image
   location = var.hcloud_location
-  ssh_keys = [hcloud_ssh_key.operator_ssh.name]
+  ssh_keys = local.hcloud_ssh_keys
 }

terraform/environment/terraform.tf

@@ -10,10 +10,6 @@ terraform {
       source  = "terraform-providers/hcloud"
       version = "~> 1.19"
     }
-    local = {
-      source = "hashicorp/local"
-      version = "~> 1.4.0"
-    }
   }
 
   backend s3 {

terraform/modules/aws-dns-records/resources.route53.tf

@@ -18,3 +18,13 @@ resource "aws_route53_record" "cname" {
   ttl     = var.ttl
   records = var.cnames
 }
+
+resource "aws_route53_record" "spf" {
+  count = var.create_spf_record ? 1 : 0
+
+  zone_id = data.aws_route53_zone.rz.zone_id
+  name    = join(".", local.name_suffix)
+  type    = "TXT"
+  ttl     = "60"
+  records = [ for ip in var.ips : "v=spf1 ip4:${ ip } -all" ]
+}

terraform/modules/aws-dns-records/variables.tf

@@ -50,4 +50,9 @@ variable "ttl" {
   type        = number
   description = "time to live for the DNS entries (defaults to 1 minute)"
   default     = 60
-}
+}
+
+variable "create_spf_record" {
+  type    = bool
+  default = false
+}

terraform/modules/hetzner-kubernetes/main.tf

@@ -0,0 +1,27 @@
+# FUTUREWORK: Attach a volume for etcd state, so we can recreate this machine
+# when we need to.
+resource "hcloud_server" "node" {
+  count       = var.node_count
+  name        = "${ var.environment }-kubenode${ format("%02d", count.index + 1 )}"
+  image       = var.image
+  server_type = var.server_type
+  ssh_keys    = var.ssh_keys
+  location    = var.location
+
+  labels = {
+    # FUTUREWORK: This label name is very undecriptive and it should be renamed
+    # to "etcd_member_name". This is kept as it is because legacy environments
+    # have it.
+    member = "etcd${ format("%02d", count.index + 1 )}"
+  }
+}
+
+module "dns_records" {
+  source = "../aws-dns-records"
+
+  environment = var.environment
+
+  zone_fqdn = var.root_domain
+  ips = [ for node in hcloud_server.node: node.ipv4_address ]
+  create_spf_record = true
+}

terraform/modules/hetzner-kubernetes/outputs.tf

@@ -0,0 +1,9 @@
+output "nodes" {
+  value = [ for node in hcloud_server.node :
+      {
+        hostname = node.name
+        ipaddress = node.ipv4_address
+        etcd_member_name = node.labels.member
+      }
+    ]
+}