Add test matrix

Signed-off-by: Tom Wieczorek <twieczorek@mirantis.com>

.github/workflows/ostests-e2e.yaml

@@ -0,0 +1,220 @@
+# OS e2e testing workflow. See ../../hack/ostests/README.md for details.
+
+name: "OS tests :: e2e"
+
+on:
+  workflow_call:
+    inputs:
+      k0s-version:
+        type: string
+        description: The k0s version to test. Will try to download a build artifact if empty.
+      e2e-parallel:
+        type: boolean
+        description: Whether to run tests in parallel.
+        default: true
+      e2e-focus:
+        type: string
+        description: The selector for the e2e tests to be run.
+        default: \[Conformance\]
+      os:
+        type: string
+        description: The operating system to test.
+        required: true
+      network-provider:
+        type: string
+        description: The k0s network provider to test.
+        required: true
+      terraform-version:
+        type: string
+        description: The Terraform version to use when provisioning test resources.
+        default: 1.4.6
+      k0sctl-version:
+        type: string
+        description: The k0sctl version to use when bootstrapping the test cluster.
+        default: 0.15.0 # k0sproject/k0sctl#495
+    secrets:
+      aws-access-key-id:
+        description: The AWS access key ID to use when provisioning test resources.
+        required: true
+      aws-secret-access-key:
+        description: The AWS secret access key to use when provisioning test resources.
+        required: true
+      aws-session-token:
+        description: The AWS session token to use when provisioning test resources.
+        required: true
+
+env:
+  # This is hopefully somewhat physically close to where the hosted GitHub
+  # runners live.
+  #
+  # https://github.com/orgs/community/discussions/24969#discussioncomment-3246032
+  AWS_REGION: us-east-1
+
+jobs:
+  e2e-tests:
+    name: e2e tests
+    runs-on: ubuntu-22.04
+
+    env:
+      E2E_FOCUS: ${{ inputs.e2e-focus }}
+      TF_VAR_os: ${{ inputs.os }}
+      TF_VAR_additional_tags: '{
+        "ostests.k0sproject.io/github-run-id"="${{ github.run_id }}",
+        "ostests.k0sproject.io/github-run-attempt"="${{ github.run_attempt }}",
+        }'
+      TF_VAR_cache_dir: ${{ github.workspace }}/.cache/k0s-ostests
+      TF_VAR_k0s_version: ${{ inputs.k0s-version }}
+      TF_VAR_k0sctl_executable_path: ${{ github.workspace }}/.cache/k0sctl
+      TF_VAR_k0s_network_provider: ${{ inputs.network-provider }}
+
+    defaults:
+      run:
+        working-directory: hack/ostests
+
+    steps:
+      - name: "Workflow run :: Checkout"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: "Workflow run :: Prepare"
+        working-directory: ./
+        run: |
+          kubernetesVersion="$(./vars.sh kubernetes_version)"
+          echo KUBERNETES_VERSION="$kubernetesVersion" >>"$GITHUB_ENV"
+
+      - name: "Terraform :: Requisites :: Download k0s"
+        if: inputs.k0s-version == ''
+        uses: actions/download-artifact@v3
+        with:
+          name: k0s-linux-amd64
+          path: ${{ github.workspace }}/.cache
+
+      - name: "Terraform :: Requisites :: Prepare"
+        env:
+          K0SCTL_VERSION: ${{ inputs.k0sctl-version }}
+          K0S_VERSION: ${{ inputs.k0s-version }}
+          K0S_EXECUTABLE_PATH: ${{ github.workspace }}/.cache/k0s
+        run: |
+          kubectl version --client
+          jq --version
+
+          mkdir -p "$(dirname -- "$TF_VAR_k0sctl_executable_path")"
+          curl -sSLo "$TF_VAR_k0sctl_executable_path" "https://github.com/k0sproject/k0sctl/releases/download/v${K0SCTL_VERSION}/k0sctl-linux-x64"
+          chmod +x -- "$TF_VAR_k0sctl_executable_path"
+          "$TF_VAR_k0sctl_executable_path" version
+
+          if [ -z "$K0S_VERSION" ]; then
+            chmod +x -- "$K0S_EXECUTABLE_PATH"
+            K0S_VERSION="$("$K0S_EXECUTABLE_PATH" version)"
+            echo TF_VAR_k0s_executable_path="$K0S_EXECUTABLE_PATH" >>"$GITHUB_ENV"
+          fi
+
+          echo TF_VAR_k0s_version="$K0S_VERSION" >>"$GITHUB_ENV"
+
+      - name: "Terraform :: Setup"
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: ${{ inputs.terraform-version }}
+          terraform_wrapper: false
+
+      - name: "Terraform :: Init"
+        id: tf-init
+        run: terraform init
+
+      - name: "Terraform :: Apply"
+        id: tf-apply
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ secrets.aws-session-token }}
+        run: |
+          terraform apply -auto-approve
+
+          kubeconfigPath="$GITHUB_WORKSPACE/.cache/kubeconfig"
+          terraform output -raw k0s_kubeconfig >"$kubeconfigPath"
+          echo "KUBECONFIG=$kubeconfigPath" >>"$GITHUB_ENV"
+
+      - name: "e2e tests :: Run parallel tests"
+        working-directory: inttest
+        if: inputs.e2e-parallel
+        timeout-minutes: 180 # three hours
+        run: |
+          make bin/sonobuoy
+          bin/sonobuoy run -p e2e --wait=120 \
+            --kubernetes-version=v"$KUBERNETES_VERSION" \
+            --plugin-env=e2e.E2E_PARALLEL=true \
+            --plugin-env=e2e.E2E_FOCUS="$E2E_FOCUS" \
+            --plugin-env=e2e.E2E_SKIP='\[Serial\]' \
+            --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS='-v --timeout=90m --procs=2'
+
+      - name: "e2e tests :: Retrieve parallel results"
+        id: ct-retrieve-parallel
+        if: always() && inputs.e2e-parallel && steps.tf-apply.conclusion == 'success'
+        working-directory: inttest
+        run: |
+          make get-conformance-results
+          mv *_sonobuoy_*.tar.gz sonobuoy-parallel.tar.gz
+
+      - name: "e2e tests :: Clear parallel run"
+        if: always() && inputs.e2e-parallel && steps.tf-apply.conclusion == 'success'
+        working-directory: inttest
+        run: |
+          bin/sonobuoy delete --all --wait
+          echo E2E_FOCUS='\[Serial\].*'"$E2E_FOCUS" >>"$GITHUB_ENV"
+
+      - name: "e2e tests :: Run serial tests"
+        working-directory: inttest
+        timeout-minutes: 180 # three hours
+        run: |
+          make bin/sonobuoy
+          bin/sonobuoy run -p e2e --wait=120 \
+            --kubernetes-version=v"$KUBERNETES_VERSION" \
+            --plugin-env=e2e.E2E_FOCUS="$E2E_FOCUS" \
+            --plugin-env=e2e.E2E_EXTRA_GINKGO_ARGS='-v --timeout=90m'
+
+      - name: "e2e tests :: Retrieve serial results"
+        id: ct-retrieve-serial
+        if: always() && steps.tf-apply.conclusion == 'success'
+        working-directory: inttest
+        run: |
+          make get-conformance-results
+          mv *_sonobuoy_*.tar.gz sonobuoy-serial.tar.gz
+
+      - name: "Terraform :: Destroy"
+        if: always() && steps.tf-init.conclusion == 'success'
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ secrets.aws-session-token }}
+        run: terraform destroy -auto-approve
+
+      - name: "e2e tests :: Upload results"
+        if: always() && (steps.ct-retrieve-parallel.conclusion == 'success' || steps.ct-retrieve-serial.conclusion == 'success')
+        uses: actions/upload-artifact@v3
+        with:
+          name: ostests-e2e-${{ inputs.os }}-${{ inputs.network-provider }}-sonobuoy-results
+          path: |
+            inttest/sonobuoy-parallel.tar.gz
+            inttest/sonobuoy-serial.tar.gz
+
+      - name: "e2e tests :: Check"
+        working-directory: inttest
+        run: |
+          for f in sonobuoy-parallel.tar.gz sonobuoy-serial.tar.gz; do
+            echo "::group::$f"
+            bin/sonobuoy results "$f"
+            numNotPassedOrSkipped=$(bin/sonobuoy results "$f" -p=e2e --mode=detailed | jq --slurp '[.[] | select(.status != "passed" and .status != "skipped")] | length')
+            echo "Number of tests that didn't pass and weren't skipped: $numNotPassedOrSkipped"
+            echo ::endgroup::
+            if [ "$numNotPassedOrSkipped" != 0 ]; then
+              exit 1
+            fi
+          done
+
+      - name: "e2e tests :: Upload k0sctl logs"
+        if: always() && steps.tf-init.conclusion == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: ostests-e2e-${{ inputs.os }}-${{ inputs.network-provider }}-k0sctl-logs
+          path: ~/.cache/k0sctl/k0sctl.log

.github/workflows/ostests-matrix.yaml

@@ -0,0 +1,73 @@
+# OS testing matrix. See ../../hack/ostests/README.md for details.
+
+name: "OS tests :: Matrix"
+
+on:
+  workflow_dispatch:
+    inputs:
+      k0s-version:
+        type: string
+        description: The k0s version to test. Will build k0s from source if empty.
+      e2e-parallel:
+        type: boolean
+        description: Whether to run tests in parallel.
+        default: true
+      e2e-focus:
+        type: string
+        description: The selector for the e2e tests to be run.
+        default: \[Conformance\]
+      oses:
+        type: string
+        description: The operating systems to test.
+        required: true
+        default: >-
+          [
+            "alpine_3_17",
+            "centos_7", "centos_8", "centos_9",
+            "debian_10", "debian_11",
+            "fcos_38",
+            "fedora_38",
+            "flatcar",
+            "oracle_7_9", "oracle_8_7", "oracle_9_1",
+            "rhel_7", "rhel_8", "rhel_9",
+            "rocky_8", "rocky_9",
+            "ubuntu_2004", "ubuntu_2204", "ubuntu_2304"
+          ]
+      network-providers:
+        type: string
+        description: The k0s network providers to test.
+        required: true
+        default: >-
+          [
+            "kuberouter",
+            "calico"
+          ]
+
+jobs:
+  build:
+    name: Build
+    if: inputs.k0s-version == ''
+    uses: ./.github/workflows/build-k0s.yml
+    with: { target-os: linux, target-arch: amd64 }
+
+  e2e-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ fromJSON(github.event.inputs.oses) }}
+        network-provider: ${{ fromJSON(github.event.inputs.network-providers) }}
+
+    name: "${{ matrix.os }} :: ${{ matrix.network-provider }}"
+    needs: build
+    if: always() && (inputs.k0s-version != '' || needs.build.result == 'success')
+    uses: ./.github/workflows/ostests-e2e.yaml
+    with:
+      k0s-version: ${{ inputs.k0s-version }}
+      e2e-parallel: ${{ inputs.e2e-parallel }}
+      e2e-focus: ${{ inputs.e2e-focus }}
+      os: ${{ matrix.os }}
+      network-provider: ${{ matrix.network-provider }}
+    secrets:
+      aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }}

.github/workflows/ostests-nightly.yaml

@@ -0,0 +1,95 @@
+# Nightly trigger for OS testing. See ../../hack/ostests/README.md for details.
+
+name: "OS tests :: Nightly"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: 30 2 * * *
+
+env:
+  DISTRIBUTIONS: >-
+    [
+      ["alpine_3_17"],
+      ["centos_7", "centos_8", "centos_9"],
+      ["debian_10", "debian_11"],
+      ["fcos_38"],
+      ["fedora_38"],
+      ["flatcar"],
+      ["oracle_7_9", "oracle_8_7", "oracle_9_1"],
+      ["rhel_7", "rhel_8", "rhel_9"],
+      ["rocky_8", "rocky_9"],
+      ["ubuntu_2004", "ubuntu_2204", "ubuntu_2304"]
+    ]
+  NETWORK_PROVIDERS: >-
+    [
+      "kuberouter",
+      "calico"
+    ]
+
+jobs:
+  select:
+    name: Select
+    runs-on: ubuntu-latest
+
+    outputs:
+      os: ${{ steps.select.outputs.os }}
+      network-provider: ${{ steps.select.outputs.network-provider }}
+
+    steps:
+      # Generate a "well-distributed" list of all possible parameter
+      # combinations and select an entry based on the current day. This approach
+      # ensures that, if run once each day, there is a significant variation in
+      # parameter values each day while also guaranteeing that every combination
+      # will be chosen at some point.
+      - name: Select
+        id: select
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const distros = JSON.parse(process.env.DISTRIBUTIONS)
+            const networkProviders = JSON.parse(process.env.NETWORK_PROVIDERS)
+
+            const oses = []
+            for (let i = 0; ; i++) {
+              let added = false
+              for (const distro of distros) {
+                if (i < distro.length) {
+                  oses.push(distro[i]);
+                  added = true
+                }
+              }
+              if (!added) {
+                break
+              }
+            }
+
+            const combinations = []
+            for (const [i, _] of networkProviders.entries()) {
+              for (const [j, os] of oses.entries()) {
+                combinations.push([os, networkProviders[(i + j) % networkProviders.length]])
+              }
+            }
+
+            const daysSinceEpoch = Math.floor(Date.now() / (24 * 60 * 60 * 1000))
+            const [os, networkProvider] = combinations[daysSinceEpoch % combinations.length]
+            console.log(`Selected ${os}/${networkProvider}`)
+            core.setOutput('os', os)
+            core.setOutput('network-provider', networkProvider)
+
+  build:
+    name: Build
+    uses: ./.github/workflows/build-k0s.yml
+    with: { target-os: linux, target-arch: amd64 }
+
+  e2e-tests:
+    name: "${{ needs.select.outputs.os }} :: ${{ needs.select.outputs.network-provider }}"
+    needs: [select, build]
+    uses: ./.github/workflows/ostests-e2e.yaml
+    with:
+      os: ${{ needs.select.outputs.os }}
+      network-provider: ${{ needs.select.outputs.network-provider }}
+    secrets:
+      aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }}