Azure · jainr · Feb 11, 2022 · Jan 28, 2022
diff --git a/cluster/sdk/infra/scripts/k8s-common-functions.sh b/cluster/sdk/infra/scripts/k8s-common-functions.sh
@@ -125,7 +125,7 @@ function setup_sparkop_role {
 
     cat <<EOF | kubectl apply -f -
 kind: Role
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: use-spark-operator
   namespace: sparkop
@@ -134,7 +134,7 @@ rules:
   resources: ["sparkapplications"]
   verbs: ["create", "delete", "deletecollection", "get", "list", "update", "watch", "patch"]
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: use-spark-operator

diff --git a/cluster/setup/feast-0.9.5-helmchart/.helmignore b/cluster/setup/feast-0.9.5-helmchart/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/cluster/setup/feast-0.9.5-helmchart/Chart.yaml b/cluster/setup/feast-0.9.5-helmchart/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Feature store for machine learning.
+name: feast
+version: 0.9.5
diff --git a/cluster/setup/feast-0.9.5-helmchart/README.md b/cluster/setup/feast-0.9.5-helmchart/README.md
@@ -0,0 +1,46 @@
+feast
+===== 
+
+Feature store for machine learning. Current chart version is `0.9.5`
+
+## Installation
+
+https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes
+
+## Chart Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+|  | feast-core | 0.9.5 |
+|  | feast-jupyter | 0.9.5 |
+|  | feast-serving | 0.9.5 |
+|  | prometheus-statsd-exporter | 0.1.2 |
+| https://charts.bitnami.com/bitnami/ | kafka | 11.8.8 |
+| https://kubernetes-charts.storage.googleapis.com/ | grafana | 5.0.5 |
+| https://kubernetes-charts.storage.googleapis.com/ | postgresql | 8.6.1 |
+| https://kubernetes-charts.storage.googleapis.com/ | prometheus | 11.0.2 |
+| https://kubernetes-charts.storage.googleapis.com/ | redis | 10.5.6 |
+
+## Chart Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| feast-core.enabled | bool | `true` | Flag to install Feast Core |
+| feast-core.postgresql.existingSecret | string | `"feast-postgresql"` | Kubernetes secrets that contains the postgresql password |
+| feast-jupyter.enabled | bool | `true` | Flag to install Feast Jupyter Notebook with SDK |
+| feast-online-serving.enabled | bool | `true` | Flag to install Feast Online Serving |
+| grafana.enabled | bool | `true` | Flag to install Grafana |
+| kafka.enabled | bool | `true` | Flag to install Kafka |
+| postgresql.enabled | bool | `true` | Flag to install Postgresql |
+| postgresql.existingSecret | string | `"feast-postgresql"` | Kubernetes secrets that contains the postgresql password |
+| prometheus-statsd-exporter.enabled | bool | `true` | Flag to install StatsD to Prometheus Exporter |
+| prometheus.enabled | bool | `true` | Flag to install Prometheus |
+| redis.enabled | bool | `true` | Flag to install Redis |
+| redis.usePassword | bool | `false` | Disable redis password |
+
+
+### Documentation development
+
+This `README.md` is generated using [helm-docs](https://github.com/norwoodj/helm-docs/).
+Please run `helm-docs` to regenerate the `README.md` every time `README.md.gotmpl`
+or `values.yaml` are updated.
diff --git a/cluster/setup/feast-0.9.5-helmchart/README.md.gotmpl b/cluster/setup/feast-0.9.5-helmchart/README.md.gotmpl
@@ -0,0 +1,18 @@
+{{ template "chart.header" . }} 
+
+{{ template "chart.description" . }} {{ template "chart.versionLine" . }}
+
+## Installation
+
+https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}
+
+
+### Documentation development
+
+This `README.md` is generated using [helm-docs](https://github.com/norwoodj/helm-docs/).
+Please run `helm-docs` to regenerate the `README.md` every time `README.md.gotmpl`
+or `values.yaml` are updated.
diff --git a/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/Chart.yaml b/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Feast Core registers feature specifications.
+name: feast-core
+version: 0.9.5
diff --git a/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/README.md b/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/README.md
@@ -0,0 +1,68 @@
+feast-core
+==========
+Feast Core registers feature specifications.
+
+Current chart version is `0.9.5`
+
+
+
+
+
+## Chart Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. |
+| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` |
+| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. |
+| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. |
+| envOverrides | object | `{}` | Extra environment variables to set |
+| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| image.repository | string | `"feastdev/feast-core"` | Docker image repository |
+| image.tag | string | `"develop"` | Image tag |
+| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges |
+| ingress.http.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service |
+| ingress.http.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.http.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.http.whitelist | string | `""` | Allowed client IP source ranges |
+| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap: <br> `-Xms2048m -Xmx2048m` |
+| livenessProbe.enabled | bool | `false` | Flag to enabled the probe |
+| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated |
+| livenessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| livenessProbe.timeoutSeconds | int | `5` | When the probe times out |
+| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` |
+| logType | string | `"Console"` | Log format, either `JSON` or `Console` |
+| nodeSelector | object | `{}` | Node labels for pod assignment |
+| podLabels | object | `{}` | Labels to be added to Feast Core pods |
+| postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres |
+| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics |
+| readinessProbe.enabled | bool | `true` | Flag to enabled the probe |
+| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated |
+| readinessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| readinessProbe.timeoutSeconds | int | `10` | When the probe times out |
+| replicaCount | int | `1` | Number of pods that will be created |
+| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) |
+| secrets | list | `[]` | List of Kubernetes secrets to be mounted on Feast Core pods. These secrets will be mounted on /etc/secrets/<secret name>. |
+| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.grpc.port | int | `6565` | Service port for GRPC requests |
+| service.grpc.targetPort | int | `6565` | Container port serving GRPC requests |
+| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.http.port | int | `80` | Service port for HTTP requests |
+| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics |
+| service.type | string | `"ClusterIP"` | Kubernetes service type |
diff --git a/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/_helpers.tpl b/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/_helpers.tpl
@@ -0,0 +1,45 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "feast-core.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "feast-core.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "feast-core.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "feast-core.labels" -}}
+app.kubernetes.io/name: {{ include "feast-core.name" . }}
+helm.sh/chart: {{ include "feast-core.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
diff --git a/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/_ingress.yaml b/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/_ingress.yaml
@@ -0,0 +1,68 @@
+{{- /*
+This takes an array of three values:
+- the top context
+- the feast component
+- the service protocol
+- the ingress context
+*/ -}}
+{{- define "feast.ingress" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+{{ include "feast.ingress.metadata" . }}
+spec:
+  rules:
+  {{- range $host := $ingressValues.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: {{ include (printf "feast-%s.fullname" $component) $top }}
+          servicePort: {{ index $top.Values "service" $protocol "port" }}
+  {{- end }}
+{{- if $ingressValues.https.enabled }}
+  tls:
+    {{- range $host := $ingressValues.hosts }}
+    - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }}
+      hosts:
+      - {{ $host }}
+    {{- end }}
+{{- end -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata" -}}
+{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }}
+{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}}
+{{- toYaml (merge $overrides $commonMetadata) -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata-overrides" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+{{- $commonFullname := include "common.fullname" $top }}
+metadata:
+  name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }}
+  annotations:
+    kubernetes.io/ingress.class: {{ $ingressValues.class | quote }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }}
+    nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }}
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user"
+    nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri"
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}}
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }}
+    # TODO: Allow choice of GRPC/GRPCS
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
+    {{- if $ingressValues.annotations -}}
+    {{ include "common.annote" $ingressValues.annotations | indent 4 }}
+    {{- end }}
+{{- end -}}
diff --git a/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/configmap.yaml b/cluster/setup/feast-0.9.5-helmchart/charts/feast-core/templates/configmap.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "feast-core.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "feast-core.name" . }}
+    component: core
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  application-generated.yaml: |
+{{- if index .Values "application-generated.yaml" "enabled" }}
+    spring:
+      datasource:
+        url: jdbc:postgresql://{{ .Release.Name }}-postgresql:5432/postgres
+
+    server:
+      port: {{ .Values.service.http.targetPort }}
+{{- end }}
+
+  application-override.yaml: |
+{{- if index .Values "application-override.yaml" "enabled" }}
+{{- toYaml (index .Values "application-override.yaml") | nindent 4 }}    
+{{- end }}