feat: supports gRPC routing (#251)

* feat: supports gRPC protocol Signed-off-by: Lin Yang <reaver@flomesh.io> * build: bump pipy to 0.90.0-123 Signed-off-by: Lin Yang <reaver@flomesh.io> * fix: detecting protocol Signed-off-by: Lin Yang <reaver@flomesh.io> * fix: ingress forward traffic to gRPC service through TLS Signed-off-by: Lin Yang <reaver@flomesh.io> * fix: connectTLS() to backend ONLY if CA is provided for this route Signed-off-by: Lin Yang <reaver@flomesh.io> * test: add testcase docs Signed-off-by: Lin Yang <reaver@flomesh.io> --------- Signed-off-by: Lin Yang <reaver@flomesh.io>
flomesh-io · Apr 10, 2023 · 4b8e448 · 4b8e448
1 parent fec822c
commit 4b8e448
Show file tree

Hide file tree

Showing 22 changed files with 668 additions and 41 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -14,8 +14,8 @@ indent_size = 4
 tab_width = 4
 
 [{*.markdown,*.md}]
-indent_size = 4
-tab_width = 4
+indent_size = 2
+tab_width = 2
 
 [{*.mk,GNUmakefile,makefile,Makefile}]
 tab_width = 4
diff --git a/charts/fsm/components/scripts.tar.gz b/charts/fsm/components/scripts.tar.gz
diff --git a/charts/fsm/components/scripts/ingress/config/main.json b/charts/fsm/components/scripts/ingress/config/main.json
@@ -16,8 +16,11 @@
     "upstreamPort": 443
   },
 
+  "detectProtocol": true,
+
   "plugins": [
     "plugins/reject-http.js",
+    "plugins/protocol.js",
     "plugins/router.js",
     "plugins/balancer.js",
     "plugins/default.js"

diff --git a/charts/fsm/components/scripts/ingress/plugins/balancer.js b/charts/fsm/components/scripts/ingress/plugins/balancer.js
@@ -58,8 +58,11 @@
                 ),
                 upstreamSSLName: v?.upstream?.sslName || null,
                 upstreamSSLVerify: v?.upstream?.sslVerify || false,
-                cert: v?.upstream?.sslCert?.cert,
-                key: v?.upstream?.sslCert?.key
+                cert: v?.upstream?.sslCert?.cert || null,
+                key: v?.upstream?.sslCert?.key || null,
+                isTLS: Boolean(v?.upstream?.sslCert?.ca),
+                isMTLS: Boolean(v?.upstream?.sslCert?.cert) && Boolean(v?.upstream?.sslCert?.key) && Boolean(v?.upstream?.sslCert?.ca),
+                protocol: v?.upstream?.proto || 'HTTP'
               }]
             ))()
           )
@@ -74,8 +77,9 @@
     _serviceVerify: null,
     _serviceCertChain: null,
     _servicePrivateKey: null,
-    _connectTLS: null,
-    _mTLS: null,
+    _connectTLS: false,
+    _mTLS: false,
+    _isOutboundGRPC: false,
 
     _serviceCache: null,
     _targetCache: null,
@@ -101,6 +105,8 @@
 
   .import({
     __route: 'main',
+    __isInboundHTTP2: 'proto',
+    __isInboundGRPC: 'proto',
   })
 
   .pipeline()
@@ -136,22 +142,25 @@
           _serviceVerify = _service?.upstreamSSLVerify,
           _serviceCertChain = _service?.cert,
           _servicePrivateKey = _service?.key,
-          _target = _serviceCache.get(_service)
+          _target = _serviceCache.get(_service),
+          _connectTLS = _service?.isTLS,
+          _mTLS = _service?.isMTLS,
+          _isOutboundGRPC = __isInboundGRPC || _service?.protocol === 'GRPC'
         ),
-        _connectTLS = upstreamIssuingCAs?.length > 0,
-        _mTLS = _connectTLS && Boolean(_serviceCertChain) && Boolean(_servicePrivateKey),
 
         console.log("[balancer] _sourceIP", _sourceIP),
         console.log("[balancer] _connectTLS", _connectTLS),
-        console.log("[balancer] _target.id", (_target || {id : ''}).id)
+        console.log("[balancer] _mTLS", _mTLS),
+        console.log("[balancer] _target.id", (_target || {id : ''}).id),
+        console.log("[balancer] _isOutboundGRPC", _isOutboundGRPC)
       )
     )
     .branch(
-      () => Boolean(_target) && !Boolean(_connectTLS), (
-        $=>$.muxHTTP(() => _targetCache.get(_target)).to(
+      () => Boolean(_target) && !_connectTLS, (
+        $=>$.muxHTTP(() => _targetCache.get(_target), { version: () => _isOutboundGRPC ? 2 : 1 }).to(
           $=>$.connect(() => _target.id)
         )
-      ), () => Boolean(_target) && Boolean(_connectTLS), (
+      ), () => Boolean(_target) && _connectTLS && !_isOutboundGRPC, (
         $=>$.muxHTTP(() => _targetCache.get(_target)).to(
           $=>$.connectTLS({
             certificate: () => (_mTLS ? {
@@ -168,6 +177,24 @@
             $=>$.connect(() => _target.id)
           )
         )
+      ), () => Boolean(_target) && _connectTLS && _isOutboundGRPC, (
+        $=>$.muxHTTP(() => _targetCache.get(_target), { version: 2 }).to(
+          $=>$.connectTLS({
+            certificate: () => (_mTLS ? {
+              cert: new crypto.Certificate(_serviceCertChain),
+              key: new crypto.PrivateKey(_servicePrivateKey),
+            } : undefined),
+            trusted: upstreamIssuingCAs,
+            sni: () => _serviceSNI || undefined,
+            alpn: 'h2',
+            verify: (ok, cert) => (
+              !_serviceVerify && (ok = true),
+                ok
+            )
+          }).to(
+            $=>$.connect(() => _target.id)
+          )
+        )
       ), (
         $=>$.chain()
       )

diff --git a/charts/fsm/components/scripts/ingress/plugins/protocol.js b/charts/fsm/components/scripts/ingress/plugins/protocol.js
@@ -0,0 +1,63 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) since 2021,  flomesh.io Authors.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+((
+    config = pipy.solve('config.js'),
+    _detectEnabled = Boolean(config.detectProtocol),
+  ) => pipy()
+
+  .export('proto', {
+    __isInboundGRPC: false,
+    __isInboundHTTP2: false,
+  })
+
+  .pipeline()
+    .branch(
+      () => _detectEnabled, (
+        $=>$.detectProtocol(
+          proto => (
+            proto === 'HTTP2' && (
+              __isInboundHTTP2 = true
+            )
+          )
+        )
+      ), (
+        $=>$
+      )
+    )
+    .handleMessageStart(
+      msg => (
+        __isInboundHTTP2 && (
+          msg?.head?.headers?.['content-type'] === 'application/grpc' ||
+          msg?.head?.headers?.['content-type'] === 'application/grpc+prot' ||
+          msg?.head?.headers?.['content-type'] === 'application/grpc+json'
+        ) && (
+          __isInboundGRPC = true
+        ),
+
+        console.log('[proto] __isInboundHTTP2: ', __isInboundHTTP2),
+        console.log('[proto] __isInboundGRPC: ', __isInboundGRPC)
+      )
+    )
+    .chain()
+)()
diff --git a/charts/fsm/values.schema.json b/charts/fsm/values.schema.json
@@ -166,7 +166,7 @@
             },
             "tag": {
               "type": "string",
-              "default": "0.90.0-18",
+              "default": "0.90.0-123",
               "title": "The tag Schema"
             }
           }

diff --git a/charts/fsm/values.yaml b/charts/fsm/values.yaml
@@ -53,11 +53,11 @@ fsm:
 
   pipy:
     imageName: pipy
-    tag: 0.90.0-18-nonroot
+    tag: 0.90.0-123-nonroot
 
   pipyRepo:
     imageName: pipy-repo
-    tag: 0.90.0-18
+    tag: 0.90.0-123
 
   waitForIt:
     imageName: wait-for-it

diff --git a/dockerfiles/ingress-pipy/Dockerfile b/dockerfiles/ingress-pipy/Dockerfile
@@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH make build/ingress-pipy
 
 # Build the final image
-FROM flomesh/pipy:0.90.0-18-$DISTROLESS_TAG
+FROM flomesh/pipy:0.90.0-123-$DISTROLESS_TAG
 WORKDIR /
 COPY --from=builder /workspace/bin/ingress-pipy .
 

diff --git a/pkg/cache/ingress.go b/pkg/cache/ingress.go
@@ -135,6 +135,10 @@ func (info BaseIngressInfo) TrustedCA() *route.CertificateSpec {
 	return info.trustedCA
 }
 
+func (info BaseIngressInfo) Protocol() string {
+	return info.upstream.Protocol
+}
+
 type IngressMap map[RouteKey]Route
 
 type RouteKey struct {
@@ -544,6 +548,18 @@ func (ict *IngressChangeTracker) enrichIngressInfo(rule *networkingv1.IngressRul
 		}
 	}
 
+	// Backend Protocol
+	backendProtocol := strings.ToUpper(ing.Annotations[ingresspipy.PipyIngressAnnotationBackendProtocol])
+	if info.upstream == nil {
+		info.upstream = &route.UpstreamSpec{}
+	}
+	switch backendProtocol {
+	case "GRPC":
+		info.upstream.Protocol = "GRPC"
+		//default:
+		//    info.upstream.Protocol = "HTTP"
+	}
+
 	return info
 }
 

diff --git a/pkg/cache/local_cache.go b/pkg/cache/local_cache.go
@@ -47,6 +47,7 @@ import (
 	"k8s.io/client-go/tools/events"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/util/async"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -305,6 +306,7 @@ func (c *LocalCache) buildIngressConfig() routepkg.IngressData {
 				Sticky:   route.SessionSticky(),
 				Balancer: route.LBType(),
 				Upstream: &routepkg.UpstreamSpec{
+					Protocol:  strings.ToUpper(route.Protocol()),
 					SSLName:   route.UpstreamSSLName(),
 					SSLVerify: route.UpstreamSSLVerify(),
 					SSLCert:   route.UpstreamSSLCert(),

diff --git a/pkg/cache/types.go b/pkg/cache/types.go
@@ -51,6 +51,7 @@ type Route interface {
 	VerifyClient() bool
 	VerifyDepth() int
 	TrustedCA() *route.CertificateSpec
+	Protocol() string
 }
 
 type ServicePortName struct {

diff --git a/pkg/ingress/commons.go b/pkg/ingress/commons.go
@@ -53,4 +53,5 @@ const (
 	PipyIngressAnnotationTLSVerifyClient    = PipyIngressAnnotationPrefix + "/tls-verify-client"
 	PipyIngressAnnotationTLSVerifyDepth     = PipyIngressAnnotationPrefix + "/tls-verify-depth"
 	PipyIngressAnnotationTLSTrustedCASecret = PipyIngressAnnotationPrefix + "/tls-trusted-ca-secret"
+	PipyIngressAnnotationBackendProtocol    = PipyIngressAnnotationPrefix + "/upstream-protocol"
 )
diff --git a/pkg/route/types.go b/pkg/route/types.go
@@ -24,21 +24,6 @@
 
 package route
 
-//type RouteBase struct {
-//	// Region,
-//	Region string `json:"region"`
-//	// Zone,
-//	Zone string `json:"zone"`
-//	// Group,
-//	Group string `json:"group"`
-//	// Cluster,
-//	Cluster string `json:"cluster"`
-//
-//	GatewayHost string `json:"gatewayHost"`
-//	GatewayIP   net.IP `json:"gatewayIP"`
-//	GatewayPort int32  `json:"gatewayPort"`
-//}
-
 type IngressData struct {
 	//RouteBase `json:",inline"`
 	// Hash
@@ -67,6 +52,7 @@ type BalancerSpec struct {
 }
 
 type UpstreamSpec struct {
+	Protocol  string             `json:"proto,omitempty"`
 	SSLName   string             `json:"sslName,omitempty"`
 	SSLCert   *CertificateSpec   `json:"sslCert,omitempty"`
 	SSLVerify bool               `json:"sslVerify,omitempty"`
@@ -83,8 +69,8 @@ type TLSSpec struct {
 }
 
 type CertificateSpec struct {
-	Cert string `json:"cert"`
-	Key  string `json:"key"`
+	Cert string `json:"cert,omitempty"`
+	Key  string `json:"key,omitempty"`
 	CA   string `json:"ca,omitempty"`
 }