[WIP] Develop merge

.github/workflows/tests.yml

@@ -13,18 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: ^1.18
+          go-version: ^1.22.6
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build
         run: go build -v ./
 
       - name: Store grip
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: gripBin
           path: grip
@@ -35,28 +35,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: ^1.18
+          go-version: ^1.22.6
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: run unit tests
         run: |
           go test ./test/... -config badger.yml
-
-
+
 
   badgerTest:
     needs: build
     name: Badger Conformance
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Badger Test
@@ -65,19 +64,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/badger.yml &
           sleep 5
           make test-conformance
-  
+
 
   pebbleTest:
     needs: build
     name: Pebble Conformance
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Pebble Test
@@ -86,19 +85,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/pebble.yml &
           sleep 5
           make test-conformance
-    
+
 
   mongoTest:
     needs: build
     name: Mongo Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Mongo Conformance
@@ -108,41 +107,41 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo.yml &
           sleep 5
           make test-conformance
-  
+
 
   mongoCoreTest:
     needs: build
     name: Mongo Core Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
-      - name: Mongo Conformance
+      - name: Mongo Core Conformance
         run: |
           chmod +x grip
           make start-mongo
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo-core-processor.yml &
           sleep 5
           make test-conformance
-  
+
 
   elasticTest:
     needs: build
     name: Elastic Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Elastic Conformance
@@ -153,19 +152,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/elastic.yml &
           sleep 5
           make test-conformance
-
 
-  portgresTest:
+
+  postgresTest:
     needs: build
-    name: Portgres Test
+    name: Postgres Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Postgres Conformance
@@ -176,21 +175,42 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/psql.yml &
           sleep 5
           python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
-
+
+
+  sqliteTest:
+    needs: build
+    name: Sqlite Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Python Dependencies for Conformance
+        run: pip install requests numpy
+      - name: Download grip
+        uses: actions/download-artifact@v4
+        with:
+          name: gripBin
+      - name: Sqlite Conformance
+        run: |
+          chmod +x grip
+          ./grip server --rpc-port 18202 --http-port 18201 --config ./test/sqlite.yml &
+          sleep 5
+          python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
+
 
   gripperTest:
     needs: build
     name: Gripper Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Update pip
         run: pip install --upgrade pip
       - name: Python Dependencies for Conformance
         run: pip install -U requests numpy grpcio-tools protobuf
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Gripper Conformance
@@ -210,11 +230,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy PyYAML
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Auth Test
@@ -224,30 +244,29 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/badger-auth.yml &
           sleep 5
           # simple auth
-          # run tests without credentials, should fail 
+          # run tests without credentials, should fail
           if make test-conformance
           then
             echo "ERROR: Conformance tests ran without credentials." ; exit 1
           else
-            echo "Got expected auth error" 
-          fi          
+            echo "Got expected auth error"
+          fi
           # run specialized role based tests
           make test-authorization ARGS="--grip_config_file_path test/badger-auth.yml"
-
-
-
+
+
 
     #gridsTest:
     #  needs: build
     #  name: GRIDs Conformance
     #  runs-on: ubuntu-latest
     #  steps:
     #  - name: Check out code
-    #    uses: actions/checkout@v2
+    #    uses: actions/checkout@v4
     #  - name: Python Dependencies for Conformance
     #    run: pip install requests numpy
     #  - name: Download grip
-    #    uses: actions/download-artifact@v2
+    #    uses: actions/download-artifact@v4
     #    with:
     #      name: gripBin
     #  - name: GRIDs unit tests

Makefile

@@ -41,7 +41,7 @@ proto:
 	  	--go_opt paths=source_relative \
 		--go-grpc_out ./ \
 		--go-grpc_opt paths=source_relative \
-		--grpc-gateway_out ./ \
+		--grpc-gateway_out allow_delete_body=true:./ \
 		--grpc-gateway_opt logtostderr=true \
 		--grpc-gateway_opt paths=source_relative \
 		--grpc-rest-direct_out . \
@@ -66,11 +66,12 @@ proto:
 
 proto-depends:
 	@git submodule update --init --recursive
-	@go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2.11.1
-	@go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.11.1
-	@go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
+	@go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@latest
+	@go install github.com/akuity/grpc-gateway-client/protoc-gen-grpc-gateway-client
+	@go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@latest
+	@go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
 	@go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
-	@go install github.com/ckaznocha/protoc-gen-lint@v0.2.4
+	@go install github.com/ckaznocha/protoc-gen-lint@latest
 	@go install github.com/bmeg/protoc-gen-grpc-rest-direct@latest
 	@go install github.com/ckaznocha/protoc-gen-lint@latest
 
@@ -91,7 +92,7 @@ lint:
 	flake8 gripql/python/ conformance/
 
 lint-depends:
-	go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.35.2
+	go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1
 	go install golang.org/x/tools/cmd/goimports
 
 # ---------------------
@@ -129,7 +130,7 @@ test-authorization:
 # ---------------------
 start-mongo:
 	@docker rm -f grip-mongodb-test > /dev/null 2>&1 || echo
-	docker run -d --name grip-mongodb-test -p 27017:27017 docker.io/mongo:3.6.4 > /dev/null
+	docker run -d --name grip-mongodb-test -p 27017:27017 mongo:7.0.13-rc0-jammy > /dev/null
 
 start-elastic:
 	@docker rm -f grip-es-test > /dev/null 2>&1 || echo

accounts/interface.go

@@ -36,6 +36,7 @@ var MethodMap = map[string]Operation{
 	"/gripql.Edit/AddVertex":    Write,
 	"/gripql.Edit/AddEdge":      Write,
 	"/gripql.Edit/BulkAdd":      Write,
+	"/gripql.Edit/BulkDelete":   Write,
 	"/gripql.Edit/AddGraph":     Write,
 	"/gripql.Edit/DeleteGraph":  Write,
 	"/gripql.Edit/DeleteVertex": Write,

accounts/util.go

@@ -141,6 +141,8 @@ func streamAuthInterceptor(auth Authenticate, access Access) grpc.StreamServerIn
 				//stream URL formatting, each write request can
 				//reference a different graph
 				return handler(srv, &BulkWriteFilter{ss, user, access})
+			} else if info.FullMethod == "/gripql.Edit/BulkDelete" {
+				return handler(srv, &BulkWriteFilter{ss, user, access})
 			} else {
 				log.Errorf("Unknown input streaming op %#v!!!", info)
 				return handler(srv, ss)
@@ -190,6 +192,9 @@ func getUnaryRequestGraph(req interface{}, info *grpc.UnaryServerInfo) (string,
 	case "/gripql.Edit/SampleSchema":
 		o := req.(*gripql.GraphID)
 		return o.Graph, nil
+	case "/gripql.Edit/BulkDelete":
+		o := req.(*gripql.DeleteData)
+		return o.Graph, nil
 	case "/gripql.Configure/StartPlugin", "/gripql.Configure/ListPlugins", "/gripql.Configure/ListDrivers":
 		return "*", nil //these operations effect all graphs
 	}

benchmark/kv-query-profile/main.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"os"
 	"runtime/pprof"
-	"strings"
 	"time"
 
 	"github.com/bmeg/grip/engine/pipeline"
@@ -17,7 +16,7 @@ import (
 	"github.com/bmeg/grip/kvgraph"
 	"github.com/bmeg/grip/kvi"
 	"github.com/dop251/goja"
-	"github.com/golang/protobuf/jsonpb"
+	"google.golang.org/protobuf/encoding/protojson"
 
 	gripqljs "github.com/bmeg/grip/gripql/javascript"
 
@@ -76,7 +75,7 @@ func main() {
 	}
 
 	query := gripql.GraphQuery{}
-	err = jsonpb.Unmarshal(strings.NewReader(string(queryJSON)), &query)
+	err = protojson.Unmarshal(queryJSON, &query)
 	if err != nil {
 		log.Printf("%s", err)
 		return