Skip to content

Commit

Permalink
Merge pull request #231 from bmeg/dig-driver
Browse files Browse the repository at this point in the history
GRIPPER plugin driver
  • Loading branch information
kellrott authored Dec 15, 2020
2 parents e86b5fc + 343496d commit 64c4ae9
Show file tree
Hide file tree
Showing 99 changed files with 8,793 additions and 2,570 deletions.
10 changes: 9 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ install:
- sudo pip install -U numpy
- sudo pip install -U requests
- sudo pip install -U flake8
- sudo pip install -U grpcio-tools

jobs:
include:
Expand Down Expand Up @@ -107,7 +108,14 @@ jobs:
- go test ./test -config psql.yml
env:
- n=postgres

- script:
- make start-gripper-test
- sleep 5
- grip server --rpc-port 18202 --http-port 18201 --config ./gripper/test-graph/config.yaml &
- sleep 5
- python conformance/run_conformance.py http://localhost:18201 --readOnly swapi
env:
- n=gripper

notifications:
email: false
10 changes: 10 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ proto:
-I ./ \
--go_out=. \
index.proto
@cd gripper/ && protoc \
-I ./ \
-I ../googleapis/ \
--go_out=Mgoogle/protobuf/struct.proto=github.com/golang/protobuf/ptypes/struct,plugins=grpc:. \
gripper.proto


proto-depends:
@git submodule update --init --recursive
Expand Down Expand Up @@ -72,6 +78,7 @@ lint:

lint-depends:
go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.22.2
go install golang.org/x/tools/cmd/goimports

# ---------------------
# Release / Snapshot
Expand Down Expand Up @@ -119,6 +126,9 @@ start-mysql:
@docker rm -f grip-mysql-test > /dev/null 2>&1 || echo
docker run -d --name grip-mysql-test -p 13306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:8.0.11 --default-authentication-plugin=mysql_native_password > /dev/null

start-gripper-test:
@cd ./gripper/test-graph && ./table-server.py swapi/table.map &

# ---------------------
# Website
# ---------------------
Expand Down
126 changes: 126 additions & 0 deletions cmd/dig/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package dig

import (
"context"
"fmt"
"log"
"os"

"github.com/bmeg/grip/gripper"

"github.com/bmeg/grip/gdbi"
"github.com/golang/protobuf/jsonpb"

"github.com/spf13/cobra"

"encoding/json"
"strings"

"github.com/bmeg/grip/gripql"
gripqljs "github.com/bmeg/grip/gripql/javascript"
"github.com/bmeg/grip/jsengine/underscore"
"github.com/dop251/goja"

"github.com/bmeg/grip/engine/pipeline"
"github.com/bmeg/grip/util"
)

var idxName string = "table.db"

func ParseQuery(queryString string) (*gripql.GraphQuery, error) {
vm := goja.New()

us, err := underscore.Asset("underscore.js")
if err != nil {
return nil, fmt.Errorf("failed to load underscore.js")
}
if _, err := vm.RunString(string(us)); err != nil {
return nil, err
}

gripqlString, err := gripqljs.Asset("gripql.js")
if err != nil {
return nil, fmt.Errorf("failed to load gripql.js")
}
if _, err := vm.RunString(string(gripqlString)); err != nil {
return nil, err
}

val, err := vm.RunString(queryString)
if err != nil {
return nil, err
}

queryJSON, err := json.Marshal(val)
if err != nil {
return nil, err
}

query := gripql.GraphQuery{}
err = jsonpb.Unmarshal(strings.NewReader(string(queryJSON)), &query)
if err != nil {
return nil, err
}
return &query, nil
}

func Query(graph gdbi.GraphInterface, query *gripql.GraphQuery) error {
marsh := jsonpb.Marshaler{}

p, err := graph.Compiler().Compile(query.Query)
if err != nil {
return err
}
workdir := "./test.workdir." + util.RandomString(6)
defer os.RemoveAll(workdir)
res := pipeline.Run(context.Background(), p, workdir)

for row := range res {
rowString, _ := marsh.MarshalToString(row)
fmt.Printf("%s\n", rowString)
}
return nil
}

// Cmd command line declaration
var Cmd = &cobra.Command{
Use: "dig <config> <query>",
Short: "Do a single query using the gripper driver",
Long: ``,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {

configFile := args[0]
queryString := args[1]

config := gripper.Config{Graphs: map[string]string{"main": configFile}}

gdb, err := gripper.NewGDB(config, "./")
if err != nil {
log.Printf("Error loading Graph: %s", err)
return err
}

graph, err := gdb.Graph("main")
if err != nil {
log.Printf("%s", err)
return err
}

query, err := ParseQuery(queryString)
if err != nil {
log.Printf("%s", err)
return err
}
log.Printf("Query: %#v", query)
Query(graph, query)

gdb.Close()
return nil
},
}

func init() {
flags := Cmd.Flags()
flags.StringVar(&idxName, "db", idxName, "Path to index db")
}
2 changes: 2 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"

"github.com/bmeg/grip/cmd/create"
"github.com/bmeg/grip/cmd/dig"
"github.com/bmeg/grip/cmd/drop"
"github.com/bmeg/grip/cmd/dump"
"github.com/bmeg/grip/cmd/info"
Expand Down Expand Up @@ -48,6 +49,7 @@ func init() {
RootCmd.PersistentFlags().BoolVar(&enableProf, "pprof", enableProf, "enable pprof on port 6060")
RootCmd.AddCommand(create.Cmd)
RootCmd.AddCommand(drop.Cmd)
RootCmd.AddCommand(dig.Cmd)
RootCmd.AddCommand(dump.Cmd)
RootCmd.AddCommand(genBashCompletionCmd)
RootCmd.AddCommand(info.Cmd)
Expand Down
4 changes: 4 additions & 0 deletions cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
esql "github.com/bmeg/grip/existing-sql"
"github.com/bmeg/grip/gdbi"
"github.com/bmeg/grip/grids"
"github.com/bmeg/grip/gripper"
"github.com/bmeg/grip/gripql"
"github.com/bmeg/grip/kvgraph"
_ "github.com/bmeg/grip/kvi/badgerdb" // import so badger will register itself
Expand Down Expand Up @@ -59,6 +60,9 @@ func Run(conf *config.Config, schemas map[string]*gripql.Graph) error {
case "existing-sql":
db, err = esql.NewGraphDB(conf.ExistingSQL)

case "gripper":
db, err = gripper.NewGDB(conf.Gripper, configFile)

default:
err = fmt.Errorf("unknown database: %s", dbname)
}
Expand Down
15 changes: 12 additions & 3 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/bmeg/grip/elastic"
esql "github.com/bmeg/grip/existing-sql"
"github.com/bmeg/grip/gripper"
"github.com/bmeg/grip/log"
"github.com/bmeg/grip/mongo"
"github.com/bmeg/grip/psql"
Expand All @@ -37,6 +38,7 @@ type Config struct {
MongoDB mongo.Config
PSQL psql.Config
ExistingSQL esql.Config
Gripper gripper.Config
Logger log.Logger
}

Expand Down Expand Up @@ -99,7 +101,7 @@ func ParseConfig(raw []byte, conf *Config) error {
if err != nil {
return err
}
err = CheckForUnknownKeys(j, conf)
err = CheckForUnknownKeys(j, conf, []string{"Gripper.Graphs."})
if err != nil {
return err
}
Expand Down Expand Up @@ -183,7 +185,7 @@ func GetKeys(obj interface{}) []string {

// CheckForUnknownKeys takes a json byte array and checks that all keys are fields
// in the reference object
func CheckForUnknownKeys(jsonStr []byte, obj interface{}) error {
func CheckForUnknownKeys(jsonStr []byte, obj interface{}, exclude []string) error {
knownMap := make(map[string]interface{})
known := GetKeys(obj)
for _, k := range known {
Expand All @@ -200,7 +202,14 @@ func CheckForUnknownKeys(jsonStr []byte, obj interface{}) error {
all := GetKeys(anon)
for _, k := range all {
if _, found := knownMap[k]; !found {
unknown = append(unknown, k)
for _, e := range exclude {
if strings.HasPrefix(k, e) {
found = true
}
}
if !found {
unknown = append(unknown, k)
}
}
}

Expand Down
Loading

0 comments on commit 64c4ae9

Please sign in to comment.