[WIP] GRIP engine embedded in a python library

.github/workflows/tests.yml

@@ -233,10 +233,28 @@ jobs:
           fi          
           # run specialized role based tests
           make test-authorization ARGS="--grip_config_file_path test/badger-auth.yml"
-
-
-
 
+  pygripTest:
+    needs: build
+    name: PyGrip UnitTest
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Python Dependencies for Conformance
+        run: pip install requests numpy PyYAML
+      - name: install gripql
+        run: |
+          cd gripql/python
+          python setup.py install --user
+      - name: install pygrip
+        run: |
+          python setup.py install --user
+      - name: unit tests
+        run: |
+          cd test
+          python -m unittest discover -s ./pygrip_test
+
     #gridsTest:
     #  needs: build
     #  name: GRIDs Conformance

gripql/python/gripql/__init__.py

@@ -36,4 +36,4 @@
     count
 ]
 
-__version__ = "0.7.1"
+__version__ = "0.8.0"

gripql/python/gripql/graph.py

@@ -179,6 +179,12 @@ def query(self):
         """
         return Query(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file)
 
+    def V(self, *args):
+        """
+        Create a vertex query handle.
+        """
+        return Query(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file).V(*args)
+
     def resume(self, job_id):
         """
         Create a query handle.

gripql/python/gripql/query.py

@@ -36,16 +36,12 @@ def _wrap_dict_value(value):
     return _wrap_value(value, dict)
 
 
-class Query(BaseConnection):
-    def __init__(self, url, graph, user=None, password=None, token=None, credential_file=None, resume=None):
-        super(Query, self).__init__(url, user, password, token, credential_file)
-        self.url = self.base_url + "/v1/graph/" + graph + "/query"
-        self.graph = graph
+class QueryBuilder:
+    def __init__(self):
         self.query = []
-        self.resume = resume
 
     def __append(self, part):
-        q = self.__class__(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file, self.resume)
+        q = self._builder()
         q.query = self.query[:]
         q.query.append(part)
         return q
@@ -343,6 +339,20 @@ def to_dict(self):
         """
         return {"query": self.query}
 
+
+
+class Query(BaseConnection, QueryBuilder):
+    def __init__(self, url, graph, user=None, password=None, token=None, credential_file=None, resume=None):
+        super(Query, self).__init__(url, user, password, token, credential_file)
+        super(QueryBuilder, self).__init__()
+        self.url = self.base_url + "/v1/graph/" + graph + "/query"
+        self.graph = graph
+        self.query = []
+        self.resume = resume
+
+    def _builder(self):
+        return self.__class__(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file, self.resume)
+
     def __iter__(self):
         return self.__stream()
 

kvi/leveldb/memdb.go

@@ -0,0 +1,168 @@
+package leveldb
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/bmeg/grip/kvi"
+	"github.com/bmeg/grip/log"
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+)
+
+var mem_loaded = kvi.AddKVDriver("memdb", NewMemKVInterface)
+
+type LevelMemKV struct {
+	db *memdb.DB
+}
+
+// NewKVInterface creates new LevelDB backed KVInterface at `path`
+func NewMemKVInterface(path string, opts kvi.Options) (kvi.KVInterface, error) {
+	log.Info("Starting In-Memory LevelDB")
+	db := memdb.New(comparer.DefaultComparer, 1000)
+	return &LevelMemKV{db: db}, nil
+}
+
+// BulkWrite implements kvi.KVInterface.
+func (l *LevelMemKV) BulkWrite(u func(bl kvi.KVBulkWrite) error) error {
+	ktx := &memIterator{l.db, nil, true, nil, nil}
+	return u(ktx)
+}
+
+// Close implements kvi.KVInterface.
+func (l *LevelMemKV) Close() error {
+	return nil
+}
+
+// Delete implements kvi.KVInterface.
+func (l *LevelMemKV) Delete(key []byte) error {
+	return l.db.Delete(key)
+}
+
+// DeletePrefix implements kvi.KVInterface.
+func (l *LevelMemKV) DeletePrefix(prefix []byte) error {
+	deleteBlockSize := 10000
+	for found := true; found; {
+		found = false
+		wb := make([][]byte, 0, deleteBlockSize)
+		it := l.db.NewIterator(nil)
+		for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix) && len(wb) < deleteBlockSize-1; it.Next() {
+			wb = append(wb, copyBytes(it.Key()))
+		}
+		it.Release()
+		for _, i := range wb {
+			l.db.Delete(i)
+			found = true
+		}
+	}
+	return nil
+
+}
+
+// Get implements kvi.KVInterface.
+func (l *LevelMemKV) Get(key []byte) ([]byte, error) {
+	return l.db.Get(key)
+}
+
+// HasKey implements kvi.KVInterface.
+func (l *LevelMemKV) HasKey(key []byte) bool {
+	_, err := l.db.Get(key)
+	return err == nil
+}
+
+// Set implements kvi.KVInterface.
+func (l *LevelMemKV) Set(key []byte, value []byte) error {
+	return l.db.Put(key, value)
+}
+
+// Update implements kvi.KVInterface.
+func (l *LevelMemKV) Update(func(tx kvi.KVTransaction) error) error {
+	panic("unimplemented")
+}
+
+// View implements kvi.KVInterface.
+func (l *LevelMemKV) View(u func(it kvi.KVIterator) error) error {
+	it := l.db.NewIterator(nil)
+	defer it.Release()
+	lit := memIterator{l.db, it, true, nil, nil}
+	return u(&lit)
+}
+
+type memIterator struct {
+	db      *memdb.DB
+	it      iterator.Iterator
+	forward bool
+	key     []byte
+	value   []byte
+}
+
+// Get retrieves the value of key `id`
+func (lit *memIterator) Get(id []byte) ([]byte, error) {
+	return lit.db.Get(id)
+}
+
+func (lit *memIterator) Set(key, val []byte) error {
+	return lit.db.Put(key, val)
+}
+
+// Key returns the key the iterator is currently pointed at
+func (lit *memIterator) Key() []byte {
+	return lit.key
+}
+
+// Value returns the valud of the iterator is currently pointed at
+func (lit *memIterator) Value() ([]byte, error) {
+	return lit.value, nil
+}
+
+// Next move the iterator to the next key
+func (lit *memIterator) Next() error {
+	var more bool
+	if lit.forward {
+		more = lit.it.Next()
+	} else {
+		more = lit.it.Prev()
+	}
+	if !more {
+		lit.key = nil
+		lit.value = nil
+		return fmt.Errorf("Invalid")
+	}
+	lit.key = copyBytes(lit.it.Key())
+	lit.value = copyBytes(lit.it.Value())
+	return nil
+}
+
+func (lit *memIterator) Seek(id []byte) error {
+	lit.forward = true
+	if lit.it.Seek(id) {
+		lit.key = copyBytes(lit.it.Key())
+		lit.value = copyBytes(lit.it.Value())
+		return nil
+	}
+	return fmt.Errorf("Invalid")
+}
+
+func (lit *memIterator) SeekReverse(id []byte) error {
+	lit.forward = false
+	if lit.it.Seek(id) {
+		//Level iterator will land on the first value above the request
+		//if we're there, move once to get below start request
+		if bytes.Compare(id, lit.it.Key()) < 0 {
+			lit.it.Prev()
+		}
+		lit.key = copyBytes(lit.it.Key())
+		lit.value = copyBytes(lit.it.Value())
+		return nil
+	}
+	return fmt.Errorf("Invalid")
+}
+
+// Valid returns true if iterator is still in valid location
+func (lit *memIterator) Valid() bool {
+	if lit.key == nil || lit.value == nil {
+		return false
+	}
+	return true
+}

pygrip/Makefile

@@ -0,0 +1,4 @@
+
+
+pygrip.so: wrapper.go
+	go build -o pygrip.so -buildmode=c-shared wrapper.go

pygrip/__init__.py

@@ -0,0 +1,74 @@
+
+
+from __future__ import print_function
+from ctypes import *
+from ctypes.util import find_library
+import os, inspect, sysconfig
+import random, string
+import json
+from gripql.query import QueryBuilder
+
+cwd = os.getcwd()
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+#print("frame: %s" % (inspect.getfile(inspect.currentframe())))
+#print("cd to %s" % (currentdir))
+os.chdir(currentdir)
+_lib = cdll.LoadLibrary("./_pygrip" + sysconfig.get_config_vars()["EXT_SUFFIX"])
+os.chdir(cwd)
+
+_lib.ReaderNext.restype = c_char_p
+
+class GoString(Structure):
+    _fields_ = [("p", c_char_p), ("n", c_longlong)]
+
+def NewMemServer():
+    return GraphDBWrapper( _lib.NewMemServer() )
+
+def getGoString(s):
+    return GoString(bytes(s, encoding="raw_unicode_escape"), len(s))
+
+def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
+    return ''.join(random.choice(chars) for _ in range(size))
+
+class QueryWrapper(QueryBuilder):
+    def __init__(self, wrapper):
+        super(QueryBuilder, self).__init__()
+        self.query = []
+        self.wrapper = wrapper
+
+    def __iter__(self):
+        jquery = json.dumps({ "graph" : "default", "query" : self.query })
+        reader = _lib.Query( self.wrapper._handle, getGoString(jquery) )
+        while not _lib.ReaderDone(reader):
+            j = _lib.ReaderNext(reader)
+            yield json.loads(j)
+
+    def _builder(self):
+        return QueryWrapper(self.wrapper)
+
+class GraphDBWrapper:
+    def __init__(self, handle) -> None:
+        self._handle = handle
+
+    def addVertex(self, gid, label, data={}):
+        """
+        Add vertex to a graph.
+        """
+        _lib.AddVertex(self._handle, getGoString(gid), getGoString(label), 
+                       getGoString(json.dumps(data)))
+
+    def addEdge(self, src, dst, label, data={}, gid=None):
+        """
+        Add edge to a graph.
+        """
+        if gid is None:
+            gid = id_generator(10)
+
+        _lib.AddEdge(self._handle, getGoString(gid), 
+                     getGoString(src), getGoString(dst), getGoString(label), 
+                     getGoString(json.dumps(data)))
+
+
+
+    def V(self, *ids):
+        return QueryWrapper(self).V(*ids)