Update Parser to Postgres 13, use new protobuf-generated structs

This is a breaking change in the API, but necessary in order to
significantly improve the performance of parsing a query into Go structs,
as well as allowing future bi-directional passing of parse trees between
Go and C, such as for a future addition of a deparser.

.gitignore

@@ -1,4 +1,5 @@
 tmp/*
+bin/
 
 *.o
 *.a

Makefile

@@ -17,7 +17,7 @@ benchmark:
 
 # --- Below only needed for releasing new versions
 
-LIB_PG_QUERY_TAG = 10-1.0.5
+LIB_PG_QUERY_TAG = 13-latest-develop
 
 root_dir := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 LIB_TMPDIR = $(root_dir)/tmp
@@ -37,16 +37,32 @@ update_source: $(LIBDIR)
 	rm -fr parser/include
 	# Reduce everything down to one directory
 	cp -a $(LIBDIR)/src/* parser/
+	rm parser/pg_query_outfuncs_protobuf_cpp.cc
 	mv parser/postgres/* parser/
 	rmdir parser/postgres
 	cp -a $(LIBDIR)/pg_query.h parser/include
-	# Make sure every .c file in the top-level directory is its own translation unit
+	# Make sure every .c and .cc file in the top-level directory is its own translation unit
 	mv parser/*{_conds,_defs,_helper}.c parser/include
+	# Protobuf definitions
+	mkdir -p $(PWD)/bin
+	GOBIN=$(PWD)/bin go install github.com/golang/protobuf/protoc-gen-go
+	PATH=$(PWD)/bin:$(PATH) protoc --proto_path=$(LIBDIR)/protobuf --go_out=. $(LIBDIR)/protobuf/pg_query.proto
+	mkdir -p parser/include/protobuf
+	cp -a $(LIBDIR)/protobuf/*.h parser/include/protobuf
+	cp -a $(LIBDIR)/protobuf/*.c parser/
+	# Protobuf library code
+	mkdir -p parser/include/protobuf-c
+	cp -a $(LIBDIR)/vendor/protobuf-c/*.h parser/include
+	cp -a $(LIBDIR)/vendor/protobuf-c/*.h parser/include/protobuf-c
+	cp -a $(LIBDIR)/vendor/protobuf-c/*.c parser/
+	# xxhash library code
+	mkdir -p parser/include/xxhash
+	cp -a $(LIBDIR)/vendor/xxhash/*.h parser/include
+	cp -a $(LIBDIR)/vendor/xxhash/*.h parser/include/xxhash
+	cp -a $(LIBDIR)/vendor/xxhash/*.c parser/
 	# Other support files
 	rm -fr testdata
 	cp -a $(LIBDIR)/testdata testdata
-	# Update nodes directory
-	ruby scripts/generate_nodes.rb
 
 clean:
 	-@ $(RM) -r $(LIB_TMPDIR)

benchmark_test.go

@@ -4,12 +4,13 @@ import (
 	"testing"
 
 	"github.com/lfittl/pg_query_go"
+	nodes "github.com/lfittl/pg_query_go/nodes"
 )
 
 // Prevent compiler optimizations by assigning all results to global variables
 var err error
 var resultStr string
-var resultTree pg_query.ParsetreeList
+var resultTree *nodes.Node
 
 func benchmarkParse(input string, b *testing.B) {
 	for i := 0; i < b.N; i++ {
@@ -19,33 +20,33 @@ func benchmarkParse(input string, b *testing.B) {
 			b.Errorf("Benchmark produced error %s\n\n", err)
 		}
 
-		if len(resultTree.Statements) == 0 {
+		/*if len(resultTree.Statements) == 0 {
 			b.Errorf("Benchmark produced empty result\n\n")
-		}
+		}*/
 	}
 }
 
 func benchmarkParseParallel(input string, b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
-		var tree pg_query.ParsetreeList
+		//var tree nodes.Node
 
 		for pb.Next() {
-			tree, err = pg_query.Parse(input)
+			_, err = pg_query.Parse(input)
 
 			if err != nil {
 				b.Errorf("Benchmark produced error %s\n\n", err)
 			}
 
-			if len(tree.Statements) == 0 {
+			/*if len(tree.Statements) == 0 {
 				b.Errorf("Benchmark produced empty result\n\n")
-			}
+			}*/
 		}
 	})
 }
 
 func benchmarkRawParse(input string, b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		resultStr, err = pg_query.ParseToJSON(input)
+		resultStr, err = pg_query.ParseToProtobuf(input)
 
 		if err != nil {
 			b.Errorf("Benchmark produced error %s\n\n", err)
@@ -62,7 +63,7 @@ func benchmarkRawParseParallel(input string, b *testing.B) {
 		var str string
 
 		for pb.Next() {
-			str, err = pg_query.ParseToJSON(input)
+			str, err = pg_query.ParseToProtobuf(input)
 
 			if err != nil {
 				b.Errorf("Benchmark produced error %s\n\n", err)
@@ -75,7 +76,7 @@ func benchmarkRawParseParallel(input string, b *testing.B) {
 	})
 }
 
-func benchmarkFingerprint(input string, b *testing.B) {
+/*func benchmarkFingerprint(input string, b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		resultTree, err = pg_query.Parse(input)
 		if err != nil {
@@ -88,7 +89,7 @@ func benchmarkFingerprint(input string, b *testing.B) {
 			b.Errorf("Benchmark produced empty result\n\n")
 		}
 	}
-}
+}*/
 
 func benchmarkFastFingerprint(input string, b *testing.B) {
 	for i := 0; i < b.N; i++ {
@@ -154,15 +155,15 @@ func BenchmarkRawParseCreateTableParallel(b *testing.B) {
 	benchmarkRawParseParallel("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
 }
 
-func BenchmarkFingerprintSelect1(b *testing.B) {
+/*func BenchmarkFingerprintSelect1(b *testing.B) {
 	benchmarkFingerprint("SELECT 1", b)
 }
 func BenchmarkFingerprintSelect2(b *testing.B) {
 	benchmarkFingerprint("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
 }
 func BenchmarkFingerprintCreateTable(b *testing.B) {
 	benchmarkFingerprint("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
-}
+}*/
 
 func BenchmarkFastFingerprintSelect1(b *testing.B) {
 	benchmarkFastFingerprint("SELECT 1", b)

fingerprint_test.go

@@ -1,5 +1,6 @@
 package pg_query_test
 
+/*
 import (
 	"encoding/json"
 	"fmt"
@@ -55,3 +56,4 @@ func TestFingerprint(t *testing.T) {
 
 	fmt.Printf("\n")
 }
+*/

go.mod

@@ -2,4 +2,9 @@ module github.com/lfittl/pg_query_go
 
 go 1.14
 
-require github.com/kr/pretty v0.2.1
+require (
+	github.com/golang/protobuf v1.4.2
+	github.com/google/go-cmp v0.5.1
+	github.com/kr/pretty v0.2.1
+	google.golang.org/protobuf v1.23.0
+)

go.sum

@@ -1,7 +1,29 @@
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=