Adds Python SDK to Buf Gen (#2827)

.github/workflows/artifact-release.yml

@@ -572,6 +572,61 @@ jobs:
           DISABLE_CHANGELOG: false
           NEOSYNC_BASE_URL: ${{ vars.DEFAULT_NEOSYNC_ENDPOINT }}
 
+  py-sdk-build:
+    name: Python SDK Build
+    runs-on: ubuntu-latest
+    #  for now, only build when we cut a new tag
+    if: startsWith(github.ref, 'refs/tags/')
+    defaults:
+      run:
+        working-directory: python
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Package Version
+        id: package_version
+        run: |
+          V_PREFIX="v"
+          VERSION="${{ github.ref_name }}"
+          TRIMMED_VERSION="${VERSION/#$V_PREFIX}"
+          echo "$TRIMMED_VERSION"
+          echo "PACKAGE_VERSION=$TRIMMED_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install pypa/build
+        run: |
+          python -m pip install build
+          python -m build
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.package_version.outputs.PACKAGE_VERSION }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: py-package
+          path: dist/
+
+  py-sdk-publish:
+    name: Python SDK Publish
+    runs-on: ubuntu-latest
+    needs:
+      - py-sdk-build
+    permissions:
+      id-token: write
+    defaults:
+      run:
+        working-directory: python
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: py-package
+          path: dist/
+      - name: Publish release distributions to PyPi
+        uses: pypa/gh-action-pypi-publish@release/v1
+
   ts-sdk:
     name: TypeScript SDK Release
     runs-on: ubuntu-latest

.github/workflows/buf.yml

@@ -2,9 +2,9 @@ on:
   pull_request:
     paths:
       - backend/protos/**
-      - backend/BUF_VERSION
-      - backend/buf.gen.yaml
-      - backend/buf.work.yaml
+      - BUF_VERSION
+      - buf.gen.yaml
+      - buf.work.yaml
 
 name: Buf
 
@@ -16,9 +16,6 @@ jobs:
   buf:
     name: buf
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: backend
     steps:
       - uses: actions/checkout@v4
 
@@ -33,6 +30,5 @@ jobs:
         with:
           version: ${{ steps.buf-version.outputs.version }}
           token: ${{ secrets.BUF_TOKEN }}
-          input: backend
           push: false # we don't want to push to the BSR
           push_disable_create: true

.github/workflows/sqlc.yaml

@@ -1,7 +1,7 @@
 on:
   pull_request:
     paths:
-      - backend/sqlc.yaml
+      - sqlc.yaml
       - backend/sql/**
       - SQLC_VERSION
 
@@ -10,9 +10,6 @@ jobs:
   sqlcdiff:
     name: sqlc-diff
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: backend
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -31,9 +28,6 @@ jobs:
   sqlcvet:
     name: sqlc-vet
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: backend
     services:
       postgres:
         image: "postgres:15"
@@ -53,7 +47,7 @@ jobs:
       - name: Patch sqlc.yaml with db-prepare lint
         uses: mikefarah/[email protected]
         with:
-          cmd: sh ./backend/scripts/sqlc-prepare-patch.sh postgresql://postgres:postgres@localhost:5432/nucleus ./backend/sqlc.yaml
+          cmd: sh ./backend/scripts/sqlc-prepare-patch.sh postgresql://postgres:postgres@localhost:5432/nucleus ./sqlc.yaml
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -62,12 +56,12 @@ jobs:
           cache-dependency-path: go.sum
 
       - name: Install golang-migrate
-        run: make install-go-tools
+        run: cd backend && make install-go-tools
 
       - name: Run migrations on database
         env:
           PG_PASSWORD: postgres
-        run: ./scripts/migrate.sh up
+        run: ./backend/scripts/migrate.sh up
 
       - name: Retrieve SQLC Version
         run: |

.gitignore

@@ -33,3 +33,6 @@ frontend/out
 env/
 
 tmp
+venv/
+venv2/
+python/dist/

BUF_VERSION

@@ -0,0 +1 @@
+1.45.0

backend/Makefile

@@ -24,7 +24,7 @@ run: ## Runs the project
 	./bin/mgmt serve connect
 
 gen: ## Runs the gen script
-	./scripts/gen.sh
+	@cd .. && ./scripts/backend-gen.sh
 
 clean: ## Cleans the bin directory
 	rm -rf bin

backend/gen/openapi/mgmt/v1alpha1/anonymization.openapi.yaml

@@ -162,6 +162,11 @@ components:
           description: |-
             Flag to indicate whether to stop processing when an error occurs
              true: stops on first error encounter
+        accountId:
+          type: string
+          title: account_id
+          format: uuid
+          description: The unique account identifier
       title: AnonymizeManyRequest
       additionalProperties: false
     mgmt.v1alpha1.AnonymizeManyResponse:
@@ -214,6 +219,11 @@ components:
             - title: default_transformers
               description: Optional default transformations for any unmapped keys
             - $ref: '#/components/schemas/mgmt.v1alpha1.DefaultTransformersConfig'
+        accountId:
+          type: string
+          title: account_id
+          format: uuid
+          description: The unique account identifier
       title: AnonymizeSingleRequest
       additionalProperties: false
     mgmt.v1alpha1.AnonymizeSingleResponse:

backend/buf.gen.yaml → buf.gen.yaml

@@ -11,34 +11,41 @@ managed:
       value: github.com/nucleuscloud/neosync/backend/gen/go/protos
 plugins:
   - remote: buf.build/protocolbuffers/go:v1.35.1
-    out: gen/go/protos
+    out: backend/gen/go/protos
     opt: paths=source_relative
   - remote: buf.build/connectrpc/go:v1.17.0
-    out: gen/go/protos
+    out: backend/gen/go/protos
     opt: paths=source_relative
 
   - remote: buf.build/bufbuild/es:v1.10.0
-    out: gen/es/protos
+    out: frontend/packages/sdk/src/client
     opt:
       - target=ts
       - import_extension=.js
   - remote: buf.build/connectrpc/es:v1.5.0
-    out: gen/es/protos
+    out: frontend/packages/sdk/src/client
     opt:
       - target=ts
       - import_extension=.js
-
-  - remote: buf.build/community/pseudomuto-doc:v1.5.1
-    out: gen/docs
-    opt: json,proto_docs.json
-
   - remote: buf.build/connectrpc/query-es:v1.4.2
-    out: gen/es/protos
+    out: frontend/packages/sdk/src/client
     opt:
       - target=ts
       - import_extension=.js
 
+  - remote: buf.build/community/pseudomuto-doc:v1.5.1
+    out: docs/protos
+    opt: json,proto_docs.json
+
   - local: protoc-gen-connect-openapi
-    out: gen/openapi
+    out: backend/gen/openapi
     opt:
       - content-types=json
+
+  - remote: buf.build/protocolbuffers/python:v28.2 # generates messages and enums
+    out: python/src/neosync
+    include_imports: true
+  - remote: buf.build/protocolbuffers/pyi:v28.2 # generates type hints
+    out: python/src/neosync
+  - remote: buf.build/grpc/python:v1.67.0 # generates grpc client
+    out: python/src/neosync

buf.lock

@@ -0,0 +1,9 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+  - name: buf.build/bufbuild/protovalidate
+    commit: 5a7b106cbb87462d9a8c9ffecdbd2e38
+    digest: b5:0f2dc6c9453e9cc9e9f36807aaa5f94022e837d91fef4dcaeed79a35c0843cc64eba28ff077aab24da3b2cb12639ad256246f9f9a36c033b99d5754b19996b7e
+  - name: buf.build/googleapis/googleapis
+    commit: e7f8d366f5264595bcc4cd4139af9973
+    digest: b5:0cd69a689ee320ed815663d57d1bc3a1d6823224a7a717d46fee3a68197c25a6f5f932c0b0e49f8370c70c247a6635969a6a54af5345cafd51e0667298768aca

backend/buf.yaml → buf.yaml

@@ -1,12 +1,12 @@
 version: v2
 modules:
-  - path: protos
+  - path: backend/protos
 deps:
   - buf.build/bufbuild/protovalidate
   - buf.build/googleapis/googleapis
 lint:
   use:
-    - DEFAULT
+    - STANDARD
   except:
     - FIELD_NOT_REQUIRED
     - PACKAGE_NO_IMPORT_CYCLE

docs/docusaurus.config.ts

@@ -92,7 +92,7 @@ const config: Config = {
       'docusaurus-protobuffet',
       {
         protobuffet: {
-          fileDescriptorsPath: './protos/data/proto_docs.json',
+          fileDescriptorsPath: './protos/proto_docs.json',
           protoDocsPath: 'protos',
           sidebarPath: './protos/proto-sidebars.js',
         },

python/README.md

@@ -0,0 +1 @@
+Python SDK

python/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "neosync"
+dynamic = ["version"]
+authors = [
+  { name = "Neosync Dev Team", email="[email protected]"}
+]
+description = "Neosync Python SDK"
+requires-python = ">=3.8"
+readme = "README.md"
+dependencies = [
+  "grpcio==1.67.*",
+  "protobuf==5.*",
+]
+
+[project.urls]
+Homepage = "https://github.com/nucleuscloud/neosync"
+Issues = "https://github.com/nucleuscloud/neosync/issues"
+
+
+[tool.hatch.version]
+source = "vcs"

python/src/neosync/__init__.py

@@ -0,0 +1,6 @@
+# Generated by the Neosync generate script. DO NOT EDIT!
+
+import sys
+import os
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))