add data combiner workflow

.github/workflows/data-combine.yml

@@ -0,0 +1,51 @@
+name: data combiner
+run-name: Building data for commit ${{ github.sha }} by ${{ github.actor }}
+on:
+  push:
+    branches:
+      - "main"
+
+permissions: write-all
+jobs:
+  build-data:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo content
+        uses: actions/checkout@v2
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: build data
+        run: python src/combine_data.py
+
+      - name: Archive generated files as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: generated-files
+          path: ./api
+
+      - name: Checkout production branch
+        uses: actions/checkout@v2
+        with:
+          ref: production
+
+      - name: Download generated files artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: generated-files
+          path: ./api
+
+      - name: Apply changes to production branch
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add .
+          git diff-index --quiet HEAD || (git commit -a -m "Add generated files from main branch" --allow-empty && git push origin production)

src/classes/data.py

@@ -1,5 +1,6 @@
 import os
 import tomllib
+import toml
 from pydantic import ValidationError
 
 from views.agreement import Agreement
@@ -16,11 +17,13 @@ def __new__(cls):
     def __init__(self):
         if not hasattr(self, "data_dir"):  # Prevent re-initialization
             self.data_dir = "data/"
+            self.output_dir = "api/"
             self.data = []
             self.errors = []
 
-    def read_data(self):
+            self.count_read = 0
 
+    def read_data(self):
         if not os.path.exists(self.data_dir):
             raise FileNotFoundError(f"Directory {self.data_dir} does not exist")
 
@@ -35,17 +38,35 @@ def read_data(self):
                 file_path = os.path.join(root, file)
                 self.read_toml(file_path)
 
+        print(f"Read {self.count_read} agreements, {len(self.errors)} errors")
+
     def read_toml(self, file_path):
+        print(file_path)
         try:
             with open(file_path, "rb") as toml_file:
 
                 data = tomllib.load(toml_file)
 
                 for aggrement_dict in data["agreements"]:
                     try:
-                        self.data.append(Agreement(**aggrement_dict))
+                        self.count_read += 1
+                        self.data.append(Agreement.from_dict(aggrement_dict))
                     except ValidationError as e:
                         self.errors.append(f"Failed on {file_path} with {e}")
 
         except (tomllib.TOMLDecodeError, FileNotFoundError, PermissionError) as e:
             print(f"Exception {e}. Failed on {file_path}")
+
+    def combine_data(self):
+        output_file = os.path.join(self.output_dir, "agreements.toml")
+
+        os.makedirs(self.output_dir, exist_ok=True)
+
+        agreements_list = [agreement.model_dump() for agreement in self.data]
+
+        toml_data = {"agreements": agreements_list}
+
+        with open(output_file, "w", encoding="utf-8") as toml_file:
+            toml.dump(toml_data, toml_file)
+
+        print(f"Combined data written to {output_file}")

src/combine_data.py

@@ -0,0 +1,10 @@
+from classes.data import Data
+
+
+if __name__ == "__main__":
+    data = Data()
+    data.read_data()
+    data.combine_data()
+
+    for error in data.errors:
+        print(error)