Allow chunking map call to take in sublists of length 10 and support lists as inputs

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: Python Poetry CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.10
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+    - name: Install dependencies
+      run: |
+        poetry install

pyproject.toml

@@ -16,7 +16,7 @@ Homepage = "https://sweep.dev"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-PyGithub = "1.58.2"
+PyGithub = "1.59.0"
 loguru = "^0.6.0"
 requests = "^2.28.2"
 urllib3 = "<2.0.0"

sweepai/utils/utils.py

@@ -215,6 +215,24 @@ def __enter__(self):
 
     @method()
     def chunk(
+            self,
+            file_content: Union[str, List[str]],
+            file_path: str,
+            score: float = 1.0,
+            additional_metadata: dict[str, str] = {},
+            max_chunk_size: int = 512 * 3,
+            chunk_size: int = 30,
+            overlap: int = 15
+    ) -> tuple[list[str], list[dict[str, str]]]:
+        if isinstance(file_content, list):
+            results = []
+            for content in file_content:
+                result = self._chunk_single(content, file_path, score, additional_metadata, max_chunk_size, chunk_size, overlap)
+                results.append(result)
+            return results
+        else:
+            return self._chunk_single(file_content, file_path, score, additional_metadata, max_chunk_size, chunk_size, overlap)
+    def _chunk_single(
             self,
             file_content: str,
             file_path: str,