Update code snippets

snippets/destination_connectors/astra.py

@@ -0,0 +1,58 @@
+```
+import os
+
+from unstructured.ingest.connector.astra import (
+    AstraAccessConfig,
+    AstraWriteConfig,
+    SimpleAstraConfig,
+)
+from unstructured.ingest.connector.local import SimpleLocalConfig
+from unstructured.ingest.interfaces import (
+    ChunkingConfig,
+    EmbeddingConfig,
+    PartitionConfig,
+    ProcessorConfig,
+    ReadConfig,
+)
+from unstructured.ingest.runner import LocalRunner
+from unstructured.ingest.runner.writers.astra import (
+    AstraWriter,
+)
+from unstructured.ingest.runner.writers.base_writer import Writer
+
+
+def get_writer() -> Writer:
+    return AstraWriter(
+        connector_config=SimpleAstraConfig(
+            access_config=AstraAccessConfig(
+                token=os.getenv("ASTRA_DB_TOKEN"), api_endpoint=os.getenv("ASTRA_DB_ENDPOINT")
+            ),
+            collection_name="test_collection",
+            embedding_dimension=384,
+        ),
+        write_config=AstraWriteConfig(batch_size=80),
+    )
+
+
+if __name__ == "__main__":
+    writer = get_writer()
+    runner = LocalRunner(
+        processor_config=ProcessorConfig(
+            verbose=True,
+            output_dir="local-output-to-astra",
+            num_processes=2,
+        ),
+        connector_config=SimpleLocalConfig(
+            input_path="example-docs/book-war-and-peace-1p.txt",
+        ),
+        read_config=ReadConfig(),
+        partition_config=PartitionConfig(),
+        chunking_config=ChunkingConfig(chunk_elements=True),
+        embedding_config=EmbeddingConfig(
+            provider="langchain-huggingface",
+        ),
+        writer=writer,
+        writer_kwargs={},
+    )
+    runner.run()
+```

snippets/destination_connectors/astra.sh.mdx → snippets/destination_connectors/astra.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/azure.py.mdx → snippets/destination_connectors/azure.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.fsspec.azure import (
     AzureAccessConfig,
     AzureWriteConfig,

snippets/destination_connectors/azure.sh.mdx → snippets/destination_connectors/azure.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/azure_cognitive_search.py.mdx → snippets/destination_connectors/azure_cognitive_search.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.azure_cognitive_search import (

snippets/destination_connectors/azure_cognitive_search.sh.mdx → snippets/destination_connectors/azure_cognitive_search.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}
 

snippets/destination_connectors/box.py.mdx → snippets/destination_connectors/box.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.box import (

snippets/destination_connectors/box.sh.mdx → snippets/destination_connectors/box.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/chroma.py.mdx → snippets/destination_connectors/chroma.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.chroma import (
     ChromaAccessConfig,
     ChromaWriteConfig,

snippets/destination_connectors/chroma.sh.mdx → snippets/destination_connectors/chroma.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/destination_connectors/clarifai.py.mdx → snippets/destination_connectors/clarifai.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.clarifai import (
     ClarifaiAccessConfig,
     ClarifaiWriteConfig,

snippets/destination_connectors/clarifai.sh.mdx → snippets/destination_connectors/clarifai.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/destination_connectors/databricks_volumes.py.mdx → snippets/destination_connectors/databricks_volumes.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.databricks_volumes import (

snippets/destination_connectors/databricks_volumes.sh.mdx → snippets/destination_connectors/databricks_volumes.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/destination_connectors/delta_table.py.mdx → snippets/destination_connectors/delta_table.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.delta_table import DeltaTableWriteConfig, SimpleDeltaTableConfig

snippets/destination_connectors/delta_table.sh.mdx → snippets/destination_connectors/delta_table.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/dropbox.py.mdx → snippets/destination_connectors/dropbox.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.dropbox import (

snippets/destination_connectors/dropbox.sh.mdx → snippets/destination_connectors/dropbox.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/elasticsearch.py.mdx → snippets/destination_connectors/elasticsearch.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.elasticsearch import (

snippets/destination_connectors/elasticsearch.sh.mdx → snippets/destination_connectors/elasticsearch.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/gcs.py.mdx → snippets/destination_connectors/gcs.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.gcs import (

snippets/destination_connectors/gcs.sh.mdx → snippets/destination_connectors/gcs.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/mongodb.py.mdx → snippets/destination_connectors/mongodb.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.local import SimpleLocalConfig

snippets/destination_connectors/mongodb.sh.mdx → snippets/destination_connectors/mongodb.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/opensearch.py.mdx → snippets/destination_connectors/opensearch.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.elasticsearch import (

snippets/destination_connectors/opensearch.sh.mdx → snippets/destination_connectors/opensearch.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/pinecone.py.mdx → snippets/destination_connectors/pinecone.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.local import SimpleLocalConfig

snippets/destination_connectors/pinecone.sh.mdx → snippets/destination_connectors/pinecone.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/qdrant.py.mdx → snippets/destination_connectors/qdrant.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.local import SimpleLocalConfig
 from unstructured.ingest.connector.qdrant import (
     QdrantWriteConfig,

snippets/destination_connectors/qdrant.sh.mdx → snippets/destination_connectors/qdrant.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/s3.py.mdx → snippets/destination_connectors/s3.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.fsspec.s3 import S3AccessConfig, S3WriteConfig, SimpleS3Config
 from unstructured.ingest.connector.local import SimpleLocalConfig
 from unstructured.ingest.interfaces import (

snippets/destination_connectors/s3.sh.mdx → snippets/destination_connectors/s3.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/destination_connectors/sql.py.mdx → snippets/destination_connectors/sql.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.interfaces import PartitionConfig, ProcessorConfig, ReadConfig

snippets/destination_connectors/sql.sh.mdx → snippets/destination_connectors/sql.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/destination_connectors/vectara.py.mdx → snippets/destination_connectors/vectara.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.local import SimpleLocalConfig

snippets/destination_connectors/vectara.sh.mdx → snippets/destination_connectors/vectara.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/destination_connectors/weaviate.py.mdx → snippets/destination_connectors/weaviate.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.local import SimpleLocalConfig
 from unstructured.ingest.connector.weaviate import (
     SimpleWeaviateConfig,

snippets/destination_connectors/weaviate.sh.mdx → snippets/destination_connectors/weaviate.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}

snippets/source_connectors/airtable.py.mdx → snippets/source_connectors/airtable.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.airtable import AirtableAccessConfig, SimpleAirtableConfig

snippets/source_connectors/airtable.sh.mdx → snippets/source_connectors/airtable.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/airtable_api.py.mdx → snippets/source_connectors/airtable_api.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.airtable import AirtableAccessConfig, SimpleAirtableConfig

snippets/source_connectors/airtable_api.sh.mdx → snippets/source_connectors/airtable_api.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/azure.py.mdx → snippets/source_connectors/azure.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.fsspec.azure import (
     AzureAccessConfig,
     SimpleAzureBlobStorageConfig,

snippets/source_connectors/azure.sh.mdx → snippets/source_connectors/azure.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/azure_api.py.mdx → snippets/source_connectors/azure_api.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.azure import (

snippets/source_connectors/azure_api.sh.mdx → snippets/source_connectors/azure_api.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/biomed.py.mdx → snippets/source_connectors/biomed.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.biomed import (
     SimpleBiomedConfig,
 )

snippets/source_connectors/biomed.sh.mdx → snippets/source_connectors/biomed.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/biomed_api.py.mdx → snippets/source_connectors/biomed_api.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.biomed import (

snippets/source_connectors/biomed_api.sh.mdx → snippets/source_connectors/biomed_api.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/box.py.mdx → snippets/source_connectors/box.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.box import BoxAccessConfig, SimpleBoxConfig

snippets/source_connectors/box.sh.mdx → snippets/source_connectors/box.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/box_api.py.mdx → snippets/source_connectors/box_api.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.fsspec.box import BoxAccessConfig, SimpleBoxConfig

snippets/source_connectors/box_api.sh.mdx → snippets/source_connectors/box_api.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/confluence.py.mdx → snippets/source_connectors/confluence.py

@@ -1,4 +1,4 @@
-```python
+```
 from unstructured.ingest.connector.confluence import ConfluenceAccessConfig, SimpleConfluenceConfig
 from unstructured.ingest.interfaces import PartitionConfig, ProcessorConfig, ReadConfig
 from unstructured.ingest.runner import ConfluenceRunner

snippets/source_connectors/confluence.sh.mdx → snippets/source_connectors/confluence.sh

@@ -1,4 +1,4 @@
-```bash
+```
 #!/usr/bin/env bash
 
 unstructured-ingest \

snippets/source_connectors/confluence_api.py.mdx → snippets/source_connectors/confluence_api.py

@@ -1,4 +1,4 @@
-```python
+```
 import os
 
 from unstructured.ingest.connector.confluence import ConfluenceAccessConfig, SimpleConfluenceConfig