Feat/refactor layoutelement textregion to vectorized data structure

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.16.15-dev0
+
+### Enhancements
+
+### Features
+- **Vectorize layout (inferred, extracted, and OCR) data structure** Using `np.ndarray` to store a group of layout elements or text regions instead of using a list of objects. This improves the memory efficiency and compute speed around layout merging and deduplication.
+
+### Fixes
+
 ## 0.16.14
 
 ### Enhancements

Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/unstructured-io/base-images:wolfi-base-latest AS base
 
 ARG PYTHON=python3.11
-ARG PIP=pip3.11
+ARG PIP="${PYTHON} -m pip"
 
 USER root
 
@@ -19,6 +19,9 @@ RUN chown -R notebook-user:notebook-user /app && \
 
 USER notebook-user
 
+# append PATH before pip install to avoid warning logs; it also avoids issues with packages that needs compilation during installation
+ENV PATH="${PATH}:/home/notebook-user/.local/bin"
+ENV TESSDATA_PREFIX=/usr/local/share/tessdata
 ENV NLTK_DATA=/home/notebook-user/nltk_data
 
 # Install Python dependencies and download required NLTK packages
@@ -28,7 +31,4 @@ RUN find requirements/ -type f -name "*.txt" -exec $PIP install --no-cache-dir -
     $PYTHON -c "from unstructured.partition.model_init import initialize; initialize()" && \
     $PYTHON -c "from unstructured_inference.models.tables import UnstructuredTableTransformerModel; model = UnstructuredTableTransformerModel(); model.initialize('microsoft/table-transformer-structure-recognition')"
 
-ENV PATH="${PATH}:/home/notebook-user/.local/bin"
-ENV TESSDATA_PREFIX=/usr/local/share/tessdata
-
 CMD ["/bin/bash"]

Makefile

@@ -308,7 +308,7 @@ docker-test:
 	$(DOCKER_IMAGE) \
 	bash -c "CI=$(CI) \
 	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) \
-	pytest $(if $(TEST_FILE),$(TEST_FILE),test_unstructured)"
+	python3 -m pytest $(if $(TEST_FILE),$(TEST_FILE),test_unstructured)"
 
 .PHONY: docker-smoke-test
 docker-smoke-test:

requirements/base.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile ./base.in
 #
-anyio==4.7.0
+anyio==4.8.0
     # via httpx
 backoff==2.2.1
     # via -r ./base.in
@@ -36,7 +36,7 @@ dataclasses-json==0.6.7
     #   unstructured-client
 deepdiff==8.1.1
     # via unstructured-client
-emoji==2.14.0
+emoji==2.14.1
     # via -r ./base.in
 exceptiongroup==1.2.2
     # via anyio
@@ -64,7 +64,7 @@ langdetect==1.0.9
     # via -r ./base.in
 lxml==5.3.0
     # via -r ./base.in
-marshmallow==3.23.2
+marshmallow==3.25.1
     # via
     #   dataclasses-json
     #   unstructured-client
@@ -150,5 +150,5 @@ urllib3==1.26.20
     #   unstructured-client
 webencodings==0.5.1
     # via html5lib
-wrapt==1.17.0
+wrapt==1.17.2
     # via -r ./base.in

requirements/deps/constraints.txt

@@ -3,8 +3,8 @@
 # extras. Putting a dependency here will only affect dependency sets that contain them -- in other
 # words, if something does not require a constraint, it will not be installed.
 ####################################################################################################
-# (jennings): Versions greater than 5.0 create dependency conflicts with other packages
-protobuf<5.0
+# we are using v3 client https://weaviate.io/developers/weaviate/client-libraries/python/python_v3
+weaviate-client>=3.26.7,<4.0.0
 # TODO: Constriant due to multiple versions being installed during pip-compile
 grpcio>=1.65.5
 # TODO: Pinned in transformers package, remove when that gets updated (https://github.com/huggingface/transformers/blob/main/setup.py)

requirements/dev.txt

@@ -15,11 +15,11 @@ click==8.1.8
     #   pip-tools
 distlib==0.3.9
     # via virtualenv
-filelock==3.16.1
+filelock==3.17.0
     # via virtualenv
-identify==2.6.4
+identify==2.6.6
     # via pre-commit
-importlib-metadata==8.5.0
+importlib-metadata==8.6.1
     # via
     #   -c ././deps/constraints.txt
     #   build
@@ -36,7 +36,7 @@ platformdirs==4.3.6
     # via
     #   -c ./test.txt
     #   virtualenv
-pre-commit==4.0.1
+pre-commit==4.1.0
     # via -r ./dev.in
 pyproject-hooks==1.2.0
     # via
@@ -51,7 +51,7 @@ tomli==2.2.1
     #   -c ./test.txt
     #   build
     #   pip-tools
-virtualenv==20.28.1
+virtualenv==20.29.1
     # via pre-commit
 wheel==0.45.1
     # via pip-tools

requirements/extra-csv.txt

@@ -20,5 +20,5 @@ six==1.17.0
     # via
     #   -c ./base.txt
     #   python-dateutil
-tzdata==2024.2
+tzdata==2025.1
     # via pandas

requirements/extra-epub.txt

@@ -4,5 +4,5 @@
 #
 #    pip-compile ./extra-epub.in
 #
-pypandoc==1.14
+pypandoc==1.15
     # via -r ./extra-epub.in

requirements/extra-markdown.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile ./extra-markdown.in
 #
-importlib-metadata==8.5.0
+importlib-metadata==8.6.1
     # via
     #   -c ././deps/constraints.txt
     #   markdown

requirements/extra-odt.txt

@@ -8,7 +8,7 @@ lxml==5.3.0
     # via
     #   -c ./base.txt
     #   python-docx
-pypandoc==1.14
+pypandoc==1.15
     # via -r ./extra-odt.in
 python-docx==1.1.2
     # via -r ./extra-odt.in

requirements/extra-paddleocr.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile ./extra-paddleocr.in
 #
-anyio==4.7.0
+anyio==4.8.0
     # via
     #   -c ./base.txt
     #   httpx
@@ -32,7 +32,7 @@ exceptiongroup==1.2.2
     # via
     #   -c ./base.txt
     #   anyio
-fonttools==4.55.3
+fonttools==4.55.4
     # via matplotlib
 h11==0.14.0
     # via
@@ -52,13 +52,13 @@ idna==3.10
     #   anyio
     #   httpx
     #   requests
-imageio==2.36.1
+imageio==2.37.0
     # via
     #   imgaug
     #   scikit-image
 imgaug==0.4.0
     # via unstructured-paddleocr
-importlib-resources==6.5.1
+importlib-resources==6.5.2
     # via matplotlib
 kiwisolver==1.4.7
     # via matplotlib
@@ -86,9 +86,9 @@ numpy==1.26.4
     #   shapely
     #   tifffile
     #   unstructured-paddleocr
-opencv-contrib-python==4.10.0.84
+opencv-contrib-python==4.11.0.86
     # via unstructured-paddleocr
-opencv-python==4.10.0.84
+opencv-python==4.11.0.86
     # via
     #   imgaug
     #   unstructured-paddleocr
@@ -113,10 +113,8 @@ pillow==11.1.0
     #   pdf2image
     #   scikit-image
     #   unstructured-paddleocr
-protobuf==4.25.5
-    # via
-    #   -c ././deps/constraints.txt
-    #   paddlepaddle
+protobuf==5.29.3
+    # via paddlepaddle
 pyclipper==1.3.0.post6
     # via unstructured-paddleocr
 pyparsing==3.2.1

requirements/extra-pandoc.txt

@@ -4,5 +4,5 @@
 #
 #    pip-compile ./extra-pandoc.in
 #
-pypandoc==1.14
+pypandoc==1.15
     # via -r ./extra-pandoc.in

requirements/extra-pdf-image.in

@@ -11,5 +11,5 @@ google-cloud-vision
 effdet
 # Do not move to constraints.in, otherwise unstructured-inference will not be upgraded
 # when unstructured library is.
-unstructured-inference==0.8.1
+unstructured-inference==0.8.4
 unstructured.pytesseract>=0.3.12