feat: add markdowns per pages

Makefile

@@ -96,6 +96,11 @@ coverage:
 
 .PHONY: test
 test:
+# Ideally, it should be ok to run without installing tparse locally.
+# However, there may be some issues that arise from running the tests
+# in the container. If you encounter any issues, please install tparse
+# locally via `go install github.com/mfridman/tparse/cmd/tparse@latest`
+# and run the tests locally.
 	@if [ "${OCR}" = "true" ]; then \
 		docker run --rm \
 			-v $(PWD):/${SERVICE_NAME} \

pkg/component/operator/document/v0/README.mdx

@@ -59,6 +59,7 @@ Convert document to text in Markdown format.
 | Images (optional) | `images` | array[string] | Images extracted from the document |
 | Error (optional) | `error` | string | Error message if any during the conversion process |
 | All Page Images (optional) | `all-page-images` | array[string] | The image contains all the pages in the document if we detect there could be images in the page. It will only support DOCX/DOC/PPTX/PPT/PDF. |
+| Markdowns (optional) | `markdowns` | array[string] | Markdown text converted from the PDF document, separated by page. |
 </div>
 
 ### Convert to Text

pkg/component/operator/document/v0/config/tasks.json

@@ -108,6 +108,16 @@
           },
           "title": "All Page Images",
           "type": "array"
+        },
+        "markdowns": {
+          "description": "Markdown text converted from the PDF document, separated by page.",
+          "instillFormat": "array:string",
+          "instillUIOrder": 5,
+          "items": {
+            "type": "string"
+          },
+          "title": "Markdowns",
+          "type": "array"
         }
       },
       "required": [

pkg/component/operator/document/v0/convert_document_to_markdown.go

@@ -53,6 +53,7 @@ func (e *execution) convertDocumentToMarkdown(ctx context.Context, job *base.Job
 			}
 			return images
 		}(),
+		Markdowns: transformerOutputStruct.Markdowns,
 	}
 
 	err = job.Output.WriteData(ctx, outputStruct)

pkg/component/operator/document/v0/convert_document_to_markdown_test.go

@@ -30,6 +30,7 @@ func TestConvertDocumentToMarkdown(t *testing.T) {
 				Body:          "# This is test file for markdown\n",
 				Images:        []format.Image{},
 				AllPageImages: []format.Image{},
+				Markdowns:     []string{"# This is test file for markdown\n"},
 			},
 		},
 		{
@@ -39,6 +40,7 @@ func TestConvertDocumentToMarkdown(t *testing.T) {
 				Body:          "# This is test file for markdown\n",
 				Images:        []format.Image{},
 				AllPageImages: []format.Image{},
+				Markdowns:     []string{"# This is test file for markdown\n"},
 			},
 		},
 		{
@@ -57,6 +59,7 @@ func TestConvertDocumentToMarkdown(t *testing.T) {
 				Body:          "# This           is     test          file       for markdown\n",
 				Images:        []format.Image{},
 				AllPageImages: []format.Image{},
+				Markdowns:     []string{"# This           is     test          file       for markdown\n"},
 			},
 		},
 		{

pkg/component/operator/document/v0/io.go

@@ -15,6 +15,7 @@ type ConvertDocumentToMarkdownOutput struct {
 	Images        []format.Image `instill:"images"`
 	Error         string         `instill:"error"`
 	AllPageImages []format.Image `instill:"all-page-images"`
+	Markdowns     []string       `instill:"markdowns"`
 }
 
 type ConvertDocumentToImagesInput struct {