From 04d1467cef580ee423651ebed86a7a5ef6e64b64 Mon Sep 17 00:00:00 2001 From: ChunHao <64747455+chuang8511@users.noreply.github.com> Date: Fri, 18 Oct 2024 05:14:45 +0100 Subject: [PATCH] feat(collection): add concat (#748) Because - we need to concat 2 arrays This commit - add concat task in collection - add recipe samples --------- Co-authored-by: Chang, Hui-Tang --- .github/workflows/test-compogen.yml | 1 - .pre-commit-config.yaml | 1 + pkg/component/ai/anthropic/v0/README.mdx | 9 ++ pkg/component/ai/cohere/v0/README.mdx | 9 ++ pkg/component/ai/fireworksai/v0/README.mdx | 9 ++ pkg/component/ai/groq/v0/README.mdx | 9 ++ pkg/component/ai/huggingface/v0/README.mdx | 9 ++ pkg/component/ai/instill/v0/README.mdx | 9 ++ pkg/component/ai/mistralai/v0/README.mdx | 9 ++ .../ai/ollama/v0/.compogen/setup-hosting.mdx | 2 +- pkg/component/ai/ollama/v0/README.mdx | 10 ++ pkg/component/ai/openai/v0/README.mdx | 9 ++ pkg/component/ai/stabilityai/v0/README.mdx | 9 ++ pkg/component/ai/universalai/v0/README.mdx | 9 ++ pkg/component/application/asana/v0/README.mdx | 9 ++ pkg/component/application/email/v0/README.mdx | 11 +- .../application/freshdesk/v0/README.mdx | 9 ++ .../application/github/v0/README.mdx | 9 ++ .../application/googlesearch/v0/README.mdx | 9 ++ .../application/hubspot/v0/README.mdx | 9 ++ .../application/instillapp/v0/README.mdx | 9 ++ pkg/component/application/jira/v0/README.mdx | 9 ++ .../application/numbers/v0/README.mdx | 9 ++ pkg/component/application/slack/v0/README.mdx | 9 ++ .../application/whatsapp/v0/README.mdx | 9 ++ pkg/component/data/bigquery/v0/README.mdx | 9 ++ pkg/component/data/chroma/v0/README.mdx | 9 ++ .../data/elasticsearch/v0/README.mdx | 9 ++ .../data/googlecloudstorage/v0/README.mdx | 9 ++ .../v0/.compogen/extra-intro.mdx | 2 +- .../data/instillartifact/v0/README.mdx | 10 ++ pkg/component/data/milvus/v0/README.mdx | 9 ++ pkg/component/data/mongodb/v0/README.mdx | 9 ++ pkg/component/data/pinecone/v0/README.mdx | 9 ++ pkg/component/data/qdrant/v0/README.mdx | 9 ++ pkg/component/data/redis/v0/README.mdx | 9 ++ pkg/component/data/sql/v0/README.mdx | 9 ++ pkg/component/data/weaviate/v0/README.mdx | 9 ++ pkg/component/data/zilliz/v0/README.mdx | 9 ++ .../collection/v0/.compogen/bottom.mdx | 71 ++++++++++++ .../generic/collection/v0/README.mdx | 105 ++++++++++++++++++ .../collection/v0/config/definition.json | 3 +- .../generic/collection/v0/config/tasks.json | 57 ++++++++++ pkg/component/generic/collection/v0/main.go | 18 ++- .../generic/collection/v0/main_test.go | 62 +++++++++++ pkg/component/generic/restapi/v0/README.mdx | 9 ++ pkg/component/operator/audio/v0/README.mdx | 9 ++ pkg/component/operator/base64/v0/README.mdx | 9 ++ .../operator/document/v0/.compogen/bottom.mdx | 2 +- pkg/component/operator/document/v0/README.mdx | 11 +- pkg/component/operator/image/v0/README.mdx | 9 ++ pkg/component/operator/json/v0/README.mdx | 9 ++ pkg/component/operator/text/v0/README.mdx | 9 ++ pkg/component/operator/video/v0/README.mdx | 9 ++ pkg/component/operator/web/v0/README.mdx | 9 ++ .../cmd/testdata/readme-component1.txt | 9 ++ .../cmd/testdata/readme-component2.txt | 9 ++ .../gen/resources/templates/readme.mdx.tmpl | 14 ++- 58 files changed, 744 insertions(+), 14 deletions(-) create mode 100644 pkg/component/generic/collection/v0/.compogen/bottom.mdx create mode 100644 pkg/component/generic/collection/v0/main_test.go diff --git a/.github/workflows/test-compogen.yml b/.github/workflows/test-compogen.yml index 69fbaecd8..9bfd6abd0 100644 --- a/.github/workflows/test-compogen.yml +++ b/.github/workflows/test-compogen.yml @@ -34,4 +34,3 @@ jobs: cd pkg/component/tools/compogen go mod tidy go test -mod=mod -race -cover ./... - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d99acf41..9faea3c61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,7 @@ repos: - id: check-yaml - id: end-of-file-fixer exclude: (?i).*testdata/ + exclude_types: [svg, mdx] - id: trailing-whitespace exclude: (?i).*testdata/ - repo: https://github.com/dnephin/pre-commit-golang diff --git a/pkg/component/ai/anthropic/v0/README.mdx b/pkg/component/ai/anthropic/v0/README.mdx index 4ae62fd57..6d38ffacb 100644 --- a/pkg/component/ai/anthropic/v0/README.mdx +++ b/pkg/component/ai/anthropic/v0/README.mdx @@ -9,14 +9,21 @@ The Anthropic component is an AI component that allows users to connect the AI m It can carry out the following tasks: - [Text Generation Chat](#text-generation-chat) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/anthropic/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/anthropic/v0/config/tasks.json) files respectively. + + + ## Setup @@ -122,6 +129,8 @@ The image URL | Output Tokens | `output-tokens` | number | The output tokens used by Anthropic | + + ## Example Recipes Recipe for the [List GitHub Repo Issues](https://instill.tech/instill-ai/pipelines/github-demo/playground) pipeline. diff --git a/pkg/component/ai/cohere/v0/README.mdx b/pkg/component/ai/cohere/v0/README.mdx index aff0563ce..56abe2291 100644 --- a/pkg/component/ai/cohere/v0/README.mdx +++ b/pkg/component/ai/cohere/v0/README.mdx @@ -11,14 +11,21 @@ It can carry out the following tasks: - [Text Embeddings](#text-embeddings) - [Text Reranking](#text-reranking) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/cohere/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/cohere/v0/config/tasks.json) files respectively. + + + ## Setup @@ -221,3 +228,5 @@ Rerank models sort text inputs by semantic relevance to a specified query. They | Search Counts | `search-counts` | number | The search count used by Cohere Models | + + diff --git a/pkg/component/ai/fireworksai/v0/README.mdx b/pkg/component/ai/fireworksai/v0/README.mdx index 937c0ab07..f4f15f99e 100644 --- a/pkg/component/ai/fireworksai/v0/README.mdx +++ b/pkg/component/ai/fireworksai/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Text Generation Chat](#text-generation-chat) - [Text Embeddings](#text-embeddings) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/fireworksai/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/fireworksai/v0/config/tasks.json) files respectively. + + + ## Setup @@ -163,6 +170,8 @@ An embedding is a list of floating point numbers that captures semantic informat | Token Count | `tokens` | number | The token count used by Fireworks AI models | + + ## Example Recipes Recipe for the [Fireworks Chinese Content Writer](https://instill.tech/instill-ai/pipelines/fireworks-chinese-content-writer/playground) pipeline. diff --git a/pkg/component/ai/groq/v0/README.mdx b/pkg/component/ai/groq/v0/README.mdx index 5db2db542..ab73699c2 100644 --- a/pkg/component/ai/groq/v0/README.mdx +++ b/pkg/component/ai/groq/v0/README.mdx @@ -9,14 +9,21 @@ The Groq component is an AI component that allows users to connect the AI models It can carry out the following tasks: - [Text Generation Chat](#text-generation-chat) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/groq/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/groq/v0/config/tasks.json) files respectively. + + + ## Setup @@ -124,6 +131,8 @@ The image URL | Output Tokens | `output-tokens` | number | The output tokens generated by GroqCloud OSS models | + + ## Example Recipes Recipe for the [Groq Interview Helper](https://instill.tech/instill-ai/pipelines/groq-interview-helper/playground) pipeline. diff --git a/pkg/component/ai/huggingface/v0/README.mdx b/pkg/component/ai/huggingface/v0/README.mdx index 92eeeee0f..58126d9dd 100644 --- a/pkg/component/ai/huggingface/v0/README.mdx +++ b/pkg/component/ai/huggingface/v0/README.mdx @@ -25,14 +25,21 @@ It can carry out the following tasks: - [Speech Recognition](#speech-recognition) - [Audio Classification](#audio-classification) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/huggingface/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/huggingface/v0/config/tasks.json) files respectively. + + + ## Setup @@ -913,3 +920,5 @@ Audio classification is the task of assigning a label or class to a given audio. | Score | `score` | number | A float that represents how likely it is that the audio file belongs to this class. | + + diff --git a/pkg/component/ai/instill/v0/README.mdx b/pkg/component/ai/instill/v0/README.mdx index b8f177e1e..f37743e2d 100644 --- a/pkg/component/ai/instill/v0/README.mdx +++ b/pkg/component/ai/instill/v0/README.mdx @@ -19,16 +19,23 @@ It can carry out the following tasks: - [Visual Question Answering](#visual-question-answering) - [Chat](#chat) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/instill/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/instill/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Classification @@ -574,3 +581,5 @@ The image URL | :--- | :--- | :--- | :--- | | Text | `text` | string | Text | + + diff --git a/pkg/component/ai/mistralai/v0/README.mdx b/pkg/component/ai/mistralai/v0/README.mdx index 5a24d9cb0..a00b4e87a 100644 --- a/pkg/component/ai/mistralai/v0/README.mdx +++ b/pkg/component/ai/mistralai/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Text Generation Chat](#text-generation-chat) - [Text Embeddings](#text-embeddings) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/mistralai/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/mistralai/v0/config/tasks.json) files respectively. + + + ## Setup @@ -164,6 +171,8 @@ An embedding is a list of floating point numbers that captures semantic informat | Token Count | `tokens` | number | The token count used by Mistral models | + + ## Example Recipes Recipe for the [Short-film Script Writer](https://instill.tech/instill-ai/pipelines/mistral-demo/playground) pipeline. diff --git a/pkg/component/ai/ollama/v0/.compogen/setup-hosting.mdx b/pkg/component/ai/ollama/v0/.compogen/setup-hosting.mdx index 7be0a9eef..d7a9757f1 100644 --- a/pkg/component/ai/ollama/v0/.compogen/setup-hosting.mdx +++ b/pkg/component/ai/ollama/v0/.compogen/setup-hosting.mdx @@ -10,4 +10,4 @@ To set up an Ollama instance on your local machine, follow the instructions belo - On Linux and macOS, open the terminal and type `ifconfig`. - On Windows, open the command prompt and type `ipconfig`. 4. Suppose the IP address is `192.168.178.88`, then the Ollama hosting endpoint would be `192.168.178.88:11434`. -5. Enjoy fast LLM inference on your local machine and integration with 💧 Instill VDP. \ No newline at end of file +5. Enjoy fast LLM inference on your local machine and integration with 💧 Instill VDP. diff --git a/pkg/component/ai/ollama/v0/README.mdx b/pkg/component/ai/ollama/v0/README.mdx index 6d80b715b..260c20528 100644 --- a/pkg/component/ai/ollama/v0/README.mdx +++ b/pkg/component/ai/ollama/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Text Generation Chat](#text-generation-chat) - [Text Embeddings](#text-embeddings) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/ollama/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/ollama/v0/config/tasks.json) files respectively. + + + ## Setup @@ -122,6 +129,7 @@ To set up an Ollama instance on your local machine, follow the instructions belo - On Windows, open the command prompt and type `ipconfig`. 4. Suppose the IP address is `192.168.178.88`, then the Ollama hosting endpoint would be `192.168.178.88:11434`. 5. Enjoy fast LLM inference on your local machine and integration with 💧 Instill VDP. + ### Text Embeddings An embedding is a list of floating point numbers that captures semantic information about the text that it represents. @@ -146,3 +154,5 @@ An embedding is a list of floating point numbers that captures semantic informat | :--- | :--- | :--- | :--- | | Embedding | `embedding` | array[number] | Embedding of the input text | + + diff --git a/pkg/component/ai/openai/v0/README.mdx b/pkg/component/ai/openai/v0/README.mdx index 2f63c801c..d61bebeee 100644 --- a/pkg/component/ai/openai/v0/README.mdx +++ b/pkg/component/ai/openai/v0/README.mdx @@ -13,14 +13,21 @@ It can carry out the following tasks: - [Text to Speech](#text-to-speech) - [Text to Image](#text-to-image) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/openai/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/openai/v0/config/tasks.json) files respectively. + + + ## Setup @@ -290,6 +297,8 @@ Generate or manipulate images with DALL·E. | Revised Prompt | `revised-prompt` | string | Revised prompt | + + ## Example Recipes Recipe for the [PicassoAI: Cubist Creations at Your Command!](https://instill.tech/instill-ai/pipelines/picasso-ai/playground) pipeline. diff --git a/pkg/component/ai/stabilityai/v0/README.mdx b/pkg/component/ai/stabilityai/v0/README.mdx index ceeff77af..9992e042f 100644 --- a/pkg/component/ai/stabilityai/v0/README.mdx +++ b/pkg/component/ai/stabilityai/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Text to Image](#text-to-image) - [Image to Image](#image-to-image) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/stabilityai/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/stabilityai/v0/config/tasks.json) files respectively. + + + ## Setup @@ -115,3 +122,5 @@ Modify an image based on a text prompt. | Images | `images` | array[string] | Generated images | | Seeds | `seeds` | array[number] | Seeds of generated images | + + diff --git a/pkg/component/ai/universalai/v0/README.mdx b/pkg/component/ai/universalai/v0/README.mdx index fd092e54d..1454707ae 100644 --- a/pkg/component/ai/universalai/v0/README.mdx +++ b/pkg/component/ai/universalai/v0/README.mdx @@ -9,14 +9,21 @@ The Universal AI component is an AI component that allows users to connect the A It can carry out the following tasks: - [Chat](#chat) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/universalai/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/universalai/v0/config/tasks.json) files respectively. + + + ## Setup @@ -163,3 +170,5 @@ Input parameter | Total Tokens | `total-tokens` | integer | Total number of tokens used in the request (prompt + completion). | + + diff --git a/pkg/component/application/asana/v0/README.mdx b/pkg/component/application/asana/v0/README.mdx index 031cfe519..e258566bc 100644 --- a/pkg/component/application/asana/v0/README.mdx +++ b/pkg/component/application/asana/v0/README.mdx @@ -12,14 +12,21 @@ It can carry out the following tasks: - [Crud Project](#crud-project) - [Crud Portfolio](#crud-portfolio) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/asana/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/asana/v0/config/tasks.json) files respectively. + + + ## Setup @@ -635,3 +642,5 @@ Create Portfolio | User Name | `name` | string | The name of the user. | + + diff --git a/pkg/component/application/email/v0/README.mdx b/pkg/component/application/email/v0/README.mdx index d949cf839..6c52af1d4 100644 --- a/pkg/component/application/email/v0/README.mdx +++ b/pkg/component/application/email/v0/README.mdx @@ -8,7 +8,9 @@ description: "Learn about how to set up a VDP Email component https://github.com The Email component is an application component that allows users to get and send email from Mail Protocol. It can carry out the following tasks: - [Send Email](#send-email) -- [Read Emails](#read-emails)You can connect to different email servers through the Email component. +- [Read Emails](#read-emails) + +You can connect to different email servers through the Email component. Emails are fetched and sent using the IMAP and SMTP protocols, respectively. You can set the server address and port for each protocol in the component configuration. @@ -16,10 +18,15 @@ Emails are fetched and sent using the IMAP and SMTP protocols, respectively. You `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/email/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/email/v0/config/tasks.json) files respectively. + + + ## Setup @@ -174,3 +181,5 @@ Take Gmail as an example, the mailbox names are following. #### Search From and Search To You need to input the exact same email address with `<` as a prefix and `>` as a suffix as the email you want to search for. For example, if you want to search for the email from `email@example.com`, you need to input ``. + + diff --git a/pkg/component/application/freshdesk/v0/README.mdx b/pkg/component/application/freshdesk/v0/README.mdx index f6984f4ea..47da79245 100644 --- a/pkg/component/application/freshdesk/v0/README.mdx +++ b/pkg/component/application/freshdesk/v0/README.mdx @@ -23,14 +23,21 @@ It can carry out the following tasks: - [Get Group](#get-group) - [Get Skill](#get-skill) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/freshdesk/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/freshdesk/v0/config/tasks.json) files respectively. + + + ## Setup @@ -627,3 +634,5 @@ Get skill using ID. This task will only work if you have admin privileges. Agent | Created At | `created-at` | string | Timestamp that denotes when the skill was created | | Updated At | `updated-at` | string | Timestamp that denotes when the skill was last updated | + + diff --git a/pkg/component/application/github/v0/README.mdx b/pkg/component/application/github/v0/README.mdx index 2b60fadb7..8a4a11ebb 100644 --- a/pkg/component/application/github/v0/README.mdx +++ b/pkg/component/application/github/v0/README.mdx @@ -17,14 +17,21 @@ It can carry out the following tasks: - [Create Issue](#create-issue) - [Create Webhook](#create-webhook) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/github/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/github/v0/config/tasks.json) files respectively. + + + ## Setup @@ -580,6 +587,8 @@ Create a webhook for a repository | Webhook URL | `url` | string | URL of the webhook | + + ## Example Recipes Recipe for the [List GitHub Repo Issues](https://instill.tech/instill-ai/pipelines/github-demo/playground) pipeline. diff --git a/pkg/component/application/googlesearch/v0/README.mdx b/pkg/component/application/googlesearch/v0/README.mdx index 991d71b4a..96fdf501a 100644 --- a/pkg/component/application/googlesearch/v0/README.mdx +++ b/pkg/component/application/googlesearch/v0/README.mdx @@ -9,14 +9,21 @@ The Google Search component is an application component that allows users to lev It can carry out the following tasks: - [Search](#search) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/googlesearch/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/googlesearch/v0/config/tasks.json) files respectively. + + + ## Setup @@ -84,3 +91,5 @@ Search data via Google Search Engine. | Title | `title` | string | The title of a search result, in plain text | + + diff --git a/pkg/component/application/hubspot/v0/README.mdx b/pkg/component/application/hubspot/v0/README.mdx index 38391788b..06f3d9fd1 100644 --- a/pkg/component/application/hubspot/v0/README.mdx +++ b/pkg/component/application/hubspot/v0/README.mdx @@ -23,14 +23,21 @@ It can carry out the following tasks: - [Get Owner](#get-owner) - [Get All](#get-all) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/hubspot/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/hubspot/v0/config/tasks.json) files respectively. + + + ## Setup @@ -586,3 +593,5 @@ Get all the IDs for a specific object (e.g. all contact IDs) | Object ID Array | `object-ids` | array[string] | An array of object ID | | Object IDs Length | `object-ids-length` | integer | The number of object IDs | + + diff --git a/pkg/component/application/instillapp/v0/README.mdx b/pkg/component/application/instillapp/v0/README.mdx index 009a43263..1cd1ed3a5 100644 --- a/pkg/component/application/instillapp/v0/README.mdx +++ b/pkg/component/application/instillapp/v0/README.mdx @@ -10,16 +10,23 @@ It can carry out the following tasks: - [Read Chat History](#read-chat-history) - [Write Chat Message](#write-chat-message) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/instillapp/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/instillapp/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Read Chat History @@ -109,3 +116,5 @@ A chat message to be written into the conversation. | Create Time | `create-time` | string | The creation time of the message in ISO 8601 format | | Update Time | `update-time` | string | The update time of the message in ISO 8601 format | + + diff --git a/pkg/component/application/jira/v0/README.mdx b/pkg/component/application/jira/v0/README.mdx index 2f6c49977..40ac84261 100644 --- a/pkg/component/application/jira/v0/README.mdx +++ b/pkg/component/application/jira/v0/README.mdx @@ -17,14 +17,21 @@ It can carry out the following tasks: - [Create Sprint](#create-sprint) - [Update Sprint](#update-sprint) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/jira/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/jira/v0/config/tasks.json) files respectively. + + + ## Setup @@ -581,6 +588,8 @@ Update a sprint in Jira | Origin Board ID (optional) | `origin-board-id` | integer | The ID of the origin board | | Goal (optional) | `goal` | string | The Goal of the sprint | + + ## Example Recipes Recipe for the [Your Project Management Assistant ](https://instill.tech/instill-ai/pipelines/pm-assistant/playground) pipeline. diff --git a/pkg/component/application/numbers/v0/README.mdx b/pkg/component/application/numbers/v0/README.mdx index 6446996c7..d759934b6 100644 --- a/pkg/component/application/numbers/v0/README.mdx +++ b/pkg/component/application/numbers/v0/README.mdx @@ -9,14 +9,21 @@ The Numbers Protocol component is an application component that allows users to It can carry out the following tasks: - [Register](#register) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/numbers/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/numbers/v0/config/tasks.json) files respectively. + + + ## Setup @@ -84,3 +91,5 @@ License | :--- | :--- | :--- | :--- | | Asset Urls | `asset-urls` | array[string] | Asset Urls | + + diff --git a/pkg/component/application/slack/v0/README.mdx b/pkg/component/application/slack/v0/README.mdx index 7126026a4..fa1a1e394 100644 --- a/pkg/component/application/slack/v0/README.mdx +++ b/pkg/component/application/slack/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Read Message](#read-message) - [Send Message](#send-message) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/slack/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/slack/v0/config/tasks.json) files respectively. + + + ## Setup @@ -135,3 +142,5 @@ send message to a specific channel | :--- | :--- | :--- | :--- | | Result | `result` | string | Result of the message delivery | + + diff --git a/pkg/component/application/whatsapp/v0/README.mdx b/pkg/component/application/whatsapp/v0/README.mdx index b03d7a28f..7de5987c1 100644 --- a/pkg/component/application/whatsapp/v0/README.mdx +++ b/pkg/component/application/whatsapp/v0/README.mdx @@ -17,14 +17,21 @@ It can carry out the following tasks: - [Send Contact Message](#send-contact-message) - [Send Interactive Call to Action URL Button Message](#send-interactive-call-to-action-url-button-message) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/whatsapp/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/whatsapp/v0/config/tasks.json) files respectively. + + + ## Setup @@ -331,3 +338,5 @@ Send interactive Call-To-Action URL button message. Note: Message can only be se | Recipient WhatsApp ID | `recipient-wa-id` | string | The WhatsApp ID of the recipient. Most likely the same with their phone number. | | Message ID | `message-id` | string | The ID of the message | + + diff --git a/pkg/component/data/bigquery/v0/README.mdx b/pkg/component/data/bigquery/v0/README.mdx index 8563424b8..c35044592 100644 --- a/pkg/component/data/bigquery/v0/README.mdx +++ b/pkg/component/data/bigquery/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Insert](#insert) - [Read](#read) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/bigquery/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/bigquery/v0/config/tasks.json) files respectively. + + + ## Setup @@ -91,3 +98,5 @@ Read data from BigQuery. | :--- | :--- | :--- | :--- | | [Data](#read-data) | `data` | array[object] | The data to be read from BigQuery | + + diff --git a/pkg/component/data/chroma/v0/README.mdx b/pkg/component/data/chroma/v0/README.mdx index 9d2d2474f..69ff38577 100644 --- a/pkg/component/data/chroma/v0/README.mdx +++ b/pkg/component/data/chroma/v0/README.mdx @@ -14,14 +14,21 @@ It can carry out the following tasks: - [Create Collection](#create-collection) - [Delete Collection](#delete-collection) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/chroma/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/chroma/v0/config/tasks.json) files respectively. + + + ## Setup @@ -227,3 +234,5 @@ Delete a collection | :--- | :--- | :--- | :--- | | Status | `status` | string | Delete collection status | + + diff --git a/pkg/component/data/elasticsearch/v0/README.mdx b/pkg/component/data/elasticsearch/v0/README.mdx index 4a22f41d1..8efe6fec1 100644 --- a/pkg/component/data/elasticsearch/v0/README.mdx +++ b/pkg/component/data/elasticsearch/v0/README.mdx @@ -16,14 +16,21 @@ It can carry out the following tasks: - [Create Index](#create-index) - [Delete Index](#delete-index) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/elasticsearch/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/elasticsearch/v0/config/tasks.json) files respectively. + + + ## Setup @@ -301,3 +308,5 @@ Delete an index in Elasticsearch | :--- | :--- | :--- | :--- | | Status | `status` | string | Delete index operation status | + + diff --git a/pkg/component/data/googlecloudstorage/v0/README.mdx b/pkg/component/data/googlecloudstorage/v0/README.mdx index e92427974..ac50c48dd 100644 --- a/pkg/component/data/googlecloudstorage/v0/README.mdx +++ b/pkg/component/data/googlecloudstorage/v0/README.mdx @@ -11,14 +11,21 @@ It can carry out the following tasks: - [Read Objects](#read-objects) - [Create Bucket](#create-bucket) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlecloudstorage/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlecloudstorage/v0/config/tasks.json) files respectively. + + + ## Setup @@ -279,3 +286,5 @@ Read objects from Google Cloud Storage. | Bucket Name (optional) | `name` | string | The name of the bucket | | Location (optional) | `location` | string | The location of the bucket | + + diff --git a/pkg/component/data/instillartifact/v0/.compogen/extra-intro.mdx b/pkg/component/data/instillartifact/v0/.compogen/extra-intro.mdx index 9ada1d17e..2fb351914 100644 --- a/pkg/component/data/instillartifact/v0/.compogen/extra-intro.mdx +++ b/pkg/component/data/instillartifact/v0/.compogen/extra-intro.mdx @@ -3,4 +3,4 @@ To use Artifact Component, you will need to set up the OpenAI API key for self-hosted deployment of Instill Core. You can do this by setting the `OPENAI_API_KEY` environment variable. Please refer to [configuring-the-embedding-feature](https://www.instill.tech/docs/core/configuration#configuring-the-embedding-feature) -p.s. In Instill Cloud case, you do not need to set up the OpenAI API key. \ No newline at end of file +p.s. In Instill Cloud case, you do not need to set up the OpenAI API key. diff --git a/pkg/component/data/instillartifact/v0/README.mdx b/pkg/component/data/instillartifact/v0/README.mdx index 4aaed715c..05391404d 100644 --- a/pkg/component/data/instillartifact/v0/README.mdx +++ b/pkg/component/data/instillartifact/v0/README.mdx @@ -16,21 +16,29 @@ It can carry out the following tasks: - [Retrieve](#retrieve) - [Ask](#ask) + + To use Artifact Component, you will need to set up the OpenAI API key for self-hosted deployment of Instill Core. You can do this by setting the `OPENAI_API_KEY` environment variable. Please refer to [configuring-the-embedding-feature](https://www.instill.tech/docs/core/configuration#configuring-the-embedding-feature) p.s. In Instill Cloud case, you do not need to set up the OpenAI API key. + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/instillartifact/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/instillartifact/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Upload File @@ -419,6 +427,8 @@ Reply the questions based on the files in the catalog | Text Content | `text-content` | string | The text content of the chunk | + + ## Example Recipes Recipe for the [Ask your Catalog](https://instill.tech/instill-ai/pipelines/ask-your-catalog/playground) pipeline. diff --git a/pkg/component/data/milvus/v0/README.mdx b/pkg/component/data/milvus/v0/README.mdx index aef1780bd..2fb075dae 100644 --- a/pkg/component/data/milvus/v0/README.mdx +++ b/pkg/component/data/milvus/v0/README.mdx @@ -18,14 +18,21 @@ It can carry out the following tasks: - [Create Index](#create-index) - [Drop Index](#drop-index) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/milvus/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/milvus/v0/config/tasks.json) files respectively. + + + ## Setup @@ -333,3 +340,5 @@ Drop an index from a collection | :--- | :--- | :--- | :--- | | Status | `status` | string | Drop index status | + + diff --git a/pkg/component/data/mongodb/v0/README.mdx b/pkg/component/data/mongodb/v0/README.mdx index 97905927f..285ee7c13 100644 --- a/pkg/component/data/mongodb/v0/README.mdx +++ b/pkg/component/data/mongodb/v0/README.mdx @@ -18,14 +18,21 @@ It can carry out the following tasks: - [Drop Search Index](#drop-search-index) - [Vector Search](#vector-search) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/mongodb/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/mongodb/v0/config/tasks.json) files respectively. + + + ## Setup @@ -355,3 +362,5 @@ Perform a vector search operation | Vectors | `vectors` | array | The vectors returned from the vector search operation | + + diff --git a/pkg/component/data/pinecone/v0/README.mdx b/pkg/component/data/pinecone/v0/README.mdx index 15f2b1189..9f2f1d247 100644 --- a/pkg/component/data/pinecone/v0/README.mdx +++ b/pkg/component/data/pinecone/v0/README.mdx @@ -10,14 +10,21 @@ It can carry out the following tasks: - [Query](#query) - [Upsert](#upsert) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/pinecone/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/pinecone/v0/config/tasks.json) files respectively. + + + ## Setup @@ -116,3 +123,5 @@ Writes vectors into a namespace. If a new value is upserted for an existing vect | :--- | :--- | :--- | :--- | | Upserted Count | `upserted-count` | integer | Number of records modified or added | + + diff --git a/pkg/component/data/qdrant/v0/README.mdx b/pkg/component/data/qdrant/v0/README.mdx index fbed526c6..ce489cec0 100644 --- a/pkg/component/data/qdrant/v0/README.mdx +++ b/pkg/component/data/qdrant/v0/README.mdx @@ -14,14 +14,21 @@ It can carry out the following tasks: - [Create Collection](#create-collection) - [Delete Collection](#delete-collection) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/qdrant/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/qdrant/v0/config/tasks.json) files respectively. + + + ## Setup @@ -224,3 +231,5 @@ Delete a collection | :--- | :--- | :--- | :--- | | Status | `status` | string | Delete collection status | + + diff --git a/pkg/component/data/redis/v0/README.mdx b/pkg/component/data/redis/v0/README.mdx index dc7221c33..20b1ddb34 100644 --- a/pkg/component/data/redis/v0/README.mdx +++ b/pkg/component/data/redis/v0/README.mdx @@ -11,14 +11,21 @@ It can carry out the following tasks: - [Write Chat Message](#write-chat-message) - [Write Multi Modal Chat Message](#write-multi-modal-chat-message) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/redis/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/redis/v0/config/tasks.json) files respectively. + + + ## Setup @@ -160,3 +167,5 @@ Write multi-modal chat message into Redis. | :--- | :--- | :--- | :--- | | Status | `status` | boolean | The status of the write operation | + + diff --git a/pkg/component/data/sql/v0/README.mdx b/pkg/component/data/sql/v0/README.mdx index 42de4c295..a81fbe081 100644 --- a/pkg/component/data/sql/v0/README.mdx +++ b/pkg/component/data/sql/v0/README.mdx @@ -15,14 +15,21 @@ It can carry out the following tasks: - [Create Table](#create-table) - [Drop Table](#drop-table) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/sql/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/sql/v0/config/tasks.json) files respectively. + + + ## Setup @@ -267,3 +274,5 @@ Drop a table in the database | :--- | :--- | :--- | :--- | | Status | `status` | string | Drop table status | + + diff --git a/pkg/component/data/weaviate/v0/README.mdx b/pkg/component/data/weaviate/v0/README.mdx index 3bf36de61..cbbfbbbaf 100644 --- a/pkg/component/data/weaviate/v0/README.mdx +++ b/pkg/component/data/weaviate/v0/README.mdx @@ -14,14 +14,21 @@ It can carry out the following tasks: - [Delete](#delete) - [Delete Collection](#delete-collection) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/weaviate/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/weaviate/v0/config/tasks.json) files respectively. + + + ## Setup @@ -222,3 +229,5 @@ Delete a collection | :--- | :--- | :--- | :--- | | Status | `status` | string | Delete collection status | + + diff --git a/pkg/component/data/zilliz/v0/README.mdx b/pkg/component/data/zilliz/v0/README.mdx index e0b0f2b9f..b595380e8 100644 --- a/pkg/component/data/zilliz/v0/README.mdx +++ b/pkg/component/data/zilliz/v0/README.mdx @@ -16,14 +16,21 @@ It can carry out the following tasks: - [Create Partition](#create-partition) - [Drop Partition](#drop-partition) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/zilliz/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/zilliz/v0/config/tasks.json) files respectively. + + + ## Setup @@ -280,3 +287,5 @@ Drop a partition from a collection | :--- | :--- | :--- | :--- | | Status | `status` | string | Drop partition status | + + diff --git a/pkg/component/generic/collection/v0/.compogen/bottom.mdx b/pkg/component/generic/collection/v0/.compogen/bottom.mdx new file mode 100644 index 000000000..e1029031c --- /dev/null +++ b/pkg/component/generic/collection/v0/.compogen/bottom.mdx @@ -0,0 +1,71 @@ +## Example Recipes + +### Combine 2 arrays using `TASK_UNION` and `TASK_CONCAT` + +```yaml +variable: + texts: + instill-format: array:string + title: Text + texts-2: + instill-format: array:string + title: Text + +component: + +# This component combines two arrays of strings in a union, ensuring that no duplicates are included. +# Examples: +# ["foo", "bar"], ["foo", "bat"] -> ["foo", "bar", "bat"] + union: + type: collection + input: + sets: + - ${variable.texts} + - ${variable.texts-2} + condition: + task: TASK_UNION + +# This component concatenates two arrays of strings. +# Examples: +# ["foo", "bar"], ["zot", "bat"] -> ["foo", "bar", "zot", "bat"] + concat: + type: collection + input: + arrays: + - ${variable.texts} + - ${variable.texts-2} + condition: + task: TASK_CONCAT + +output: + union-result: + title: Union Result + value: ${union.output.set} + concat-result: + title: Concat Result + value: ${concat.output.array} +``` + + +### Use `TASK_ASSIGN` to create an object with `type` and `text` keys +```yaml +variable: + text: + instill-format: string + title: Text + +component: + text-object: + type: collection + input: + data: + type: text + text: ${variable.text} + condition: + task: TASK_ASSIGN + +output: + result: + title: Object + value: ${text-object.output.data} +``` diff --git a/pkg/component/generic/collection/v0/README.mdx b/pkg/component/generic/collection/v0/README.mdx index 9e6374fe6..c6bf8c8b3 100644 --- a/pkg/component/generic/collection/v0/README.mdx +++ b/pkg/component/generic/collection/v0/README.mdx @@ -12,17 +12,25 @@ It can carry out the following tasks: - [Union](#union) - [Intersection](#intersection) - [Difference](#difference) +- [Concat](#concat) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/collection/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/collection/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Assign @@ -146,3 +154,100 @@ Find the difference between the two sets, i.e. `set-a` \ `set-b`, identifying th | :--- | :--- | :--- | :--- | | Array | `set` | array | The difference set. | + +### Concat + +Concatenate the arrays. i.e. `[1, 2] + [3, 4] = [1, 2, 3, 4]` + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_CONCAT` | +| Arrays (required) | `arrays` | array | The arrays to be concatenated. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Array | `array` | array | The concatenated arrays. | +
+ + +## Example Recipes + +### Combine 2 arrays using `TASK_UNION` and `TASK_CONCAT` + +```yaml +variable: + texts: + instill-format: array:string + title: Text + texts-2: + instill-format: array:string + title: Text + +component: + +# This component combines two arrays of strings in a union, ensuring that no duplicates are included. +# Examples: +# ["foo", "bar"], ["foo", "bat"] -> ["foo", "bar", "bat"] + union: + type: collection + input: + sets: + - ${variable.texts} + - ${variable.texts-2} + condition: + task: TASK_UNION + +# This component concatenates two arrays of strings. +# Examples: +# ["foo", "bar"], ["zot", "bat"] -> ["foo", "bar", "zot", "bat"] + concat: + type: collection + input: + arrays: + - ${variable.texts} + - ${variable.texts-2} + condition: + task: TASK_CONCAT + +output: + union-result: + title: Union Result + value: ${union.output.set} + concat-result: + title: Concat Result + value: ${concat.output.array} +``` + + +### Use `TASK_ASSIGN` to create an object with `type` and `text` keys +```yaml +variable: + text: + instill-format: string + title: Text + +component: + text-object: + type: collection + input: + data: + type: text + text: ${variable.text} + condition: + task: TASK_ASSIGN + +output: + result: + title: Object + value: ${text-object.output.data} +``` diff --git a/pkg/component/generic/collection/v0/config/definition.json b/pkg/component/generic/collection/v0/config/definition.json index 148c55bb8..42ef31092 100644 --- a/pkg/component/generic/collection/v0/config/definition.json +++ b/pkg/component/generic/collection/v0/config/definition.json @@ -4,7 +4,8 @@ "TASK_APPEND", "TASK_UNION", "TASK_INTERSECTION", - "TASK_DIFFERENCE" + "TASK_DIFFERENCE", + "TASK_CONCAT" ], "custom": false, "documentationUrl": "https://www.instill.tech/docs/component/generic/collection", diff --git a/pkg/component/generic/collection/v0/config/tasks.json b/pkg/component/generic/collection/v0/config/tasks.json index 3d260d248..33aaaacd4 100644 --- a/pkg/component/generic/collection/v0/config/tasks.json +++ b/pkg/component/generic/collection/v0/config/tasks.json @@ -316,5 +316,62 @@ "title": "Output", "type": "object" } + }, + "TASK_CONCAT": { + "instillShortDescription": "Concatenate the arrays. i.e. `[1, 2] + [3, 4] = [1, 2, 3, 4]`", + "input": { + "description": "Input", + "instillEditOnNodeFields": [ + "arrays" + ], + "instillUIOrder": 0, + "properties": { + "arrays": { + "description": "The arrays to be concatenated.", + "instillAcceptFormats": [ + "array:array:*" + ], + "instillUIMultiline": true, + "instillUIOrder": 0, + "instillUpstreamTypes": [ + "value", + "reference", + "template" + ], + "items": {}, + "required": [], + "title": "Arrays", + "type": "array" + } + }, + "required": [ + "arrays" + ], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillEditOnNodeFields": [ + "array" + ], + "instillUIOrder": 0, + "properties": { + "array": { + "description": "The concatenated arrays.", + "instillEditOnNodeFields": [], + "instillFormat": "array:*", + "instillUIOrder": 0, + "required": [], + "title": "Array", + "type": "array" + } + }, + "required": [ + "array" + ], + "title": "Output", + "type": "object" + } } } diff --git a/pkg/component/generic/collection/v0/main.go b/pkg/component/generic/collection/v0/main.go index 76ac74ad6..63b6a0c8d 100644 --- a/pkg/component/generic/collection/v0/main.go +++ b/pkg/component/generic/collection/v0/main.go @@ -1,4 +1,4 @@ -//go:generate compogen readme ./config ./README.mdx +//go:generate compogen readme ./config ./README.mdx --extraContents bottom=.compogen/bottom.mdx package collection import ( @@ -23,6 +23,7 @@ const ( taskIntersection = "TASK_INTERSECTION" taskDifference = "TASK_DIFFERENCE" taskAppend = "TASK_APPEND" + taskConcat = "TASK_CONCAT" ) var ( @@ -73,6 +74,8 @@ func (c *component) CreateExecution(x base.ComponentExecution) (base.IExecution, e.execute = e.difference case taskAppend: e.execute = e.append + case taskConcat: + e.execute = e.concat default: return nil, errmsg.AddMessage( fmt.Errorf("not supported task: %s", x.Task), @@ -82,6 +85,19 @@ func (c *component) CreateExecution(x base.ComponentExecution) (base.IExecution, return e, nil } +func (e *execution) concat(in *structpb.Struct) (*structpb.Struct, error) { + arrays := in.Fields["arrays"].GetListValue().Values + concat := &structpb.ListValue{Values: []*structpb.Value{}} + + for _, a := range arrays { + concat.Values = append(concat.Values, a.GetListValue().Values...) + } + + out := &structpb.Struct{Fields: make(map[string]*structpb.Value)} + out.Fields["array"] = structpb.NewListValue(concat) + return out, nil +} + func (e *execution) union(in *structpb.Struct) (*structpb.Struct, error) { sets := in.Fields["sets"].GetListValue().Values cache := [][]string{} diff --git a/pkg/component/generic/collection/v0/main_test.go b/pkg/component/generic/collection/v0/main_test.go new file mode 100644 index 000000000..79aed5fd4 --- /dev/null +++ b/pkg/component/generic/collection/v0/main_test.go @@ -0,0 +1,62 @@ +package collection + +import ( + "testing" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + qt "github.com/frankban/quicktest" +) + +func Test_Concat(t *testing.T) { + c := qt.New(t) + + c.Run("Concat", func(c *qt.C) { + e := &execution{} + + inputArrays := []*structpb.Value{ + structpb.NewListValue(&structpb.ListValue{ + Values: []*structpb.Value{ + structpb.NewStringValue("a"), + structpb.NewStringValue("b"), + }, + }), + structpb.NewListValue(&structpb.ListValue{ + Values: []*structpb.Value{ + structpb.NewStringValue("c"), + structpb.NewStringValue("d"), + }, + }), + } + + inputStruct := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "arrays": structpb.NewListValue(&structpb.ListValue{ + Values: inputArrays, + }), + }, + } + + expectedOutput := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "array": structpb.NewListValue(&structpb.ListValue{ + Values: []*structpb.Value{ + structpb.NewStringValue("a"), + structpb.NewStringValue("b"), + structpb.NewStringValue("c"), + structpb.NewStringValue("d"), + }, + }), + }, + } + + out, err := e.concat(inputStruct) + + c.Assert(err, qt.IsNil) + + c.Assert(proto.Equal(out, expectedOutput), qt.IsTrue) + + }) + +} diff --git a/pkg/component/generic/restapi/v0/README.mdx b/pkg/component/generic/restapi/v0/README.mdx index 53a3b7d42..e74fcf657 100644 --- a/pkg/component/generic/restapi/v0/README.mdx +++ b/pkg/component/generic/restapi/v0/README.mdx @@ -15,14 +15,21 @@ It can carry out the following tasks: - [Head](#head) - [Options](#options) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/restapi/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/restapi/v0/config/tasks.json) files respectively. + + + ## Setup @@ -288,3 +295,5 @@ Send a HTTP OPTIONS request. | Body | `body` | any | The body of the response | | Header | `header` | object | The HTTP header of the response | + + diff --git a/pkg/component/operator/audio/v0/README.mdx b/pkg/component/operator/audio/v0/README.mdx index d0f187730..82da2651e 100644 --- a/pkg/component/operator/audio/v0/README.mdx +++ b/pkg/component/operator/audio/v0/README.mdx @@ -10,16 +10,23 @@ It can carry out the following tasks: - [Chunk Audios](#chunk-audios) - [Slice Audio](#slice-audio) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/audio/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/audio/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Chunk Audios @@ -72,6 +79,8 @@ Specify a time range to slice an audio file | :--- | :--- | :--- | :--- | | Audio | `audio` | string | Base64 encoded audio slice | + + ## Example Recipes Recipe for the [Audio Transcription Generator](https://instill.tech/instill-ai/pipelines/audio-transcription/playground) pipeline. diff --git a/pkg/component/operator/base64/v0/README.mdx b/pkg/component/operator/base64/v0/README.mdx index d9179da47..e83c1b46d 100644 --- a/pkg/component/operator/base64/v0/README.mdx +++ b/pkg/component/operator/base64/v0/README.mdx @@ -10,16 +10,23 @@ It can carry out the following tasks: - [Encode](#encode) - [Decode](#decode) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/base64/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/base64/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Encode @@ -69,3 +76,5 @@ Decode the base64 string. | :--- | :--- | :--- | :--- | | Data | `data` | string | Data | + + diff --git a/pkg/component/operator/document/v0/.compogen/bottom.mdx b/pkg/component/operator/document/v0/.compogen/bottom.mdx index ba1f7bbaa..8151ac812 100644 --- a/pkg/component/operator/document/v0/.compogen/bottom.mdx +++ b/pkg/component/operator/document/v0/.compogen/bottom.mdx @@ -59,4 +59,4 @@ output: contract_summary: title: Contract Summary value: ${gpt-4-summary.output.texts} -``` \ No newline at end of file +``` diff --git a/pkg/component/operator/document/v0/README.mdx b/pkg/component/operator/document/v0/README.mdx index 4dc150aad..c64353679 100644 --- a/pkg/component/operator/document/v0/README.mdx +++ b/pkg/component/operator/document/v0/README.mdx @@ -11,16 +11,23 @@ It can carry out the following tasks: - [Convert to Text](#convert-to-text) - [Convert to Images](#convert-to-images) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/document/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/document/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Convert to Markdown @@ -108,6 +115,8 @@ Convert Document to images. | Images | `images` | array[string] | Images converted from the document | | Filenames (optional) | `filenames` | array[string] | The filenames of the images. The filenames will be appended with the page number. e.g. 'example-1.jpg' | + + ## Example Recipes Recipe for the [Content Reviewer](https://instill.tech/instill-ai/pipelines/contract-reviewer/playground) pipeline. @@ -169,4 +178,4 @@ output: contract_summary: title: Contract Summary value: ${gpt-4-summary.output.texts} -``` \ No newline at end of file +``` diff --git a/pkg/component/operator/image/v0/README.mdx b/pkg/component/operator/image/v0/README.mdx index 6b1d3eec6..9d1ef152e 100644 --- a/pkg/component/operator/image/v0/README.mdx +++ b/pkg/component/operator/image/v0/README.mdx @@ -17,16 +17,23 @@ It can carry out the following tasks: - [Draw Instance Segmentation](#draw-instance-segmentation) - [Draw Semantic Segmentation](#draw-semantic-segmentation) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/image/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/image/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Concat @@ -407,3 +414,5 @@ A list of RLE binary masks. | :--- | :--- | :--- | :--- | | Image | `image` | string | Output image | + + diff --git a/pkg/component/operator/json/v0/README.mdx b/pkg/component/operator/json/v0/README.mdx index 93d8b7499..f3def7e45 100644 --- a/pkg/component/operator/json/v0/README.mdx +++ b/pkg/component/operator/json/v0/README.mdx @@ -11,16 +11,23 @@ It can carry out the following tasks: - [Unmarshal](#unmarshal) - [jq](#jq) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/json/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/json/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Marshal @@ -118,6 +125,8 @@ to access such fields in a `jq` filter is by wrapping the key in double quotes: | Input JSON | `jq` filter | Output | | :--- | :--- | :--- | | `[{"key-a": "value1"}, {"key-a": "value2"}]` | `.[] \| ."key-a"` | `["value1", "value2"]` | + + ## Example Recipes Recipe for the [Resume Screening](https://instill.tech/instill-ai/pipelines/structured-resume-screening/playground) pipeline. diff --git a/pkg/component/operator/text/v0/README.mdx b/pkg/component/operator/text/v0/README.mdx index e2cb9077a..ecacc8520 100644 --- a/pkg/component/operator/text/v0/README.mdx +++ b/pkg/component/operator/text/v0/README.mdx @@ -9,16 +9,23 @@ The Text component is an operator component that allows users to extract and man It can carry out the following tasks: - [Chunk Text](#chunk-text) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/text/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/text/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Chunk Text @@ -130,3 +137,5 @@ This text splitter is specially designed for Markdown format. | Token Count | `token-count` | integer | Count of tokens in a chunk | + + diff --git a/pkg/component/operator/video/v0/README.mdx b/pkg/component/operator/video/v0/README.mdx index d19f74dc5..f24196459 100644 --- a/pkg/component/operator/video/v0/README.mdx +++ b/pkg/component/operator/video/v0/README.mdx @@ -10,16 +10,23 @@ It can carry out the following tasks: - [Subsample Video](#subsample-video) - [Subsample Video Frames](#subsample-video-frames) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/video/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/video/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Subsample Video @@ -75,3 +82,5 @@ Subsample video into frames | :--- | :--- | :--- | :--- | | Frames | `frames` | array[string] | Base64 encoded sub-sampled frames | + + diff --git a/pkg/component/operator/web/v0/README.mdx b/pkg/component/operator/web/v0/README.mdx index 5ad402ab0..a94fa6d83 100644 --- a/pkg/component/operator/web/v0/README.mdx +++ b/pkg/component/operator/web/v0/README.mdx @@ -11,16 +11,23 @@ It can carry out the following tasks: - [Scrape Page](#scrape-page) - [Scrape Sitemap](#scrape-sitemap) + + ## Release Stage `Alpha` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/web/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/web/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Crawl Site @@ -177,6 +184,8 @@ This task extracts data directly from a website’s sitemap. A sitemap is typica + + ## Example Recipes ```yaml diff --git a/pkg/component/tools/compogen/cmd/testdata/readme-component1.txt b/pkg/component/tools/compogen/cmd/testdata/readme-component1.txt index b6d79cfb0..7cffd7e48 100644 --- a/pkg/component/tools/compogen/cmd/testdata/readme-component1.txt +++ b/pkg/component/tools/compogen/cmd/testdata/readme-component1.txt @@ -288,14 +288,21 @@ The Dummy component is a data component that allows users to perform an action. It can carry out the following tasks: - [Dummy](#dummy) + + ## Release Stage `Coming Soon` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/pkg/component/blob/main/data/dummy/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/pkg/component/blob/main/data/dummy/v0/config/tasks.json) files respectively. + + + ## Setup An account at [dummy.io](https://dummy.io) is required. @@ -441,3 +448,5 @@ This text splitter is specially designed for Markdown format. | Replied Message | `message` | string | message to reply a conversation | + + diff --git a/pkg/component/tools/compogen/cmd/testdata/readme-component2.txt b/pkg/component/tools/compogen/cmd/testdata/readme-component2.txt index a983b8bb6..ecfc7ea59 100644 --- a/pkg/component/tools/compogen/cmd/testdata/readme-component2.txt +++ b/pkg/component/tools/compogen/cmd/testdata/readme-component2.txt @@ -188,16 +188,23 @@ It can carry out the following tasks: - [Dummy](#dummy) - [Dummier](#dummier) + + ## Release Stage `Beta` + + ## Configuration The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/pkg/component/blob/main/operator/dummy/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/pkg/component/blob/main/operator/dummy/v0/config/tasks.json) files respectively. + + + ## Supported Tasks ### Dummy @@ -269,6 +276,8 @@ This task is dummier than `TASK_DUMMY`. | Tincidunt tortor | `tortor` | string | Tincidunt tortor aliquam nulla | + + ## Final words Thanks for reaching this point! No one really reads documentation thoroughly (: diff --git a/pkg/component/tools/compogen/pkg/gen/resources/templates/readme.mdx.tmpl b/pkg/component/tools/compogen/pkg/gen/resources/templates/readme.mdx.tmpl index 85ff5be15..f207af086 100644 --- a/pkg/component/tools/compogen/pkg/gen/resources/templates/readme.mdx.tmpl +++ b/pkg/component/tools/compogen/pkg/gen/resources/templates/readme.mdx.tmpl @@ -10,19 +10,20 @@ It can carry out the following tasks: {{- range .Tasks }} - [{{ .Title }}](#{{ asAnchor .Title}}){{- end }} -{{- loadExtraContent "intro" }} +{{ loadExtraContent "intro" }} ## Release Stage `{{ .ReleaseStage.String }}` -{{- loadExtraContent "release" }} +{{ loadExtraContent "release" }} ## Configuration The component definition and tasks are defined in the [definition.json]({{ .SourceURL }}/config/definition.json) and [tasks.json]({{ .SourceURL }}/config/tasks.json) files respectively. -{{- loadExtraContent "config" }} +{{ loadExtraContent "config" }} + {{ if or .SetupConfig.Prerequisites .SetupConfig.Properties }} ## Setup {{ with .SetupConfig.Prerequisites}} @@ -181,6 +182,7 @@ ${connection.}`. {{- end }} {{ end -}} -{{- loadExtraContent $task.ID -}} -{{- end -}} -{{- loadExtraContent "bottom" -}} +{{ loadExtraContent $task.ID -}} +{{- end }} + +{{ loadExtraContent "bottom" -}}