-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into sdg-rename-notebooks
- Loading branch information
Showing
30 changed files
with
3,531 additions
and
36 deletions.
There are no files selected for viewing
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
11 changes: 11 additions & 0 deletions
11
cli/foundation-models/system/inference/visual-chat-completion/deploy.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json | ||
| name: phi-3-vision | ||
| instance_type: Standard_NC48ads_A100_v4 | ||
| instance_count: 1 | ||
| liveness_probe: | ||
| initial_delay: 180 | ||
| period: 180 | ||
| failure_threshold: 49 | ||
| timeout: 299 | ||
| request_settings: | ||
| request_timeout_ms: 180000 |
79 changes: 79 additions & 0 deletions
79
cli/foundation-models/system/inference/visual-chat-completion/online-endpoint-deployment.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| set -x | ||
|
|
||
| # script inputs | ||
| subscription_id="<SUBSCRIPTION_ID>" | ||
| resource_group_name="<RESOURCE_GROUP>" | ||
| workspace_name="<WORKSPACE_NAME>" | ||
|
|
||
| # This is the model from system registry that needs to be deployed | ||
| registry_name="azureml" | ||
| model_name="Phi-3-vision-128k-instruct" | ||
| deployment_sku="Standard_NC48ads_A100_v4" | ||
|
|
||
| # Validate the existence of the model in the registry and get the latest version | ||
| model_list=$(az ml model list --name ${model_name} --registry-name ${registry_name} 2>&1) | ||
| if [[ ${model_list} == *"[]"* ]]; then | ||
| echo "Model doesn't exist in registry. Check the model list and try again."; exit 1; | ||
| fi | ||
| version_temp=${model_list#*\"version\": \"} | ||
| model_version=${version_temp%%\"*} | ||
|
|
||
| version=$(date +%s) | ||
| endpoint_name="phi-3-mini-v-instruct-$version" | ||
|
|
||
| # scoring_file | ||
| scoring_file="./sample_chat_completions_score.json" | ||
|
|
||
| # 1. Setup pre-requisites | ||
| if [ "$subscription_id" = "<SUBSCRIPTION_ID>" ] || \ | ||
| ["$resource_group_name" = "<RESOURCE_GROUP>" ] || \ | ||
| [ "$workspace_name" = "<WORKSPACE_NAME>" ]; then | ||
| echo "Please update the script with the subscription_id, resource_group_name and workspace_name" | ||
| exit 1 | ||
| fi | ||
|
|
||
| az account set -s $subscription_id | ||
| workspace_info="--resource-group $resource_group_name --workspace-name $workspace_name" | ||
|
|
||
| # 2. Check if the model exists in the registry | ||
| # need to confirm model show command works for registries outside the tenant (aka system registry) | ||
| if ! az ml model show --name $model_name --version $model_version --registry-name $registry_name | ||
| then | ||
| echo "Model $model_name:$model_version does not exist in registry $registry_name" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 3. Deploy the model to an endpoint | ||
| # create online endpoint | ||
| az ml online-endpoint create --name $endpoint_name $workspace_info || { | ||
| echo "endpoint create failed"; exit 1; | ||
| } | ||
|
|
||
| # deploy model from registry to endpoint in workspace | ||
| az ml online-deployment create --file deploy.yml $workspace_info --all-traffic --set \ | ||
| endpoint_name=$endpoint_name model=azureml://registries/$registry_name/models/$model_name/versions/$model_version \ | ||
| instance_type=$deployment_sku || { | ||
| echo "deployment create failed"; exit 1; | ||
| } | ||
|
|
||
| # 4. Try a sample scoring request | ||
|
|
||
| # Check if scoring data file exists | ||
| if [ -f $scoring_file ]; then | ||
| echo "Invoking endpoint $endpoint_name with following input:\n\n" | ||
| cat $scoring_file | ||
| echo "\n\n" | ||
| else | ||
| echo "Scoring file $scoring_file does not exist" | ||
| exit 1 | ||
| fi | ||
|
|
||
| az ml online-endpoint invoke --name $endpoint_name --request-file $scoring_file $workspace_info || { | ||
| echo "endpoint invoke failed"; exit 1; | ||
| } | ||
|
|
||
| # 6. Delete the endpoint | ||
| az ml online-endpoint delete --name $endpoint_name $workspace_info --yes || { | ||
| echo "endpoint delete failed"; exit 1; | ||
| } | ||
|
|
22 changes: 22 additions & 0 deletions
22
...ndation-models/system/inference/visual-chat-completion/sample_chat_completions_score.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| { | ||
| "input_data": { | ||
| "input_string": [ | ||
| { | ||
| "role": "user", | ||
| "content": [ | ||
| { | ||
| "type": "image_url", | ||
| "image_url": { | ||
| "url": "https://www.ilankelman.org/stopsigns/australia.jpg" | ||
| } | ||
| }, | ||
| { | ||
| "type": "text", | ||
| "text": "What is shown in this image? Be extremely detailed and specific." | ||
| } | ||
| ] | ||
| } | ||
| ], | ||
| "parameters": { "temperature": 0.7, "max_new_tokens": 2048 } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# Use litellm with JAIS in Azure AI and Azure ML\n", | ||
| "\n", | ||
| "Use `litellm` to consume JAIS deployments in Azure AI and Azure ML. Notice that JAIS in Azure only supports chat completions API.\n", | ||
| "\n", | ||
| "> Review the [documentation](https://aka.ms/jais-azure-ai-studio-docs) for the JAIS 30b Chat model in AI Studio and for ML Studio for details on how to provision inference endpoints, regional availability, pricing and inference schema reference." | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Prerequisites\n", | ||
| "\n", | ||
| "Before we start, there are certain steps we need to take to deploy the models:\n", | ||
| "\n", | ||
| "* Register for a valid Azure account with subscription \n", | ||
| "* Make sure you have access to [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio?tabs=home)\n", | ||
| "* Create a project and resource group\n", | ||
| "* Select `Jais-30b-Chat` in the model catalog.\n", | ||
| "\n", | ||
| " > Notice that some models may not be available in all the regions in Azure AI and Azure Machine Learning. On those cases, you can create a workspace or project in the region where the models are available and then consume it with a connection from a different one. To learn more about using connections see [Consume models with connections](https://learn.microsoft.com/en-us/azure/ai-studio//deployments-connections)\n", | ||
| "\n", | ||
| "* Deploy with \"Serverless APIs\"\n", | ||
| "\n", | ||
| "Once deployed successfully, you should be assigned for an API endpoint and a security key for inference.\n", | ||
| "\n", | ||
| "For more information, you should consult Azure's official documentation [here](https://aka.ms/jais-azure-ai-studio-docs) for model deployment and inference.\n", | ||
| "\n", | ||
| "To complete this tutorial, you will need to:\n", | ||
| "\n", | ||
| "* Install `litellm`:\n", | ||
| "\n", | ||
| " ```bash\n", | ||
| " pip install litellm\n", | ||
| " ```" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Example\n", | ||
| "\n", | ||
| "The following is an example about how to use `litellm` with a JAIS model deployed in Azure AI and Azure ML:" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": { | ||
| "name": "imports" | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import litellm" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "You will need to have a Endpoint url and Authentication Key associated with that endpoint. This can be acquired from previous steps. To work with `litellm`, configure the client as follows:\n", | ||
| "\n", | ||
| "- `base_url`: Use the endpoint URL from your deployment. Include the `/v1` in the URL.\n", | ||
| "- `api_key`: Use your API key." | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": { | ||
| "name": "chat_client" | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "client = litellm.LiteLLM(\n", | ||
| " base_url=\"https://<endpoint-name>.<region>.inference.ai.azure.com/v1\",\n", | ||
| " api_key=\"<key>\",\n", | ||
| ")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "Use the client to create chat completions requests:" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": { | ||
| "name": "chat_invoke" | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "response = client.chat.completions.create(\n", | ||
| " messages=[{\"content\": \"List the emirates of the UAE.\", \"role\": \"user\"}],\n", | ||
| " model=\"openai\",\n", | ||
| " custom_llm_provider=\"custom_openai\",\n", | ||
| ")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "The generated text can be accessed as follows:" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": { | ||
| "name": "chat_response" | ||
| }, | ||
| "outputs": [], | ||
| "source": [ | ||
| "print(response.choices[0].message.content)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Aditional resources\n", | ||
| "\n", | ||
| "Here are some additional reference: \n", | ||
| "\n", | ||
| "* [Plan and manage costs (marketplace)](https://learn.microsoft.com/azure/ai-studio//costs-plan-manage#monitor-costs-for-models-offered-through-the-azure-marketplace)" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3.10 - SDK v2", | ||
| "language": "python", | ||
| "name": "python310-sdkv2" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.10.11" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 2 | ||
| } |
Oops, something went wrong.