From c09162ed1228a9d959ca4497b21473067d83f06d Mon Sep 17 00:00:00 2001 From: Basilio Bogado <541149+basiliskus@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:23:17 -0700 Subject: [PATCH 1/3] Update env variable name for automated test sender (#1490) * Update env variable name and key for automated test sender * Updated env variable name * Added pull_request trigger for testing * Removed pull_request trigger --- .github/workflows/automated-staging-test-submit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-staging-test-submit.yml b/.github/workflows/automated-staging-test-submit.yml index 20f367c39..412bcd66d 100644 --- a/.github/workflows/automated-staging-test-submit.yml +++ b/.github/workflows/automated-staging-test-submit.yml @@ -23,7 +23,7 @@ jobs: - name: Write private key to file run: | - echo "${{ secrets.AUTOMATED_STAGING_RS_INTEGRATION_PRIVATE_KEY }}" > /tmp/staging_private_key.pem + echo "${{ secrets.SIMULATED_SENDER_STAGING_PRIVATE_KEY }}" > /tmp/staging_private_key.pem chmod 600 /tmp/staging_private_key.pem - name: Send HL7 sample messages to staging RS From bfdafa4ba9303feec837eff9940eb910eb4379c5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 21:25:06 +0000 Subject: [PATCH 2/3] Update dependency ch.qos.logback:logback-classic to v1.5.12 (#1494) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- shared/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/build.gradle b/shared/build.gradle index d44257a2c..9ba99ae91 100644 --- a/shared/build.gradle +++ b/shared/build.gradle @@ -18,7 +18,7 @@ dependencies { // logging implementation 'org.slf4j:slf4j-api:2.0.16' - implementation 'ch.qos.logback:logback-classic:1.5.11' + implementation 'ch.qos.logback:logback-classic:1.5.12' implementation 'net.logstash.logback:logstash-logback-encoder:8.0' //jackson From e1b9cf5f60ae391884af6e35e3c41278253832ea Mon Sep 17 00:00:00 2001 From: Basilio Bogado <541149+basiliskus@users.noreply.github.com> Date: Mon, 28 Oct 2024 08:12:18 -0700 Subject: [PATCH 3/3] Refactoring of hurl scripts (#1475) * Moved and renamed hrl script to simplify and clean up file org * Simplify command hurl parameter + cleanup * Refactored to make more generic and reusable functions * Renamed message_submission_utils.sh => utils.sh * Renamed message_submission_utils.sh => utils.sh * Refactored to extract functions and reuse utils + clean up * [WIP] Added handling of deployed environments for submit_message.sh * Refactored to handle deployed environments, change parameters and variable naming for consistency, and clean up * Added error handling for hurl calls, add user prompt for outbound submission id when not possible to get, plus clean up * Typo fix * Updated readme to add deployed env options * Added handling of empty ti client private key and improved help messages --- scripts/hurl/README.md | 151 +++++++++++- scripts/hurl/epic.sh | 19 ++ scripts/hurl/epic/README.md | 10 - scripts/hurl/epic/hrl | 13 -- scripts/hurl/message_submission_utils.sh | 158 ------------- scripts/hurl/rs.sh | 103 ++++++++ scripts/hurl/rs/README.md | 51 ---- scripts/hurl/rs/hrl | 131 ----------- scripts/hurl/submit_message.sh | 68 +++++- scripts/hurl/ti.sh | 95 ++++++++ scripts/hurl/ti/README.md | 55 ----- scripts/hurl/ti/hrl | 115 --------- scripts/hurl/update_examples.sh | 9 +- scripts/hurl/utils.sh | 284 +++++++++++++++++++++++ 14 files changed, 711 insertions(+), 551 deletions(-) create mode 100755 scripts/hurl/epic.sh delete mode 100644 scripts/hurl/epic/README.md delete mode 100755 scripts/hurl/epic/hrl delete mode 100755 scripts/hurl/message_submission_utils.sh create mode 100755 scripts/hurl/rs.sh delete mode 100644 scripts/hurl/rs/README.md delete mode 100755 scripts/hurl/rs/hrl create mode 100755 scripts/hurl/ti.sh delete mode 100644 scripts/hurl/ti/README.md delete mode 100755 scripts/hurl/ti/hrl create mode 100755 scripts/hurl/utils.sh diff --git a/scripts/hurl/README.md b/scripts/hurl/README.md index aba1fb1ca..576a9fb75 100644 --- a/scripts/hurl/README.md +++ b/scripts/hurl/README.md @@ -5,24 +5,157 @@ - [hurl](https://hurl.dev/) - [jq](https://jqlang.github.io/jq/) - [azure-cli](https://learn.microsoft.com/en-us/cli/azure/) +- [jwt-cli](https://github.com/mike-engel/jwt-cli) - `CDCTI_HOME` environment variable ([see here](../README.md)) ## Available Hurl Scripts -- [ReportStream](./rs/): scripts to send requests to ReportStream's endpoints -- [CDC Intermediary](./ti/): scripts to send requests to the CDC Intermediary's endpoints -- [Epic/UCSD](./epic/): scripts to send requests to Epic endpoints for UCSD +### ReportStream -## Local Submission Scripts +#### Usage -- `submit_message.sh`: sends a HL7 message to a locally running RS instance. It also grabs the snapshots of the file in azurite after converting to FHIR, after applying transformations in TI, and after converting back to HL7. It copies these files to the same folder where the submitted file is +``` +Usage: ./rs.sh [OPTIONS] + +ENDPOINT_NAME: + The name of the endpoint to call (required) + +Options: + -f Path to the hl7/fhir file to submit (Required for waters API) + -r Root path to the hl7/fhir files (Default: /Users/bbogado/Code/Flexion/CDC-TI/trusted-intermediary/examples/) + -t Content type for the message (Default: application/hl7-v2) + -e Environment: local|staging|production (Default: local) + -c Client ID (Default: flexion) + -s Client sender (Default: simulated-sender) + -k Path to the client private key (Required for non-local environments) + -i Submission ID for history API (Required for history API) + -v Verbose mode + -h Display this help and exit + +Environment Variables: + CDCTI_HOME Base directory for CDC TI repository (Required) +``` + +#### Examples + +Sending an order to local environment + +``` +./rs.sh waters -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 +``` + +Sending a result to local environment + +``` +./rs.sh waters -f Test/Results/002_AL_ORU_R01_NBS_Fully_Populated_0_initial_message.hl7 +``` + +Sending an order to staging + +``` +./rs.sh waters -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 -e staging -k /path/to/client/staging/private/key +``` + +Checking the history in local environment for a submission id + +``` +./rs.sh history -i 100 +``` + +Checking the history in staging for a submission id + +``` +./rs.sh history -i 100 -e staging -k /path/to/client/staging/private/key +``` + +### CDC Intermediary + +#### Usage + +``` +Usage: ./ti.sh [OPTIONS] + +ENDPOINT_NAME: + The name of the endpoint to call (required) + +Options: + -f Path to the hl7/fhir file to submit (Required for orders and results APIs) + -r Root path to the hl7/fhir files (Default: /Users/bbogado/Code/Flexion/CDC-TI/trusted-intermediary/examples/) + -e Environment: local|staging (Default: local) + -c Client ID to create JWT with (Default: report-stream) + -k Path to the client private key (Required for non-local environments) + -i Submission ID for metadata API (Required for orders, results and metadata API) + -v Verbose mode + -h Display this help and exit + +Environment Variables: + CDCTI_HOME Base directory for CDC TI repository (Required) +``` + +#### Examples + +Submit an order to local environment: +``` +./ti.sh orders -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_1_hl7_translation.fhir -i 100 +``` + +Submit an order to staging: +``` +./ti.sh orders -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 -e staging -k /path/to/client/staging/private/key + +``` + +Submit a result to local environment: +``` +./ti.sh results -f Test/Results/002_AL_ORU_R01_NBS_Fully_Populated_1_hl7_translation.fhir -i 100 +``` + +Get metadata from local environment: +``` +./ti.sh metadata -i 100 +``` + +Authenticate to local environment: +``` +./ti.sh auth +``` + +Get OpenAPI docs from local environment: +``` +./ti.rs openapi +``` + +Get Health info from local environment: +``` +./ti.sh health +``` + +### Epic/UCSD + +#### Before running the script + +- Add the `client` id to `epic.rs` +- Update the `secret` variable path + +#### Usage + +`./epic.sh results` + +## High Level Scripts + +- `submit_message.sh`: sends a HL7 message to RS and tracks its status throughout the flow until final delivery. When running locally, it grabs the snapshots of the file in azurite after converting to FHIR, after applying transformations in TI, and after converting back to HL7; and it copies those files to the same folder where the submitted file is. If running in a deployed environment we currently don't have a way to download the files from Azure, but the script will print the relative path for the files in the blob storage container. ``` - ./submit_message.sh /path/to/message.hl7 + Usage: submit_message.sh -f [-e ] + + Options: + -f Message file path (Required) + -e Environment: local|staging|production (Default: ) + -x Path to the client private key for authentication with RS API (Required for non-local environments) + -z Path to the client private key for authentication with TI API (Optional for all environments) + -h Display this help and exit ``` - `update_examples.sh`: sends all the HL7 files with `_0_initial_message.hl7` suffix in the `/examples` folder to a locally running RS instance. As the previous script, it copies the snapshots at each stage ``` ./update_examples.sh ``` -- `message_submission_utils.sh`: utility functions for the previous scripts. It has functions to submit requests to RS, check the submission status throughout the whole flow, and downloading snapshots from azurite - -**Note**: these scripts require both RS and TI to be running locally +- `utils.sh`: utility functions for the previous scripts. It has functions to submit requests to RS, check the submission status throughout the whole flow, and downloading snapshots from azurite diff --git a/scripts/hurl/epic.sh b/scripts/hurl/epic.sh new file mode 100755 index 000000000..57835798d --- /dev/null +++ b/scripts/hurl/epic.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +source ./utils.sh + +client= +audience=https://epicproxy-np.et0502.epichosted.com/FhirProxy/oauth2/token +secret=/path/to/ucsd-epic-private-key.pem +root=$CDCTI_HOME/examples/CA/ +fpath="$1" +shift + +jwt_token=$(generate_jwt "$client" "$audience" "$secret") || fail "Failed to generate JWT token" + +hurl \ + --variable "fpath=$fpath" \ + --file-root "$root" \ + --variable "jwt=$jwt_token" \ + epic/results.hurl \ + $@ diff --git a/scripts/hurl/epic/README.md b/scripts/hurl/epic/README.md deleted file mode 100644 index c66c84aa8..000000000 --- a/scripts/hurl/epic/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Epic/UCSD Hurl Script - -## Before running the script - -- Add the `client` id to `.hrl` -- Update the `secret` variable path - -## Usage - -`./hrl ` diff --git a/scripts/hurl/epic/hrl b/scripts/hurl/epic/hrl deleted file mode 100755 index 06c0cb53a..000000000 --- a/scripts/hurl/epic/hrl +++ /dev/null @@ -1,13 +0,0 @@ -client= -audience=https://epicproxy-np.et0502.epichosted.com/FhirProxy/oauth2/token -secret=/path/to/ucsd-epic-private-key.pem -root=$CDCTI_HOME/examples/CA/ -fpath="$1" -shift - -hurl \ - --variable fpath=$fpath \ - --file-root $root \ - --variable jwt=$(jwt encode --exp='+5min' --jti $(uuidgen) --alg RS256 -k $client -i $client -s $client -a $audience --no-iat -S @$secret) \ - results.hurl \ - $@ diff --git a/scripts/hurl/message_submission_utils.sh b/scripts/hurl/message_submission_utils.sh deleted file mode 100755 index ea5a0abab..000000000 --- a/scripts/hurl/message_submission_utils.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -RS_HRL_SCRIPT_PATH="$CDCTI_HOME/scripts/hurl/rs" -TI_HRL_SCRIPT_PATH="$CDCTI_HOME/scripts/hurl/ti" -FILE_NAME_SUFFIX_STEP_0="_0_initial_message" -FILE_NAME_SUFFIX_STEP_1="_1_hl7_translation" -FILE_NAME_SUFFIX_STEP_2="_2_fhir_transformation" -FILE_NAME_SUFFIX_STEP_3="_3_hl7_translation_final" - -AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" # pragma: allowlist secret - -RS_LOCAL_API="http://localhost:7071" -TI_LOCAL_API="http://localhost:8080" - -check_prerequisites() { - # Check if the RS and TO local APIs are reachable - local apis=("$RS_LOCAL_API" "$TI_LOCAL_API") - for service in "${apis[@]}"; do - if ! curl -s --head --fail "$service" | grep "200 OK" >/dev/null; then - echo "The service at $service is not reachable" - exit 1 - fi - done - - # Check required CLI tools - local required_commands=("hurl" "jq" "az") - for cmd in "${required_commands[@]}"; do - if ! command -v "$cmd" &>/dev/null; then - echo "$cmd could not be found. Please install $cmd to proceed." - exit 1 - fi - done -} - -check_submission_status() { - local submission_id=$1 - local timeout=180 # 3 minutes - local retry_interval=10 # Retry every 10 seconds - - start_time=$(date +%s) - - while true; do - history_response=$( - cd "$RS_HRL_SCRIPT_PATH" || exit 1 - ./hrl history.hurl -i "$submission_id" - ) - overall_status=$(echo "$history_response" | jq -r '.overallStatus') - - echo -n " Status: $overall_status" - if [ "$overall_status" == "Delivered" ]; then - echo "!" - return 0 - else - echo ". Retrying in $retry_interval seconds..." - fi - - # Check if the timeout has been reached - current_time=$(date +%s) - elapsed_time=$((current_time - start_time)) - if [ "$elapsed_time" -ge "$timeout" ]; then - echo " Timeout reached after $elapsed_time seconds. Status is still: $overall_status." - return 1 - fi - - sleep $retry_interval - done -} - -extract_submission_id() { - local history_response=$1 - echo "$history_response" | jq '.destinations[0].sentReports[0].externalName' | sed 's/.*-\([0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}\)-.*/\1/' -} - -download_from_azurite() { - local blob_name=$1 - local file_path=$2 - - echo " Downloading from blob storage '$blob_name' to '$file_path'" - az storage blob download \ - --container-name "reports" \ - --name "$blob_name" \ - --file "$file_path" \ - --connection-string "$AZURITE_CONNECTION_STRING" \ - --output none || { - echo "Download failed for blob '$blob_name'." - exit 1 - } - - if [[ "$file_path" == *.fhir ]]; then - echo " Formatting the content for: $file_path" - formatted_content=$(jq '.' "$file_path") - echo "$formatted_content" >"$file_path" - fi -} - -submit_message() { - local file=$1 - local message_file_path=$(dirname "$file") - local message_file_name=$(basename "$file") - local message_base_name="${message_file_name%.hl7}" - message_base_name="${message_base_name%"$FILE_NAME_SUFFIX_STEP_0"}" - - msh9=$(awk -F'|' '/^MSH/ { print $9 }' "$file") - case "$msh9" in - "ORU^R01^ORU_R01") - first_leg_receiver="flexion.etor-service-receiver-results" - second_leg_receiver="flexion.simulated-hospital" - ;; - "OML^O21^OML_O21" | "ORM^O01^ORM_O01") - first_leg_receiver="flexion.etor-service-receiver-orders" - second_leg_receiver="flexion.simulated-lab" - ;; - *) - echo "Unknown receivers for MSH-9 value '$msh9'. Skipping the message" - return - ;; - esac - - echo "Assuming receivers are '$first_leg_receiver' and '$second_leg_receiver' because of MSH-9 value '$msh9'" - - waters_response=$( - cd "$RS_HRL_SCRIPT_PATH" || exit 1 - ./hrl waters.hurl -f "$message_file_name" -r "$message_file_path" - ) - submission_id=$(echo "$waters_response" | jq -r '.id') - - echo "[First leg] Checking submission status for ID: $submission_id" - if ! check_submission_status "$submission_id"; then - echo "Failed to deliver the first leg of the message. Skipping the next steps." - return - fi - - inbound_submission_id=$(extract_submission_id "$history_response") - translated_blob_name="ready/$first_leg_receiver/$inbound_submission_id.fhir" - translated_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_1.fhir" - download_from_azurite "$translated_blob_name" "$translated_file_path" - - metadata_response=$( - cd "$TI_HRL_SCRIPT_PATH" || exit 1 - ./hrl metadata.hurl -i "$inbound_submission_id" - ) - outbound_submission_id=$(echo "$metadata_response" | jq -r '.issue[] | select(.details.text == "outbound submission id") | .diagnostics') - - transformed_blob_name="receive/flexion.etor-service-sender/$outbound_submission_id.fhir" - transformed_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_2.fhir" - download_from_azurite "$transformed_blob_name" "$transformed_file_path" - - echo "[Second leg] Checking submission status for ID: $outbound_submission_id" - if ! check_submission_status "$outbound_submission_id"; then - echo "Failed to deliver the second leg of the message. Skipping the next steps." - return - fi - - final_submission_id=$(extract_submission_id "$history_response") - final_blob_name="ready/$second_leg_receiver/$final_submission_id.hl7" - final_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_3.hl7" - download_from_azurite "$final_blob_name" "$final_file_path" -} diff --git a/scripts/hurl/rs.sh b/scripts/hurl/rs.sh new file mode 100755 index 000000000..39a65d669 --- /dev/null +++ b/scripts/hurl/rs.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +source ./utils.sh + +# default values +env=local +root=$CDCTI_HOME/examples/ +content_type=application/hl7-v2 +client_id=flexion +client_sender=simulated-sender + +show_usage() { + cat < [OPTIONS] + +ENDPOINT_NAME: + The name of the endpoint to call (required) + +Options: + -f Path to the hl7/fhir file to submit (Required for waters API) + -r Root path to the hl7/fhir files (Default: $root) + -t Content type for the message (Default: $content_type) + -e Environment: local|staging|production (Default: $env) + -c Client ID (Default: $client_id) + -s Client sender (Default: $client_sender) + -k Path to the client private key (Required for non-local environments) + -i Submission ID for history API (Required for history API) + -v Verbose mode + -h Display this help and exit + +Environment Variables: + CDCTI_HOME Base directory for CDC TI repository (Required) +EOF +} + +parse_arguments() { + if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + show_usage + exit 0 + fi + + [ $# -eq 0 ] && fail "Missing required argument " + endpoint_name="rs/$1.hurl" + shift # Remove endpoint name from args + + while getopts ':f:r:t:e:c:s:k:i:v' opt; do + case "$opt" in + f) fpath="$OPTARG" ;; + r) root="$OPTARG" ;; + t) content_type="$OPTARG" ;; + e) env="$OPTARG" ;; + c) client_id="$OPTARG" ;; + s) client_sender="$OPTARG" ;; + k) private_key="$OPTARG" ;; + i) submission_id="--variable submissionid=$OPTARG" ;; + v) verbose="--verbose" ;; + ?) fail "Invalid option -$OPTARG" ;; + esac + done + + shift "$(($OPTIND - 1))" + remaining_args="$*" +} + +setup_credentials() { + if [ -z "$private_key" ] && [ "$client_id" = "flexion" ] && [ "$env" = "local" ]; then + if [ -f "$RS_CLIENT_LOCAL_PRIVATE_KEY_PATH" ]; then + private_key="$RS_CLIENT_LOCAL_PRIVATE_KEY_PATH" + else + fail "Local environment client private key not found at: $RS_CLIENT_LOCAL_PRIVATE_KEY_PATH" + fi + fi + + if [ "$env" != "local" ]; then + [ -z "$private_key" ] && fail "Client private key (-k) is required for non-local environments" + fi + + [ ! -f "$private_key" ] && fail "Client private key file not found: $private_key" +} + +run_hurl_command() { + url=$(get_api_url "$env" "rs") + host=$(extract_host_from_url "$url") + jwt_token=$(generate_jwt "$client_id.$client_sender" "$host" "$private_key") || fail "Failed to generate JWT token" + + hurl \ + --variable "fpath=$fpath" \ + --file-root "$root" \ + --variable "url=$url" \ + --variable "content-type=$content_type" \ + --variable "client-id=$client_id" \ + --variable "client-sender=$client_sender" \ + --variable "jwt=$jwt_token" \ + ${submission_id:-} \ + ${verbose:-} \ + "$endpoint_name" \ + ${remaining_args:+$remaining_args} +} + +check_env_vars CDCTI_HOME +parse_arguments "$@" +setup_credentials +run_hurl_command diff --git a/scripts/hurl/rs/README.md b/scripts/hurl/rs/README.md deleted file mode 100644 index 98d39c9c9..000000000 --- a/scripts/hurl/rs/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# ReportStream Hurl Script - -## Usage - -``` -Usage: ./hrl [OPTIONS] - -Options: - -f The path to the hl7/fhir file to submit, relative the root path (Required for waters API) - -r The root path to the hl7/fhir files (Default: $CDCTI_HOME/examples/) - -t The content type for the message (e.g. 'application/hl7-v2' or 'application/fhir+ndjson') (Default: application/hl7-v2) - -e [local | staging | production ] The environment to run the test in (Default: local) - -c The client id to use (Default: flexion) - -s The client sender to use (Default: simulated-sender) - -x The path to the client private key for the environment - -i The submissionId to call the history API with (Required for history API) - -v Verbose mode - -h Display this help and exit -``` - -## Examples - -Sending an order to local environment - -``` -./hrl waters.hurl -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 -``` - -Sending a result to local environment - -``` -./hrl waters.hurl -f Test/Results/002_AL_ORU_R01_NBS_Fully_Populated_0_initial_message.hl7 -``` - -Sending an order to staging - -``` -./hrl waters.hurl -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 -e staging -x /path/to/staging/private/key -``` - -Checking the history in local environment for a submission id - -``` -./hrl history.hurl -i 100 -``` - -Checking the history in staging for a submission id - -``` -./hrl history.hurl -i 100 -e staging -x /path/to/staging/private/key -``` diff --git a/scripts/hurl/rs/hrl b/scripts/hurl/rs/hrl deleted file mode 100755 index 6383a4103..000000000 --- a/scripts/hurl/rs/hrl +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -# Check if $CDCTI_HOME is set -if [ -z "$CDCTI_HOME" ]; then - echo "Error: CDCTI_HOME is not set. Please set this environment variable before running the script." - exit 1 -fi - -# default values -env=local -root=$CDCTI_HOME/examples/ -content_type=application/hl7-v2 -client_id=flexion -client_sender=simulated-sender -verbose="" -submission_id="" - -show_help() { - echo "Usage: $(basename $0) [OPTIONS]" - echo - echo "Options:" - echo " -f The path to the hl7/fhir file to submit, relative the root path (Required for waters API)" - echo " -r The root path to the hl7/fhir files (Default: $root)" - echo " -t The content type for the message (e.g. 'application/hl7-v2' or 'application/fhir+ndjson') (Default: $content_type)" - echo " -e [local | staging | production ] The environment to run the test in (Default: $env)" - echo " -c The client id to use (Default: $client_id)" - echo " -s The client sender to use (Default: $client_sender)" - echo " -x The path to the client private key for the environment" - echo " -i The submissionId to call the history API with (Required for history API)" - echo " -v Verbose mode" - echo " -h Display this help and exit" -} - -# Check if required HURL_FILE is provided -if [ $# -eq 0 ]; then - echo "Error: Missing required argument " - show_help - exit 1 -fi - -# Check if first argument is -h -if [ "$1" = "-h" ]; then - show_help - exit 0 -fi - -hurl_file="$1" # Assign the first argument to hurl_file -shift # Remove the first argument from the list of arguments - -while getopts ':f:r:t:e:c:s:x:i:vh' opt; do - case "$opt" in - f) - fpath="$OPTARG" - ;; - r) - root="$OPTARG" - ;; - t) - content_type="$OPTARG" - ;; - e) - env="$OPTARG" - ;; - c) - client_id="$OPTARG" - ;; - s) - client_sender="$OPTARG" - ;; - x) - secret="$OPTARG" - ;; - i) - submission_id="--variable submissionid=$OPTARG" - ;; - v) - verbose="--verbose" - ;; - h) - show_help - exit 0 - ;; - :) - echo -e "Option requires an argument" - show_help - exit 1 - ;; - ?) - echo -e "Invalid command option" - show_help - exit 1 - ;; - esac -done -shift "$(($OPTIND - 1))" - -if [ "$env" = "local" ]; then - host=localhost - url=http://$host:7071 - if [ -z "$secret" ] && [ "$client_id" = "flexion" ]; then - secret="$CDCTI_HOME/mock_credentials/organization-trusted-intermediary-private-key-local.pem" - fi -elif [ "$env" = "staging" ]; then - host=staging.prime.cdc.gov - url=https://$host:443 -elif [ "$env" = "production" ]; then - host=prime.cdc.gov - url=https://$host:443 -else - echo "Error: Invalid environment $env" - show_help - exit 1 -fi - -if [ -z "$secret" ]; then - echo "Error: Please provide the private key for $client_id" - exit 1 -fi - -hurl \ - --variable fpath=$fpath \ - --file-root $root \ - --variable url=$url \ - --variable content-type=$content_type \ - --variable client-id=$client_id \ - --variable client-sender=$client_sender \ - --variable jwt=$(jwt encode --exp='+5min' --jti $(uuidgen) --alg RS256 -k $client_id.$client_sender -i $client_id.$client_sender -s $client_id.$client_sender -a $host --no-iat -S @$secret) \ - $submission_id \ - $verbose \ - $hurl_file \ - $@ diff --git a/scripts/hurl/submit_message.sh b/scripts/hurl/submit_message.sh index 83045be57..35c7ccfdb 100755 --- a/scripts/hurl/submit_message.sh +++ b/scripts/hurl/submit_message.sh @@ -1,12 +1,68 @@ #!/bin/bash -source ./message_submission_utils.sh +source ./utils.sh -check_prerequisites +env="local" -if [ $# -eq 0 ]; then - echo "Usage: $0 /path/to/message.hl7" +show_usage() { + cat < [-e ] + +Options: + -f Message file path (Required) + -e Environment: local|staging|production (Default: $DEFAULT_ENV) + -x Path to the client private key for authentication with RS API (Required for non-local environments) + -z Path to the client private key for authentication with TI API (Optional for all environments) + -h Display this help and exit +EOF exit 1 -fi +} + +parse_arguments() { + # Show help if no arguments + [ $# -eq 0 ] && show_usage + + while getopts "e:f:x:z:h" opt; do + case $opt in + e) env="$OPTARG" ;; + f) file="$OPTARG" ;; + x) rs_client_private_key="$OPTARG" ;; + z) ti_client_private_key="$OPTARG" ;; + h) show_usage ;; + ?) fail "Invalid option: -$OPTARG" ;; + esac + done + + [ -z "$file" ] && fail "File (-f) is required" + [ -f "$file" ] || fail "File not found: $file" + + # Validate environment + case "$env" in + local | staging | production) ;; + *) fail "Invalid environment '$env'. Must be local, staging, or production" ;; + esac +} + +setup_credentials() { + # Handle RS client key + if [ "$env" = "local" ] && [ -z "$rs_client_private_key" ]; then + rs_client_private_key="$RS_CLIENT_LOCAL_PRIVATE_KEY_PATH" + fi + + [ "$env" != "local" ] && [ -z "$rs_client_private_key" ] && fail "RS client private key (-x) is required for non-local environments" + [ ! -f "$rs_client_private_key" ] && fail "RS client private key file not found: $rs_client_private_key" + + # Handle optional TI client key + if [ "$env" = "local" ] && [ -z "$ti_client_private_key" ]; then + ti_client_private_key="$TI_CLIENT_LOCAL_PRIVATE_KEY_PATH" + fi + + # Only verify TI key if provided + [ -n "$ti_client_private_key" ] && [ ! -f "$ti_client_private_key" ] && fail "TI client private key file not found: $ti_client_private_key" +} -submit_message "$1" +check_installed_commands hurl jq az +check_apis "$(get_api_url "$env" "rs")" "$(get_api_url "$env" "ti")" +parse_arguments "$@" +setup_credentials +submit_message "$env" "$file" "$rs_client_private_key" "$ti_client_private_key" diff --git a/scripts/hurl/ti.sh b/scripts/hurl/ti.sh new file mode 100755 index 000000000..9278dc147 --- /dev/null +++ b/scripts/hurl/ti.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +source ./utils.sh + +# default values +env=local +root=$CDCTI_HOME/examples/ +client=report-stream + +show_usage() { + cat < [OPTIONS] + +ENDPOINT_NAME: + The name of the endpoint to call (required) + +Options: + -f Path to the hl7/fhir file to submit (Required for orders and results APIs) + -r Root path to the hl7/fhir files (Default: $root) + -e Environment: local|staging (Default: $env) + -c Client ID to create JWT with (Default: $client) + -k Path to the client private key (Required for non-local environments) + -i Submission ID for metadata API (Required for orders, results and metadata API) + -v Verbose mode + -h Display this help and exit + +Environment Variables: + CDCTI_HOME Base directory for CDC TI repository (Required) +EOF +} + +parse_arguments() { + if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + show_usage + exit 0 + fi + + [ $# -eq 0 ] && fail "Missing required argument " + endpoint_name="ti/$1.hurl" + shift # Remove endpoint name from args + + while getopts ':f:r:e:c:k:i:v' opt; do + case "$opt" in + f) fpath="$OPTARG" ;; + r) root="$OPTARG" ;; + e) env="$OPTARG" ;; + c) client="$OPTARG" ;; + k) private_key="$OPTARG" ;; + i) submission_id="--variable submissionid=$OPTARG" ;; + v) verbose="--verbose" ;; + ?) fail "Invalid option -$OPTARG" ;; + esac + done + + shift "$(($OPTIND - 1))" + remaining_args="$*" +} + +setup_credentials() { + if [ -z "$private_key" ] && [ "$client" = "report-stream" ] && [ "$env" = "local" ]; then + if [ -f "$TI_CLIENT_LOCAL_PRIVATE_KEY_PATH" ]; then + private_key="$TI_CLIENT_LOCAL_PRIVATE_KEY_PATH" + else + fail "Local environment client private key not found at: $TI_CLIENT_LOCAL_PRIVATE_KEY_PATH" + fi + fi + + if [ "$env" != "local" ]; then + [ -z "$private_key" ] && fail "Client private key (-k) is required for non-local environments" + fi + + [ ! -f "$private_key" ] && fail "Client private key file not found: $private_key" +} + +run_hurl_command() { + url=$(get_api_url "$env" "ti") + host=$(extract_host_from_url "$url") + jwt_token=$(generate_jwt "$client" "$host" "$private_key") || fail "Failed to generate JWT token" + + hurl \ + --variable "fpath=$fpath" \ + --file-root "$root" \ + --variable "url=$url" \ + --variable "client=$client" \ + --variable "jwt=$jwt_token" \ + ${submission_id:-} \ + ${verbose:-} \ + "$endpoint_name" \ + ${remaining_args:+$remaining_args} +} + +check_env_vars CDCTI_HOME +parse_arguments "$@" +setup_credentials +run_hurl_command diff --git a/scripts/hurl/ti/README.md b/scripts/hurl/ti/README.md deleted file mode 100644 index 704cb30b0..000000000 --- a/scripts/hurl/ti/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# CDC Intermediary Hurl Script - -## Usage - -``` -Usage: hrl [OPTIONS] - -Options: - -f The path to the hl7/fhir file to submit, relative the root path (Required for orders and results APIs) - -r The root path to the hl7/fhir files (Default: $CDCTI_HOME/examples/) - -e [local | staging] The environment to run the test in (Default: local) - -c The client id to use (Default: report-stream) - -j The JWT to use for authentication - -i The submissionId to call the metadata API with (Required for metadata API) - -v Verbose mode - -h Display this help and exit -``` - -## Examples - -Submit an order to local environment: -``` -./hrl orders.hurl -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_1_hl7_translation.fhir -i 100 -``` - -Submit an order to staging: -``` -./hrl orders.hurl -f Test/Orders/003_AL_ORM_O01_NBS_Fully_Populated_0_initial_message.hl7 -e staging -j eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ - -``` - -Submit a result to local environment: -``` -./hrl results.hurl -f Test/Results/002_AL_ORU_R01_NBS_Fully_Populated_1_hl7_translation.fhir -i 100 -``` - -Get metadata from local environment: -``` -./hrl metadata -i 100 -``` - -Authenticate to local environment: -``` -./hrl auth.hurl -``` - -Get OpenAPI docs from local environment: -``` -./hrl openapi.hurl -``` - -Get Health info from local environment: -``` -./hrl health.hurl -``` diff --git a/scripts/hurl/ti/hrl b/scripts/hurl/ti/hrl deleted file mode 100755 index 465ff3c92..000000000 --- a/scripts/hurl/ti/hrl +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -# Check if $CDCTI_HOME is set -if [ -z "$CDCTI_HOME" ]; then - echo "Error: CDCTI_HOME is not set. Please set this environment variable before running the script." - exit 1 -fi - -# default values -env=local -root=$CDCTI_HOME/examples/ -client=report-stream -verbose="" - -show_help() { - echo "Usage: $(basename $0) [OPTIONS]" - echo - echo "Options:" - echo " -f The path to the hl7/fhir file to submit, relative the root path (Required for orders and results APIs)" - echo " -i The submissionId to call the metadata API with (Required for orders, results and metadata API)" - echo " -r The root path to the hl7/fhir files (Default: $root)" - echo " -e [local | staging] The environment to run the test in (Default: $env)" - echo " -c The client id to use (Default: $client)" - echo " -j The JWT to use for authentication" - echo " -v Verbose mode" - echo " -h Display this help and exit" -} - -# Check if required HURL_FILE is provided -if [ $# -eq 0 ]; then - echo "Error: Missing required argument " - show_help - exit 1 -fi - -# Check if first argument is -h -if [ "$1" = "-h" ]; then - show_help - exit 0 -fi - -hurl_file="$1" # Assign the first argument to hurl_file -shift # Remove the first argument from the list of arguments - -while getopts ':f:r:e:c:j:i:vh' opt; do - case "$opt" in - f) - fpath="$OPTARG" - ;; - i) - submission_id="--variable submissionid=$OPTARG" - ;; - r) - root="$OPTARG" - ;; - e) - env="$OPTARG" - ;; - c) - client="$OPTARG" - ;; - j) - jwt="$OPTARG" - ;; - v) - verbose="--verbose" - ;; - h) - show_help - exit 0 - ;; - :) - echo -e "Option requires an argument" - show_help - exit 1 - ;; - ?) - echo -e "Invalid command option" - show_help - exit 1 - ;; - esac -done -shift "$(($OPTIND - 1))" - -if [ "$env" = "local" ]; then - host=localhost - url=http://$host:8080 - if [ -z "$jwt" ] && [ "$client" = "report-stream" ]; then - jwt=$(cat "$CDCTI_HOME/mock_credentials/report-stream-valid-token.jwt") - fi -elif [ "$env" = "staging" ]; then - host=cdcti-stg-api.azurewebsites.net - url=https://$host:443 -else - echo "Error: Invalid environment $env" - show_help - exit 1 -fi - -if [ -z "$jwt" ]; then - echo "Error: Please provide the JWT for $client" - exit 1 -fi - -hurl \ - --variable fpath=$fpath \ - --file-root $root \ - --variable url=$url \ - --variable client=$client \ - --variable jwt=$jwt \ - $submission_id \ - $verbose \ - $hurl_file \ - $@ diff --git a/scripts/hurl/update_examples.sh b/scripts/hurl/update_examples.sh index f0c6ac3da..d140ebfab 100755 --- a/scripts/hurl/update_examples.sh +++ b/scripts/hurl/update_examples.sh @@ -1,11 +1,14 @@ #!/bin/bash -source ./message_submission_utils.sh +source ./utils.sh -check_prerequisites +env=local + +check_installed_commands hurl jq az +check_apis "$(get_api_url "$env" "rs")" "$(get_api_url "$env" "ti")" find "$CDCTI_HOME/examples" -type f -name "*$FILE_NAME_SUFFIX_STEP_0.hl7" | while read -r file; do echo "-----------------------------------------------------------------------------------------------------------" echo "Submitting message: $file" - submit_message "$file" + submit_message "$env" "$file" "$RS_CLIENT_LOCAL_PRIVATE_KEY_PATH" "$TI_CLIENT_LOCAL_PRIVATE_KEY_PATH" done diff --git a/scripts/hurl/utils.sh b/scripts/hurl/utils.sh new file mode 100755 index 000000000..b9d8903f7 --- /dev/null +++ b/scripts/hurl/utils.sh @@ -0,0 +1,284 @@ +#!/bin/bash + +FILE_NAME_SUFFIX_STEP_0="_0_initial_message" +FILE_NAME_SUFFIX_STEP_1="_1_hl7_translation" +FILE_NAME_SUFFIX_STEP_2="_2_fhir_transformation" +FILE_NAME_SUFFIX_STEP_3="_3_hl7_translation_final" + +AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" # pragma: allowlist secret + +RS_API_LCL_URL="http://localhost:7071" +RS_API_STG_URL="https://staging.prime.cdc.gov:443" +RS_API_PRD_URL="https://prime.cdc.gov:443" +TI_API_LCL_URL="http://localhost:8080" +TI_API_STG_URL="https://cdcti-stg-api.azurewebsites.net:443" +TI_API_PRD_URL="https://cdcti-prd-api.azurewebsites.net:443" + +RS_CLIENT_LOCAL_PRIVATE_KEY_PATH="$CDCTI_HOME/mock_credentials/organization-trusted-intermediary-private-key-local.pem" +TI_CLIENT_LOCAL_PRIVATE_KEY_PATH="$CDCTI_HOME/mock_credentials/organization-report-stream-private-key-local.pem" + +fail() { + echo "Error: $1" >&2 + exit 1 +} + +check_installed_commands() { + for cmd in "$@"; do + if ! command -v "$cmd" &>/dev/null; then + echo "$cmd could not be found. Please install $cmd to proceed." + exit 1 + fi + done +} + +check_apis() { + for service in "$@"; do + if ! curl -s --head --fail "$service" | grep "200 OK" >/dev/null; then + echo "The service at $service is not reachable" + exit 1 + fi + done +} + +check_env_vars() { + local env_vars=("$@") + for var in "${env_vars[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: Environment variable '$var' is not set" + exit 1 + fi + done +} + +get_api_url() { + local env=$1 + local type=$2 + + case "$type" in + "rs") + case "$env" in + "local") echo $RS_API_LCL_URL ;; + "staging") echo $RS_API_STG_URL ;; + "production") echo $RS_API_PRD_URL ;; + *) + echo "Invalid environment: $env" >&2 + exit 1 + ;; + esac + ;; + "ti") + case "$env" in + "local") echo $TI_API_LCL_URL ;; + "staging") echo $TI_API_STG_URL ;; + "production") echo $TI_API_PRD_URL ;; + *) + echo "Invalid environment: $env" >&2 + exit 1 + ;; + esac + ;; + esac +} + +extract_host_from_url() { + local url=$1 + echo "$url" | sed 's|^.*://\([^/:]*\)[:/].*|\1|' +} + +generate_jwt() { + # requires: jwt-cli + local client=$1 + local audience=$2 + local secret_path=$3 + + jwt encode \ + --exp='+5min' \ + --jti "$(uuidgen)" \ + --alg RS256 \ + -k "$client" \ + -i "$client" \ + -s "$client" \ + -a "$audience" \ + --no-iat \ + -S "@$secret_path" +} + +extract_rs_history_submission_id() { + # requires: jq + local history_response=$1 + local externalName + externalName=$(echo "$history_response" | jq -r '.destinations[0].sentReports[0].externalName') || return 1 + [[ -z "$externalName" || "$externalName" == "null" ]] && return 1 + echo "$externalName" | sed 's/.*-\([0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}\)-.*/\1/' +} + +extract_ti_metadata_submission_id() { + # requires: jq + local metadata_response=$1 + echo "$metadata_response" | jq -r '.issue[] | select(.details.text == "outbound submission id") | .diagnostics' +} + +download_from_azurite() { + # requires: jq, az + local blob_name=$1 + local file_path=$2 + + echo " Downloading from blob storage '$blob_name' to '$file_path'" + az storage blob download \ + --container-name "reports" \ + --name "$blob_name" \ + --file "$file_path" \ + --connection-string "$AZURITE_CONNECTION_STRING" \ + --output none || { + echo "Download failed for blob '$blob_name'." + exit 1 + } + + if [[ "$file_path" == *.fhir ]]; then + echo " Formatting the content for: $file_path" + formatted_content=$(jq '.' "$file_path") + echo "$formatted_content" >"$file_path" + fi +} + +check_submission_status() { + # requires: hurl, jq + local env=$1 + local submission_id=$2 + local private_key=$3 + + local timeout=180 # 3 minutes + local retry_interval=10 # Retry every 10 seconds + + start_time=$(date +%s) + + while true; do + history_response=$(./rs.sh history -i "$submission_id" -e "$env" -k "$private_key") || { + exit_code=$? + if [ $exit_code -ne 0 ]; then + fail "Expected exit code 0 but got $exit_code for RS history API call" + fi + } + overall_status=$(echo "$history_response" | jq -r '.overallStatus') + + echo -n " Status: $overall_status" + if [ "$overall_status" == "Delivered" ]; then + echo "!" + return 0 + else + echo ". Retrying in $retry_interval seconds..." + fi + + # Check if the timeout has been reached + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + echo " Timeout reached after $elapsed_time seconds. Status is still: $overall_status." + return 1 + fi + + sleep $retry_interval + done +} + +submit_message() { + # requires: hurl, jq, az + local env=$1 + local file=$2 + local rs_client_private_key=$3 + local ti_client_private_key=$4 + + local message_file_path message_file_name message_base_name + message_file_path="$(dirname "${file}")" + message_file_name="$(basename "${file}")" + message_base_name="${message_file_name%.hl7}" + message_base_name="${message_base_name%"$FILE_NAME_SUFFIX_STEP_0"}" + + msh9=$(awk -F'|' '/^MSH/ { print $9 }' "$file") + case "$msh9" in + "ORU^R01^ORU_R01") + first_leg_receiver="flexion.etor-service-receiver-results" + second_leg_receiver="flexion.simulated-hospital" + ;; + "OML^O21^OML_O21" | "ORM^O01^ORM_O01") + first_leg_receiver="flexion.etor-service-receiver-orders" + second_leg_receiver="flexion.simulated-lab" + ;; + *) + echo "Unknown receivers for MSH-9 value '$msh9'. Skipping the message" + return + ;; + esac + + echo "Assuming receivers are '$first_leg_receiver' and '$second_leg_receiver' because of MSH-9 value '$msh9'" + + waters_response=$(./rs.sh waters -f "$message_file_name" -r "$message_file_path" -e "$env" -k "$rs_client_private_key") || { + exit_code=$? + if [ $exit_code -ne 0 ]; then + fail "Expected exit code 0 but got $exit_code for RS waters API call" + fi + } + submission_id=$(echo "$waters_response" | jq -r '.id') + + echo "[First leg] Checking submission status for ID: $submission_id" + if ! check_submission_status "$env" "$submission_id" "$rs_client_private_key"; then + echo "Failed to deliver the first leg of the message. Skipping the next steps." + return + fi + + inbound_submission_id=$(extract_rs_history_submission_id "$history_response") + + translated_blob_name="ready/$first_leg_receiver/$inbound_submission_id.fhir" + translated_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_1.fhir" + if [ "$env" = "local" ]; then + download_from_azurite "$translated_blob_name" "$translated_file_path" + else + echo " Snapshot for translated message in blob storage at: $translated_blob_name" + fi + + echo "[Intermediary] Getting outbound submission ID" + if [ -n "$ti_client_private_key" ]; then + echo " Attempting to get outbound submission ID from TI's metadata API..." + metadata_response=$(./ti.sh metadata -i "$inbound_submission_id" -e "$env" -k "$ti_client_private_key") || { + echo "Failed to get metadata for inbound submission ID: $inbound_submission_id" + outbound_submission_id="" + } + if [ -n "$metadata_response" ]; then + outbound_submission_id=$(extract_ti_metadata_submission_id "$metadata_response") + fi + fi + if [ -z "$outbound_submission_id" ] || [ "$outbound_submission_id" = "null" ]; then + echo -n " Please enter the outbound submission ID manually (you may find it in the TI $env logs): " + read -r outbound_submission_id + if [ -z "$outbound_submission_id" ]; then + fail "No outbound submission ID provided" + fi + fi + + if [ -z "$outbound_submission_id" ] || [ "$outbound_submission_id" = "null" ]; then + outbound_submission_id=$(extract_ti_metadata_submission_id "$metadata_response") + fi + + transformed_blob_name="receive/flexion.etor-service-sender/$outbound_submission_id.fhir" + transformed_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_2.fhir" + if [ "$env" = "local" ]; then + download_from_azurite "$transformed_blob_name" "$transformed_file_path" + else + echo " Snapshot for transformed message in blob storage at: $translated_blob_name" + fi + + echo "[Second leg] Checking submission status for ID: $outbound_submission_id" + if ! check_submission_status "$env" "$outbound_submission_id" "$rs_client_private_key"; then + echo "Failed to deliver the second leg of the message. Skipping the next steps." + return + fi + + final_submission_id=$(extract_rs_history_submission_id "$history_response") + final_blob_name="ready/$second_leg_receiver/$final_submission_id.hl7" + final_file_path="$message_file_path/$message_base_name$FILE_NAME_SUFFIX_STEP_3.hl7" + if [ "$env" = "local" ]; then + download_from_azurite "$final_blob_name" "$final_file_path" + else + echo " Snapshot for final message in blob storage at: $translated_blob_name" + fi +}