dhusget.sh

#!/bin/bash

#-------------------------------------------------------------------------------------------	#
# Demo script illustrating some examples using the OData interface	#
# of the Data Hub Service (DHuS)                                                     	#
#-------------------------------------------------------------------------------------------
# CHANGE LOG
# v0.3.1:
#	- usage switch fixed
#	- usage text updated to include the download of Sentinel 2 products
# 	- introduction of parallel download with check of the server error messages (option -n)
#	- insertion of MD5 check
#
# Serco SpA 2015
# CHANGE LOG
# v0.3.2:
#       - fixed "-f" option
#       - upgraded "-f"
#       - added the following options: -s, -e, -S, -E, -F
#
# CHANGE LOG
# v0.3.3:
#       - added the following options: -O, -L, -R, -r
#
# CHANGE LOG
# v0.3.4:
#       - added the following options: -m, -i, -l, -P, -q, -C, -N, -D
#
# Serco SpA 2015
#
#CHANGE LOG
#v0.3.5:
#		-changed "==" with "=" in the MD5 check section
#
#CHANGE LOG
#v0.3.6
#		- added Sentinel-5 P references
#		- changed "==" with "=" in the MD5 check section [line 818]
#       - added CheckSum Echo
#
#CHANGE LOG
#v0.3.7
#		- fix download of ".NetCDF" file
#                                                                            	#
#CHANGE LOG
#v0.3.8
#		- added management of offline products download
#
#-------------------------------------------------------------------------------------------	#
export VERSION=0.3.8

WD=dhusget_tmp
PIDFILE=$WD/pid

test -d $WD || mkdir -p $WD

#-

bold=$(tput bold)
normal=$(tput sgr0)
print_script=$(echo "$0" | rev | cut -d'/' -f1 | rev)

function print_usage {
	print_script=$(echo "$1" | rev | cut -d'/' -f1 | rev)
	echo " "
	echo "${bold}NAME${normal}"
	echo " "
	echo "  DHuSget $VERSION - The non interactive Sentinels product retriever from the Sentinels Data Hubs"
	echo " "
	echo "${bold}USAGE${normal}"
	echo " "
	echo "  $print_script [LOGIN OPTIONS]... [SEARCH QUERY OPTIONS]... [SEARCH RESULT OPTIONS]... [DOWNLOAD OPTIONS]... "
	echo " "
	echo "${bold}DESCRIPTION${normal}"
	echo " "
	echo "  This script allows to get products from Sentinels Data Hubs executing query with different filter. The products can be visualized on shell and saved in list file"
	echo "  or downloaded in a zip file."
	echo "  Recommendation: If this script is run as a cronjob, to avoid traffic load, please do not schedule it exactly at on-the-clock hours (e.g 6:00, 5:00)."
	echo " "
	echo "${bold}OPTIONS"
	echo " "
	echo "  ${bold}LOGIN OPTIONS:${normal}"
	echo " "
	echo "   -d <DHuS URL>		: specify the URL of the Data Hub Service;"
	echo "   -u <username>		: data hub username;"
	echo "   -p <password>		: data hub password (note: if not provided by command line it is read by stdin);"
	echo " "
	echo " "
	echo "  ${bold}SEARCH QUERY OPTIONS:${normal}"
	echo " "
	echo "   -m <mission name>		: Sentinel mission name. Possible options are: Sentinel-1, Sentinel-2, Sentinel-3, Sentinel-5 P);"
	echo ""
	echo "   -i <instrument name>		: instrument name. Possible options are: SAR, MSI, OLCI, SLSTR, SRAL, TROPOMI);"
	echo ""
	echo "   -t <time in hours>		: search for products ingested in the last <time in hours> (integer) from the time of"
	echo " 				  execution of the script."
	echo "   				  (e.g. '-t 24' to search for products ingested in the last 24 Hours);"
	echo ""
	echo "   -s <ingestion_date_FROM>	: Search for products ingested ${bold}after${normal} the date and time specified by <ingestion_date_FROM>."
	echo "   				  The date format is ISO 8601, YYYY-MM-DDThh:mm:ss.cccZ (e.g. -s 2016-10-02T06:00:00.000Z);"
	echo ""
	echo "   -e <ingestion_date_TO>	: Search for products ingested ${bold}before${normal} the date specified by <ingestion_date_TO>."
	echo "   				  The date format is ISO 8601, YYYY-MM-DDThh:mm:ss.cccZ (e.g. -e 2016-10-10T12:00:00.000Z);"
	echo ""
	echo "   -S <sensing_date_FROM>	: Search for products with sensing date ${bold}greater than${normal} the date and time specified by <sensing_date_FROM>."
	echo "   				  The date format is ISO 8601, YYYY-MM-DDThh:mm:ss.cccZ (e.g. -S 2016-10-02T06:00:00.000Z);"
	echo ""
	echo "   -E <sensing_date_TO>		: Search for products with sensing date ${bold}less than${normal} the date and time specified by <sensing_date_TO>."
	echo "   				  The date format is ISO 8601, YYYY-MM-DDThh:mm:ss.cccZ (e.g. -E 2016-10-10T12:00:00.000Z);"
	echo ""
	echo "   -f <ingestion_date_file>	: Search for products ingested after the date and time provided through an input file. This option overrides option -s"
	echo "   				  The date format shall be ISO 8601 (YYYY-MM-DDThh:mm:ss.cccZ)."
	echo "   				  <ingestion_date_file> is automatically updated at the end of the script execution"
	echo "   				  with the ingestion date of the last sucessfully downloaded product;"
	echo " "
	echo "   -c <lon1,lat1:lon2,lat2> 	: Search for products intersecting a rectangular Area of Interst (or Bounding Box)"
	echo "   				  by providing the geographical coordinates of two opposite vertices. "
	echo "   				  Coordinates need to be provided in Decimal Degrees and with the following syntax:"
	echo " "
	echo "   				     -    ${bold}lon1,lat1:lon2,lat2${normal}"
	echo " "
	echo "   				  where lon1 and lat1 are respectively the longitude and latitude of the first vertex and"
	echo "  				  lon2 and lat2 the longitude and latitude of the second vertex."
	echo "   				  (e.g. '-c -4.530,29.850:26.750,46.800' is a bounding box enclosing the Mediterranean Sea);"
	echo " "
	echo "   -T <product type>		: Search products according to the specified product type."
	echo "   				  Sentinel-1 possible options are:  SLC, GRD, OCN and RAW. "
	echo "   				  Sentinel-2 posiible option is: S2MSI1C ;"
	echo "   				  Sentinel-5 P possible options are:  RA, IR, O3, NO2, CO, CLOUD, AER and NP. "
	echo " "
	echo "   -F <free OpenSearch query>	: free text OpenSearch query. The query must be written enclosed by single apexes '<query>'. "
	echo "   				  (e.g. -F 'platformname:Sentinel-1 AND producttype:SLC'). "
	echo "   				  (e.g. -F 'platformname:Sentinel-5 P AND producttype:RA'). "
	echo "   				  Note: the free text OpenSearch query is in ${bold}AND${normal} with the other possible sspecified search options."
	echo " "
	echo " "
	echo "  ${bold}SEARCH RESULT OPTIONS:${normal}"
	echo " "
	echo "   -l <results>			: maximum number of results per page [1,2,3,4,..]; default value = 25"
	echo " "
	echo "   -P <page>			: page number [1,2,3,4,..]; default value = 1"
	echo " "
	echo "   -q <XMLfile>			: write the OpenSearch query results in a specified XML file. Default file is './OSquery-result.xml'"
	echo " "
	echo "   -C <CSVfile>			: write the list of product results in a specified CSV file. Default file is './products-list.csv'"
	echo " "
	echo " "
	echo "  ${bold}DOWNLOAD OPTIONS:${normal}"
	echo " "
	echo "   -o <download>		: THIS OPTION IS MANDATORY FOR DOWNLOADING. Accepted values for <download> are:"
	echo "   				  	-  ${bold}product${normal} : download the Product ZIP files (manifest file included)"
	echo "   				  	-  ${bold}manifest${normal} : download only the manifest files"
	echo "   				  	-  ${bold}all${normal} : download both the Product ZIP files and the manifest files, and"
	echo "   				  		 provide them in separate folders."
	echo ""
	echo "   				  	  By default the Product ZIP files are stored in ./product"
	echo "   				   	  unless differently specified by option ${bold}-O${normal}."
	echo ""
	echo "   				  	  By default the manifest files are stored in ./manifest ;"
	echo " "
	echo " "
	echo "   -O <folder>			: save the Product ZIP files in a specified folder. "
	echo " "
	echo "   -N <1...n>			: set number of wget download retries. Default value is 5. Fatal errors like 'connection refused'"
	echo "   				  or 'not found' (404), are not retried;"
	echo " "
	echo "   -R <file>			: write in <file> the list of products that have failed the MD5 integrity check."
	echo "   				  By default the list is written in ./failed_MD5_check_list.txt ;"
	echo "   				  The format of the output file is compatible with option ${bold}-r${normal} ;"
	echo " "
	echo "   -D  				: if specified, remove the products that have failed the MD5 integrity check from disk."
	echo "   				  By deafult products are not removed;"
	echo " "
	echo "   -r <file>			: download the products listed in an input <file> written according to the following format:"
	echo "   				  - One product per line."
	echo "   				  - <space><one character><space><UUID><space><one character><space><filename>."
	echo "   			Examples:"
	echo "   			' x 67c7491a-d98a-4eeb-9ca0-8952514c7e1e x S1A_EW_GRDM_1SSH_20160411T113221_20160411T113257_010773_010179_7BE0'"
	echo "   			' 0 67c7491a-d98a-4eeb-9ca0-8952514c7e1e 0 S1A_EW_GRDM_1SSH_20160411T113221_20160411T113257_010773_010179_7BE0'"
	echo " "
	echo "   -L <lock folder>		: by default only one instance of dhusget can be executed at a time. This is ensured by the creation"
	echo "   				  of a temporary lock folder $HOME/dhusget_tmp/lock which is removed a the end of each run."
	echo "   				  For running more than one dhusget instance at a time is sufficient to assign different lock folders"
	echo "   				  using the -L option (e.g. '-L foldername') to each dhusget instance;"
	echo " "
	echo "   -n <1...n>			: number of concurrent downloads (either products or manifest files). Default value is 2; this value"
	echo "   				  doesn't override the quota limit set on the server side for the user"
	echo " "
	echo "   -w <1...n>			: minutes to wait until retrying the download of an offline product. Default value is 10;"
	echo " "
	echo " "
	echo "   -W <1...n>			: number of download retries for offline products. Default value is 40."
	echo " "
	echo " "
	echo "   'wget' is necessary to run the dhusget"
	echo " "
	exit -1
}

function print_version {
	echo "dhusget $VERSION"
	exit -1
}

#------------------------------------------------------------------------------
#--- added methods and variables for offline product download management BEGIN
#------------------------------------------------------------------------------
export OFFLINE_PRODUCT_LIST=offline_product_list.txt
export TMP_OFFLINE_PRODUCT_LIST=offline_product_list.csv
export OFFLINE_RETRY_ATTEMPTS_COUNTER=0

# export -f add_offline_product_to_list

function download_product {
	cat $1 | xargs -n 4 -P $2 sh -c ' while : ; do
	echo "Downloading product ${3} from link ${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/\$value";	
# ---added control on S5P products
        prod5P=${3}
		mission5P="S5P"
		if echo $prod5P  | grep -q "$mission5P"; then
	${WC} ${AUTH} ${TRIES} --progress=dot -e dotbytes=10M -c --output-file=./logs/log.${3}.log -O $output_folder/${3}".nc" "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/\$value";
	else
	
        ${WC} ${AUTH} ${TRIES} --progress=dot -e dotbytes=10M -c --output-file=./logs/log.${3}.log -O $output_folder/${3}".zip" "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/\$value";
	fi
	test=$?;
# --- check HTTP response status in case of offline products download	
	
	http_response=`cat ./logs/log.${3}.log| grep  '\''403 Forbidden\|202 Accepted\|503 Service Unavailable'\''`;	
	echo "http_response is $http_response";
	if [ ! -z "$http_response" ]; then
		
		echo "The requested product is offline" ;
		echo "Adding product ${3} in the offline product download queue" ;
		echo "${2},${1},${3}" | tee -a "${TMP_OFFLINE_PRODUCT_LIST}" ;
		
	else		
		if [ $test -eq 0 ]; then 
			echo "Product ${3} successfully downloaded at " `tail -2 ./logs/log.${3}.log | head -1 | awk -F"(" '\''{print $2}'\'' | awk -F")" '\''{print $1}'\''`;
			remoteMD5=$( ${WC} -qO- ${AUTH} ${TRIES} -c "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/Checksum/Value/$value" | awk -F">" '\''{print $3}'\'' | awk -F"<" '\''{print $1}'\'');
			if echo $prod5P  | grep -q "$mission5P"; then 
				localMD5=$( openssl md5 $output_folder/${3}".nc" | awk '\''{print $2}'\'');
			else
				localMD5=$( openssl md5 $output_folder/${3}".zip" | awk '\''{print $2}'\'');
			fi		

			localMD5Uppercase=$(echo "$localMD5" | tr '\''[:lower:]'\'' '\''[:upper:]'\'');
			#localMD5Uppercase=1;
			
			if [ "$remoteMD5" = "$localMD5Uppercase" ]; then
				echo "Product ${3} successfully MD5 checked";
				echo "Remote MD5: $remoteMD5"
			    echo "Local  MD5: $localMD5Uppercase"
			else
				echo "Checksum for product ${3} failed";
				echo "${0} ${1} ${2} ${3}" >> .failed.control.now.txt;
				if [ ! -z $save_products_failed ];then  
				      rm $output_folder/${3}".zip"
				fi
			fi; 
		else
			
            echo "${0} ${1} ${2} ${3}" >> .failed.control.now.txt;
            if [ ! -z $save_products_failed ];then  
		    	rm $output_folder/${3}".zip"
            fi
        fi;
	fi;
        break;
	done '
}
#------------------------------------------------------------------------------
#--- added methods and variables for offline product download management END
#------------------------------------------------------------------------------

#----------------------
#---  Load input parameter
export DHUS_DEST="https://scihub.copernicus.eu/dhus"
export USERNAME=""
export PASSWORD=""
export TIME_SUBQUERY=""
export PRODUCT_TYPE=""
export INGEGESTION_TIME_FROM="1970-01-01T00:00:00.000Z"
export INGEGESTION_TIME_TO="NOW"
export SENSING_TIME_FROM="1970-01-01T00:00:00.000Z"
export SENSING_TIME_TO="NOW"
unset TIMEFILE

while getopts ":d:u:p:l:P:q:C:m:i:t:s:e:S:E:f:c:T:o:V:h:F:R:D:r:O:N:L:n:w:W:" opt; do
	case $opt in
	d)
		export DHUS_DEST="$OPTARG"
		export DHUS_DEST_IS_SET='OK'
		;;
	u)
		export USERNAME="$OPTARG"
		;;
	p)
		export PASSWORD="$OPTARG"
		;;
	l)
		export ROWS="$OPTARG"
		;;
	P)
		export PAGE="$OPTARG"
		;;
	q)
		export NAMEFILERESULTS="$OPTARG"
		;;
	C)
		export PRODUCTLIST="$OPTARG"
		;;
	m)
		export MISSION="$OPTARG"
		;;
	i)
		export INSTRUMENT="$OPTARG"
		;;
	t)
		export TIME="$OPTARG"
		export INGEGESTION_TIME_FROM="NOW-${TIME}HOURS"
		export isselected_filtertime_lasthours='OK'
		;;
	s)
		export TIME="$OPTARG"
		export INGEGESTION_TIME_FROM="$OPTARG"
		export isselected_filtertime_ingestion_date='OK'
		;;
	e)
		export TIME="$OPTARG"
		export INGEGESTION_TIME_TO="$OPTARG"
		export isselected_filtertime_ingestion_date='OK'
		;;
	S)
		export SENSING_TIME="$OPTARG"
		export SENSING_TIME_FROM="$OPTARG"
		;;
	E)
		export SENSING_TIME="$OPTARG"
		export SENSING_TIME_TO="$OPTARG"
		;;
	f)
		export TIMEFILE="$OPTARG"
		if [ -s $TIMEFILE ]; then
			export INGEGESTION_TIME_FROM="$(cat $TIMEFILE)"
		else
			export INGEGESTION_TIME_FROM="1970-01-01T00:00:00.000Z"
		fi
		;;
	c)
		ROW=$OPTARG

		FIRST=$(echo "$ROW" | awk -F\: '{print \$1}')
		SECOND=$(echo "$ROW" | awk -F\: '{print \$2}')

		#--
		export x1=$(echo ${FIRST} | awk -F, '{print $1}')
		export y1=$(echo ${FIRST} | awk -F, '{print $2}')
		export x2=$(echo ${SECOND} | awk -F, '{print $1}')
		export y2=$(echo ${SECOND} | awk -F, '{print $2}')
		;;

	T)
		export PRODUCT_TYPE="$OPTARG"
		;;
	o)
		export TO_DOWNLOAD="$OPTARG"
		;;
	V)
		print_version $0
		;;
	h)
		print_usage $0
		;;
	F)
		FREE_SUBQUERY_CHECK="OK"
		FREE_SUBQUERY="$OPTARG"
		;;
	R)
		export FAILED="$OPTARG"
		export check_save_failed='OK'
		;;
	D)
		export save_products_failed='OK'
		;;
	r)
		export FAILED_retry="$OPTARG"
		export check_retry='OK'
		;;
	O)
		export output_folder="$OPTARG"
		export OUTPUT_FOLDER_IS_SET='OK'
		;;
	N)
		export number_tries="$OPTARG"
		;;
	L)
		export lock_file="$OPTARG"
		export LOCK_FILE_IS_SET='OK'
		;;
	n)
		export THREAD_NUMBER="$OPTARG"
		export THREAD_NUMBER_IS_SET='OK'
		;;
	w)
		export OFFLINE_WAIT="${OPTARG}m"
		;;
	W)
		export OFFLINE_RETRY_ATTEMPTS="$OPTARG"
		;;
	esac
done
echo ""
echo "================================================================================================================"
echo ""
echo "dhusget version: $VERSION"
echo ""
echo "USAGE: $print_script [LOGIN OPTIONS]... [SEARCH QUERY OPTIONS]... [SEARCH RESULT OPTIONS]... [DOWNLOAD OPTIONS]... "
echo ""
echo "Type '$print_script -help' for usage information"
echo ""
echo "================================================================================================================"
ISSELECTEDEXIT=false
trap ISSELECTEDEXIT=true INT

if [ -z $lock_file ]; then
	export lock_file="$WD/lock"
fi

mkdir $lock_file

if [ ! $? == 0 ]; then
	echo -e "Error! An instance of \"dhusget\" retriever is running !\n Pid is: "$(cat ${PIDFILE}) "if it isn't running delete the lockdir  ${lock_file}"

	exit
else
	echo $$ >$PIDFILE
fi

trap "rm -fr ${lock_file}" EXIT

export TIME_SUBQUERY="ingestiondate:[$INGEGESTION_TIME_FROM TO $INGEGESTION_TIME_TO]  "

export SENSING_SUBQUERY="beginPosition:[$SENSING_TIME_FROM TO $SENSING_TIME_TO]  "

if [ -z $THREAD_NUMBER ]; then
	export THREAD_NUMBER="2"
fi

if [ -z $output_folder ]; then
	export output_folder="PRODUCT"
fi

export WC="wget --no-check-certificate --continue -nc"

echo "LOGIN"

printf "\n"

if [ -z $DHUS_DEST_IS_SET ]; then
	echo "'-d option' not specified. "
	echo "Default Data Hub Service URL is: "
else
	echo "Specified Data Hub Service URL is:"
fi
echo $DHUS_DEST

printf "\n"

if [ ! -z $USERNAME ] && [ -z $PASSWORD ]; then
	echo "You have inserted only USERNAME"
	echo ""
fi

if [ -z $USERNAME ] && [ ! -z $PASSWORD ]; then
	echo "You have inserted only PASSWORD"
	echo ""
fi

if [ -z $USERNAME ]; then
	read -p "Enter username: " VAL
	printf "\n"
	export USERNAME=${VAL}
fi

if [ -z $PASSWORD ]; then
	read -s -p "Enter password: " VAL
	printf "\n\n"
	export PASSWORD=${VAL}
fi

export AUTH="--user=${USERNAME} --password=${PASSWORD}"

if [ -z $number_tries ]; then
	export TRIES="--tries=5"
else
	export TRIES="--tries=${number_tries}"
fi

mkdir -p './logs/'

if [ ! -z $check_retry ] && [ -s $FAILED_retry ]; then
	cp $FAILED_retry .failed.control.retry.now.txt
	export INPUT_FILE=.failed.control.retry.now.txt

	mkdir -p $output_folder

	if [ -f .failed.control.now.txt ]; then
		rm .failed.control.now.txt
	fi
	cat ${INPUT_FILE} | xargs -n 4 -P ${THREAD_NUMBER} sh -c ' while : ; do
        echo "Downloading product ${3} from link ${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/\$value"; 
        ${WC} ${AUTH} ${TRIES} --progress=dot -e dotbytes=10M -c --output-file=./logs/log.${3}.log -O $output_folder/${3}".zip" "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/\$value";
        test=$?;
        if [ $test -eq 0 ]; then
                echo "Product ${3} successfully downloaded at " `tail -2 ./logs/log.${3}.log | head -1 | awk -F"(" '\''{print $2}'\'' | awk -F")" '\''{print $1}'\''`;
                remoteMD5=$( ${WC} -qO- ${AUTH} ${TRIES} -c "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/Checksum/Value/$value" | awk -F">" '\''{print $3}'\'' | awk -F"<"     '\''{print $1}'\'');
                localMD5=$( openssl md5 $output_folder/${3}".zip" | awk '\''{print $2}'\'');
                localMD5Uppercase=$(echo "$localMD5" | tr '\''[:lower:]'\'' '\''[:upper:]'\'');
				
				echo "Remote MD5: $remoteMD5" 
				echo "Local  MD5: $localMD5Uppercase" 
								
                if [ "$remoteMD5" = "$localMD5Uppercase" ]; then
                        echo "Product ${3} successfully MD5 checked";
                else
				echo "Checksum for product ${3} failed";
                echo "${0} ${1} ${2} ${3}" >> .failed.control.now.txt;
                if [ ! -z $save_products_failed ];then  
		      rm $output_folder/${3}".zip"
		fi
                fi; 
        else
                echo "${0} ${1} ${2} ${3}" >> .failed.control.now.txt;
                if [ ! -z $save_products_failed ];then
		      rm $output_folder/${3}".zip"
                fi
        fi;
        break;
done '
	rm .failed.control.retry.now.txt
fi

#----- Options value check
echo "================================================================================================================"
echo ""
echo "SEARCH QUERY OPTIONS"
if [ -z $TIME ] && [ -z $TIMEFILE ] && [ -z $ROW ] && [ -z $PRODUCT_TYPE ] && [ -z $FREE_SUBQUERY_CHECK ] && [ -z $SENSING_TIME ] && [ -z $MISSION ] && [ -z $INSTRUMENT ]; then
	echo ""
	echo "No Search Options specified. Default query is q='*'."
	echo ""
	export QUERY_STATEMENT="*"
else
	echo ""
fi
if [ -z $MISSION ]; then
	echo "'-m option' not specified. Search is performed on all available sentinel missions."
	echo ""
else
	echo "'-m option' mission is set to $MISSION. "
	echo ""
fi
if [[ -z $INSTRUMENT ]]; then
	echo "'-i option' not specified. Search is performed on all available instruments."
	echo ""
else
	echo "'-i option' instrument is set to $INSTRUMENT. "
	echo ""
fi
if [ -z $TIME ]; then
	echo " Ingestion date options not specified ('-t', '-s', '-e').  "
	echo ""
else
	if [ ! -z $isselected_filtertime_ingestion_date ] && [ ! -z $isselected_filtertime_lasthours ]; then
		if [ -z $TIMEFILE ]; then
			echo "'-s option' and '-e option' are set to $INGEGESTION_TIME_FROM and $INGEGESTION_TIME_TO. Search is performed for all products ingested in the period [$INGEGESTION_TIME_FROM,$INGEGESTION_TIME_TO]. "
			echo ""
		else
			echo "'-f option' is specified. Search is performed for all products ingested in the period [$INGEGESTION_TIME_FROM,$INGEGESTION_TIME_TO]. "
			echo ""
		fi
	else
		if [ ! -z $isselected_filtertime_lasthours ]; then
			if [ -z $TIMEFILE ]; then
				echo "'-t option' is set to $TIME. Search is performed for all products ingested in the last $TIME hours. "
				echo ""
			fi
		else
			echo "'-s option' and '-e option' are set to $INGEGESTION_TIME_FROM and $INGEGESTION_TIME_TO. Search is performed for all products ingested in the period [$INGEGESTION_TIME_FROM,$INGEGESTION_TIME_TO]. "
			echo ""
		fi
	fi
fi
if [ -z $SENSING_TIME ]; then
	echo " Sensing date options not specified ('-S', '-E')."
	echo ""
else
	echo "'-S option' and '-E option' are set to $SENSING_TIME_FROM and $SENSING_TIME_TO. Search for all products having sensing date included in [$SENSING_TIME_FROM,$SENSING_TIME_TO]. "
	echo ""
fi
if [ ! -z $TIMEFILE ] && [ -z $isselected_filtertime_ingestion_date ]; then
	echo "'-f option' is set to $TIMEFILE. The ingestion date provided through $TIMEFILE is used to search all products ingested in the period [DATEINGESTION,NOW]. The file is updated with the ingestion date of the last available product found. "
	echo ""
fi
if [ -z $ROW ]; then
	echo "'-c option' not specified. No specified Area of Interest. Search is performed on the whole globe."
	echo ""
else
	echo "'-c option' is set to $x1,$y1:$x2,$y2. Search is performed on an Area of interest defined as a bounding box delimited by the two opposite vertices P1=[lon1=$x1,lat1=$y1] and P2=[lon2=$x2,lat2=$y2]. "
	echo ""
fi
if [ -z $PRODUCT_TYPE ]; then
	echo "'-T option' not specified. Search is performed on all available product types. "
	echo ""
else
	echo "'-T option' product type is set to $PRODUCT_TYPE. "
	echo ""
fi
if [ ! -z $FREE_SUBQUERY_CHECK ]; then
	echo "'-F option' is set to $FREE_SUBQUERY. This OpenSearch query will be in AND with other search options. "
	echo ""
fi
echo "SEARCH RESULT OPTIONS"
echo ""
if [ -z $ROWS ]; then
	echo "'-l option' not specified. Default Maximum Number of Results per page is 25. "
	echo ""
else
	echo "'-l option' is set to $ROWS. The number of results per page is $ROWS. "
	echo ""
fi
if [ -z $PAGE ]; then
	echo "'-P option' not specified. Default Page Number is 1. "
	echo ""
else
	echo "'-P option' is set to $PAGE. The page visualized is $PAGE. "
	echo ""
fi
if [ -z $NAMEFILERESULTS ]; then
	echo "'-q option' not specified. OpenSearch results are stored by default in ./OSquery-result.xml. "
	echo ""
else
	echo "'-q option' is set to $NAMEFILERESULTS. OpenSearch results are stored in $NAMEFILERESULTS. "
	echo ""
fi
if [ -z $PRODUCTLIST ]; then
	echo "'-C option' not specified. List of results are stored by default in the CSV file ./products-list.csv. "
	echo ""
else
	echo "'-C option' is set to $PRODUCTLIST. List of results are stored in the specified CSV file $PRODUCTLIST. "
	echo ""
fi
if [ ! -z $TO_DOWNLOAD ] || [ ! -z $check_retry ]; then
	CHECK_VAR=true
else
	CHECK_VAR=false
fi
echo "DOWNLOAD OPTIONS"
echo ""
if [ $CHECK_VAR == false ]; then
	echo "No download options specified. No files will be downloaded. "
	echo ""
fi
if [ ! -z $TO_DOWNLOAD ]; then
	if [ $TO_DOWNLOAD=="product" ]; then
		echo "'-o option' is set to $TO_DOWNLOAD. Downloads are active. By default product downloads are stored in ./PRODUCT unless differently specified by the '-O option'. "
		echo ""
	else
		if [ $TO_DOWNLOAD=="manifest" ]; then
			echo "'-o option' is set to $TO_DOWNLOAD. Only manifest files are downloaded. Manifest files are stored in ./manifest. "
			echo ""
		else
			echo "'-o option' is set to $TO_DOWNLOAD. Downloads are active. Products and manifest files are downloded separately. "
			echo ""
		fi
	fi
fi

if [[ $CHECK_VAR == true && ! -z $OUTPUT_FOLDER_IS_SET ]]; then
	echo "'-O option' is set to $output_folder. Product downloads are stored in ./$output_folder. "
	echo ""
fi
if [[ $CHECK_VAR == true && -z $number_tries ]]; then
	echo "'-N option' not specified. By default the number of wget download retries is 5. "
	echo ""
else
	if [[ $CHECK_VAR == true && ! -z $number_tries ]]; then
		echo "'-N option' is set to $number_tries. The number of wget download retries is $number_tries. "
		echo ""
	fi
fi
if [[ $CHECK_VAR == true && -z $check_save_failed ]]; then
	echo "'-R option' not specified. By default the list of products failing the MD5 integrity check is saved in ./failed_MD5_check_list.txt. "
	echo ""
else
	if [[ $CHECK_VAR == true && ! -z $check_save_failed ]]; then
		echo "'-R option' is set to $FAILED. The list of products failing the MD5 integrity check is saved in $FAILED. "
		echo ""
	fi
fi
if [[ $CHECK_VAR == true && ! -z $save_products_failed ]]; then
	echo "'-D option' is active. Products that have failed the MD5 integrity check are deleted from the local disks. "
	echo ""
fi
if [[ $CHECK_VAR == true && ! -z $check_retry ]]; then
	echo "'-r option' is set to $FAILED_retry. It retries the download of the products listed in $FAILED_retry. "
	echo ""
fi
if [ ! -z $LOCK_FILE_IS_SET ]; then
	echo "'-L option' is set to $lock_file. This instance of dhusget can be executed in parallel to other instances. "
	echo ""
fi
if [[ $CHECK_VAR == true && -z $THREAD_NUMBER_IS_SET ]]; then
	echo "'-n option' not specified. By default the number of concurrent downloads (either products or manifest files) is 2. "
	echo ""
else
	if [[ $CHECK_VAR == true && ! -z $THREAD_NUMBER_IS_SET ]]; then
		echo "'-n option' is set to $THREAD_NUMBER. The number of concurrent downloads (either products or manifest files) is set to $THREAD_NUMBER. Attention, this value doesn't override the quota limit set on the server side for the user. "
		echo ""
	fi
fi
if [[ $CHECK_VAR == true && -z $OFFLINE_WAIT ]]; then
	export OFFLINE_WAIT="10m"
fi
if [[ $CHECK_VAR == true && -z $OFFLINE_RETRY_ATTEMPTS ]]; then
	export OFFLINE_RETRY_ATTEMPTS="40"
fi

echo "================================================================================================================"
echo ""
if [[ ! -z $MISSION ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT platformname:$MISSION"
	QUERY_STATEMENT_CHECK='OK'
fi
if [[ ! -z $INSTRUMENT ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT instrumentshortname:$INSTRUMENT"
	QUERY_STATEMENT_CHECK='OK'
fi
if [[ ! -z $PRODUCT_TYPE ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT producttype:$PRODUCT_TYPE"
	QUERY_STATEMENT_CHECK='OK'
fi
if [[ ! -z $TIME ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT ${TIME_SUBQUERY}"
	QUERY_STATEMENT_CHECK='OK'
fi

if [[ ! -z $SENSING_TIME ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT ${SENSING_SUBQUERY}"
	QUERY_STATEMENT_CHECK='OK'
fi

if [[ ! -z $TIMEFILE ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT ${TIME_SUBQUERY}"

	QUERY_STATEMENT_CHECK='OK'
fi

if [[ ! -z $FREE_SUBQUERY_CHECK ]]; then
	if [ ! -z $QUERY_STATEMENT_CHECK ]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export QUERY_STATEMENT="$QUERY_STATEMENT $FREE_SUBQUERY"
	QUERY_STATEMENT_CHECK='OK'
fi

#---- Prepare query polygon statement
if [[ ! -z $x1 ]]; then
	if [[ ! -z $QUERY_STATEMENT ]]; then
		export QUERY_STATEMENT="$QUERY_STATEMENT AND "
	fi
	export GEO_SUBQUERY=$(
		LC_NUMERIC=en_US.UTF-8
		printf "( footprint:\"Intersects(POLYGON((%.13f %.13f,%.13f %.13f,%.13f %.13f,%.13f %.13f,%.13f %.13f )))\")" $x1 $y1 $x2 $y1 $x2 $y2 $x1 $y2 $x1 $y1
	)
	export QUERY_STATEMENT=${QUERY_STATEMENT}" ${GEO_SUBQUERY}"
else
	export GEO_SUBQUERY=""
fi
#- ... append on query (without repl
if [[ -z $ROWS ]]; then
	export ROWS=25
fi

if [[ -z $PAGE ]]; then
	export PAGE=1
fi
START=$((PAGE - 1))
START=$((START * ROWS))
export QUERY_STATEMENT="${DHUS_DEST}/search?q="${QUERY_STATEMENT}"&rows="${ROWS}"&start="${START}""
echo "HTTP request done: "$QUERY_STATEMENT""
echo ""
#--- Execute query statement
if [ -z $NAMEFILERESULTS ]; then
	export NAMEFILERESULTS="OSquery-result.xml"
fi
/bin/rm -f $NAMEFILERESULTS
${WC} ${AUTH} ${TRIES} -c -O "${NAMEFILERESULTS}" "${QUERY_STATEMENT}"
LASTDATE=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)
sleep 2

cat $PWD/"${NAMEFILERESULTS}" | grep '<id>' | tail -n +2 | cut -f2 -d'>' | cut -f1 -d'<' | cat -n >.product_id_list

cat $PWD/"${NAMEFILERESULTS}" | grep '<link rel="alternative" href=' | cut -f4 -d'"' | cat -n | sed 's/\/$//' >.product_link_list

cat $PWD/"${NAMEFILERESULTS}" | grep '<title>' | tail -n +2 | cut -f2 -d'>' | cut -f1 -d'<' | cat -n >.product_title_list
if [ ! -z $TIMEFILE ]; then
	if [ $(cat "${NAMEFILERESULTS}" | grep '="ingestiondate"' | head -n 1 | cut -f2 -d'>' | cut -f1 -d'<' | wc -l) -ne 0 ]; then
		lastdate=$(cat $PWD/"${NAMEFILERESULTS}" | grep '="ingestiondate"' | head -n 1 | cut -f2 -d'>' | cut -f1 -d'<')
		years=$(echo $lastdate | tr "T" '\n' | head -n 1)
		hours=$(echo $lastdate | tr "T" '\n' | tail -n 1)
		echo $(date +%Y-%m-%d --date="$years")"T"$(date +%T.%NZ -u --date="$hours + 0.001 seconds") >$TIMEFILE
	fi
fi

paste -d\\n .product_id_list .product_title_list | sed 's/[",:]/ /g' >product_list

cat .product_title_list .product_link_list | sort -k1n,1 -k2r,2 | sed 's/[",:]/ /g' | sed 's/https \/\//https:\/\//' >.product_list_withlink

rm -f .product_id_list .product_link_list .product_title_list .product_ingestion_time_list

echo ""

cat "${NAMEFILERESULTS}" | grep '<subtitle>' | cut -f2 -d'>' | cut -f1 -d'<' | cat -n

NPRODUCT=$(cat "${NAMEFILERESULTS}" | grep '<subtitle>' | cut -f2 -d'>' | cut -f1 -d'<' | cat -n | cut -f11 -d' ')

echo ""

if [ "${NPRODUCT}" == "0" ]; then exit 1; fi

cat .product_list_withlink
if [ -z $PRODUCTLIST ]; then
	export PRODUCTLIST="products-list.csv"
fi
cp .product_list_withlink $PRODUCTLIST
cat $PRODUCTLIST | cut -f2 -d$'\t' >.products-list-tmp.csv
cat .products-list-tmp.csv | grep -v 'https' >.list_name_products.csv
cat .products-list-tmp.csv | grep 'https' >.list_link_to_products.csv
paste -d',' .list_name_products.csv .list_link_to_products.csv >$PRODUCTLIST
rm .product_list_withlink .products-list-tmp.csv .list_name_products.csv .list_link_to_products.csv
export rv=0
if [ "${TO_DOWNLOAD}" == "manifest" -o "${TO_DOWNLOAD}" == "all" ]; then
	export INPUT_FILE=product_list

	if [ ! -f ${INPUT_FILE} ]; then
		echo "Error: Input file ${INPUT_FILE} not present "
		exit
	fi

	mkdir -p MANIFEST/

	cat ${INPUT_FILE} | xargs -n 4 -P ${THREAD_NUMBER} sh -c 'while : ; do
	echo "Downloading manifest ${3} from link ${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/Nodes('\''"$3".SAFE'\'')/Nodes('\'manifest.safe\'')/\$value"; 
	${WC} ${AUTH} ${TRIES} --progress=dot -e dotbytes=10M -c --output-file=./logs/log.${3}.log -O ./MANIFEST/manifest.safe-${3} "${DHUS_DEST}/odata/v1/Products('\''"$1"'\'')/Nodes('\''"$3".SAFE'\'')/Nodes('\'manifest.safe\'')/\$value" ;
	test=$?;
	if [ $test -eq 0 ]; then
		echo "Manifest ${3} successfully downloaded at " `tail -2 ./logs/log.${3}.log | head -1 | awk -F"(" '\''{print $2}'\'' | awk -F")" '\''{print $1}'\''`;
	fi;
	[[ $test -ne 0 ]] || break;
done '
fi

if [ "${TO_DOWNLOAD}" == "product" -o "${TO_DOWNLOAD}" == "all" ]; then

	export INPUT_FILE=product_list

	mkdir -p $output_folder

	#Xargs works here as a thread pool, it launches a download for each thread (P 2), each single thread checks
	#if the download is completed succesfully.
	#The condition "[[ $? -ne 0 ]] || break" checks the first operand, if it is satisfied the break is skipped, instead if it fails
	#(download completed succesfully (?$=0 )) the break in the OR is executed exiting from the intitial "while".
	#At this point the current thread is released and another one is launched.
	if [ -f .failed.control.now.txt ]; then
		rm .failed.control.now.txt
	fi

	# Call method for product download, if the download option is selected
	# remove the Temporary offline product download list before starting the current download
	if [ -f ${TMP_OFFLINE_PRODUCT_LIST} ]; then

		rm ${TMP_OFFLINE_PRODUCT_LIST}
	fi

	download_product ${INPUT_FILE} ${THREAD_NUMBER}
	#echo "OFFLINE_RETRY_ATTEMPTS  is ${OFFLINE_RETRY_ATTEMPTS}"
	# Check if there are product offline scheduled for download and if the max number of download attempts has been reached
	while [[ -f ${TMP_OFFLINE_PRODUCT_LIST} ]] && [[ ${OFFLINE_RETRY_ATTEMPTS_COUNTER} -lt ${OFFLINE_RETRY_ATTEMPTS} ]]; do
		offline_products=$(cat ${TMP_OFFLINE_PRODUCT_LIST} | wc -l)
		echo "Found ${offline_products} offline products. The download of these products will be tried again in ${OFFLINE_WAIT::-1} minutes"
		#remove old list
		if [ -f ${OFFLINE_PRODUCT_LIST} ]; then

			rm ${OFFLINE_PRODUCT_LIST}
		fi
		#Create new input list for offline products
		cat "${TMP_OFFLINE_PRODUCT_LIST}" | while IFS=, read idx uuid name; do
			echo "$idx $uuid" | tee -a ${OFFLINE_PRODUCT_LIST}
			echo "$idx $name" | tee -a ${OFFLINE_PRODUCT_LIST}
			# do something
		done
		rm ${TMP_OFFLINE_PRODUCT_LIST}
		#echo "sleep time is ${OFFLINE_WAIT}"
		# wait for a configurable number of minutes
		sleep ${OFFLINE_WAIT}
		# increase the number of offline products download attempts
		OFFLINE_RETRY_ATTEMPTS_COUNTER=$((OFFLINE_RETRY_ATTEMPTS_COUNTER + 1))
		# try again to download the offline products
		echo "Attempt #${OFFLINE_RETRY_ATTEMPTS_COUNTER} to get offline products..."
		download_product ${OFFLINE_PRODUCT_LIST} ${THREAD_NUMBER}
	done
	if [[ ${OFFLINE_RETRY_ATTEMPTS_COUNTER} -ge ${OFFLINE_RETRY_ATTEMPTS} ]]; then
		offline_products=$(cat ${TMP_OFFLINE_PRODUCT_LIST} | wc -l)
		echo "Reached the maximum number of attempts to retry the offline products download".
		echo "Products not downloaded are ${offline_products} ".
	fi

fi

if [ ! -z $check_save_failed ]; then
	if [ -f .failed.control.now.txt ]; then
		mv .failed.control.now.txt $FAILED
	else
		if [ ! -f .failed.control.now.txt ] && [ $CHECK_VAR == true ] && [ ! ISSELECTEDEXIT ]; then
			echo "All downloaded products have successfully passed MD5 integrity check"
		fi
	fi
else
	if [ -f .failed.control.now.txt ]; then
		mv .failed.control.now.txt failed_MD5_check_list.txt
	else
		if [ ! -f .failed.control.now.txt ] && [ $CHECK_VAR == true ] && [ ! ISSELECTEDEXIT ]; then
			echo "All downloaded products have successfully passed MD5 integrity check"
		fi
	fi
fi
echo 'the end'