diff --git a/CHANGELOG b/CHANGELOG
index 2d6f67a2..d2012105 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,9 @@
+2020-05-21 Release 1.20
+ - fixed bug with JSONPath &selector
+ - added long description in setup.py for PyPi (from README.md)
+ - added link files for in-browser tests
+ - remove unneeded system library from DockerFile
+
2020-04-27 Release 1.19
- add UI support for labeling steps in a recipe
- added field on data-source page to specify a JSONpath selector to find the HXL data
diff --git a/Dockerfile b/Dockerfile
index d030a80d..7f07c3b9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,8 @@ RUN apk update && \
python3-dev && \
pip3 install gevent && \
apk del \
- .gevent-deps && \
+ .gevent-deps \
+ libffi-dev && \
rm -rf /root/.cache && \
rm -rf /var/cache/apk/*
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..ee1fb4b4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,94 @@
+########################################################################
+# Makefile to automate common tasks
+#
+# Test-related targets:
+#
+# (These require the Python3 venv module)
+#
+# build-venv - (re)build the Python virtual environment if needed
+# test - run the unit tests (building a virtual environment if needed)
+# test-install - test a fresh installation in a temporary virtual environment
+#
+# Git-related targets:
+#
+# (All of these check out the dev branch at the end)
+#
+# close-issue - merge the current issue branch into dev and delete
+# push-dev - push current dev branch to upstream
+# merge-test - merge the dev branch into the test branch and push
+# merge-master - merge the test branch into the master branch and push
+#
+# Other:
+#
+# etags - build an Emacs TAGS file
+# restart - touch hxl-proxy.wsgi to restart the app
+########################################################################
+
+
+# figure out what branch we're on currently
+BRANCH=$(shell git symbolic-ref --short HEAD)
+
+# activation script for the Python virtual environment
+VENV=venv/bin/activate
+
+
+# run unit tests
+test: $(VENV)
+ . $(VENV) && python setup.py test
+
+# alias to (re)build the Python virtual environment
+build-venv: $(VENV)
+
+# (re)build the virtual environment if it's missing, or whenever setup.py changes
+$(VENV): setup.py
+ rm -rf venv && virtualenv venv
+ . $(VENV) && cd ../libhxl-python && python setup.py develop && cd ../hxl-proxy && python setup.py develop
+
+# close the current issue branch and merge into dev
+close-issue:
+ git checkout dev && git merge "$(BRANCH)" && git branch -d "$(BRANCH)"
+
+# push the dev branch to origin
+push-dev:
+ git checkout dev && git push
+
+# merge the dev branch into test and push both to origin
+merge-test:
+ git checkout test && git merge dev && git push && git checkout dev
+
+# merge the test branch into master and push both to origin
+merge-master: merge-test
+ git checkout master && git merge test && git push && git checkout dev
+
+# do a cold install in a temporary virtual environment and run unit tests
+test-install:
+ rm -rf venv-test && python3 -m venv venv-test && . venv-test/bin/activate && python setup.py install && python setup.py test
+ rm -rf venv-test # make sure we clean up
+
+# Add target for docker build
+test-docker:
+ pkexec docker build --no-cache -t "unocha/hxl-proxy:local" `pwd`
+
+# browser tests for dev.proxy.hxlstandard.org
+browser-tests-dev:
+ cat tests/browser-tests/dev-urls.txt | xargs -d '\n' firefox
+
+# browser tests for beta.proxy.hxlstandard.org
+browser-tests-beta:
+ cat tests/browser-tests/beta-urls.txt | xargs -d '\n' firefox
+
+# browser tests for dev.proxy-hxlstandard-org.ahconu.org
+browser-tests-staging:
+ cat tests/browser-tests/staging-urls.txt | xargs -d '\n' firefox
+
+# browser tests for proxy.hxlstandard.org
+browser-tests-prod:
+ cat tests/browser-tests/prod-urls.txt | xargs -d '\n' firefox
+
+# (re)generate emacs TAGS file
+etags:
+ find hxl_proxy tests -name '*.py' -o -name '*.csv' | xargs etags
+
+# restart the web app by touching the WSGI file (depends on the platform)
+restart:
+ touch hxl-proxy.wsgi
diff --git a/hxl_proxy/__init__.py b/hxl_proxy/__init__.py
index b28d3e8e..62911df8 100644
--- a/hxl_proxy/__init__.py
+++ b/hxl_proxy/__init__.py
@@ -16,7 +16,7 @@
import werkzeug.datastructures
from . import reverse_proxied
-__version__="1.19"
+__version__="1.20"
"""Module version number
See https://www.python.org/dev/peps/pep-0396/
"""
diff --git a/hxl_proxy/templates/about.html b/hxl_proxy/templates/about.html
index dae73d75..a01a84e0 100644
--- a/hxl_proxy/templates/about.html
+++ b/hxl_proxy/templates/about.html
@@ -90,7 +90,7 @@
What about private data?
Proxy web application on your personal computer, or on an
Intranet inside your organisation's firewall. There is a
Docker
+ href="https://hub.docker.com/r/unocha/hxl-proxy">Docker
image available for fast deployment.
diff --git a/hxl_proxy/templates/includes/metadata-params.html b/hxl_proxy/templates/includes/metadata-params.html
index 2c959065..a219376a 100644
--- a/hxl_proxy/templates/includes/metadata-params.html
+++ b/hxl_proxy/templates/includes/metadata-params.html
@@ -1,6 +1,6 @@
- {% for name in ['url', 'sheet', 'name', 'description', 'stub', 'schema_url', 'cloneable', 'authorization_token' ] %}
+ {% for name in ['url', 'sheet', 'name', 'description', 'stub', 'schema_url', 'cloneable', 'authorization_token', 'selector' ] %}
{% if recipe.args.get(name) and (not excludes or name not in excludes) %}
{% endif %}
diff --git a/requirements.txt b/requirements.txt
index e7e69d69..09f1d16e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-libhxl==4.19
+libhxl==4.20
requests>=2.11
ckanapi>=3.5
flask>=1.0
diff --git a/setup.py b/setup.py
index 36d15f48..a990302f 100755
--- a/setup.py
+++ b/setup.py
@@ -9,12 +9,17 @@
if sys.version_info.major != 3:
raise SystemExit("The HXL Proxy requires Python 3.x")
+with open("README.md", "r") as fh:
+ long_description = fh.read()
+
setuptools.setup(
name = 'hxl-proxy',
packages = ['hxl_proxy'],
package_data={'hxl_proxy': ['*.sql']},
- version = "1.19",
+ version = "1.20",
description = 'Flask-based web proxy for HXL',
+ long_description=long_description,
+ long_description_content_type="text/markdown",
author='David Megginson',
author_email='contact@megginson.com',
url='https://github.com/HXLStandard/hxl-proxy',
@@ -22,7 +27,7 @@
zip_safe = False,
install_requires=[
'requests>=2.11',
- 'libhxl==4.19',
+ 'libhxl==4.20',
'ckanapi>=3.5',
'flask>=1.0',
'flask-caching',
diff --git a/tests/browser-tests/README.md b/tests/browser-tests/README.md
new file mode 100644
index 00000000..2a0536f2
--- /dev/null
+++ b/tests/browser-tests/README.md
@@ -0,0 +1,15 @@
+Links for in-browser tests
+==========================
+
+This directory contains lists of URLs for quick (at-a-glance) in-browser tests for the HXL Proxy.
+
+# Usage
+
+ $ cat FILE | xargs -d '\n' firefox
+
+# Available URL files
+
+* dev-urls.txt - development test links for dev.proxy.hxlstandard.org (assuming it's set in your /etc/hosts file to a local instance)
+* beta-urls.txt - public beta test links for beta.proxy.hxlstandard.org
+* staging-urls.txt - pre-release staging test links for dev.proxy-hxlstandard-org.ahconu.org
+* prod-urls.txt - production test links for proxy.hxlstandard.org
diff --git a/tests/browser-tests/beta-urls.txt b/tests/browser-tests/beta-urls.txt
new file mode 100644
index 00000000..b95de250
--- /dev/null
+++ b/tests/browser-tests/beta-urls.txt
@@ -0,0 +1,9 @@
+https://beta.proxy.hxlstandard.org/about.html
+https://beta.proxy.hxlstandard.org/data/edit?filter_count=7&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ytPD-f4a8CbNKTfMS3EqZOpBo9LWCk_NDKxJCgmpXA8%2Fedit%23gid%3D1101521524&format=html
+https://beta.proxy.hxlstandard.org/data/bf52b1
+https://beta.proxy.hxlstandard.org/data/edit?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1_PstA2iB0W1VdA9zhj2hqyQPFkf_7eZJBhOO3MjxNaE%2Fedit%3Fusp%3Dsharing
+https://beta.proxy.hxlstandard.org/data?filter01=select&select-query01-01=%23country%2Bcode%3DMLI&filter02=select&select-query02-01=date%2Byear%2Breference+is+max&filter03=add&add-tag03=%23indicator%2Bnutrition%2Bat_risk%2Bpct&add-value03=%7B%7B+round%28%23affected%2Btotal_at_risk+%2F+%23population+%2A+100%29+%7D%7D&add-header03=Percentage+at+risk&add-before03=on&force=on&tagger-match-all=on&tagger-01-header=adm0_name&tagger-01-tag=%23country%2Bname&tagger-03-header=adm0_pcod3&tagger-03-tag=%23country%2Bcode%2Bv_iso3&tagger-04-header=adm0_pcod2&tagger-04-tag=%23country%2Bcode%2Bv_iso2&tagger-05-header=region&tagger-05-tag=%23region&tagger-06-header=adm1_name&tagger-06-tag=%23adm1%2Bname&tagger-09-header=adm1_pcod2&tagger-09-tag=%23adm1%2Bcode%2Bv_pcode2&tagger-10-header=adm2_name&tagger-10-tag=%23adm2%2Bname&tagger-13-header=adm2_pcod2&tagger-13-tag=%23adm2%2Bcode%2Bv_pcode2&tagger-14-header=population&tagger-14-tag=%23population&tagger-15-header=phase_class&tagger-15-tag=%23meta%2Bphase_class&tagger-16-header=phase1&tagger-16-tag=%23affected%2Bnutrition%2Bminimal&tagger-17-header=phase2&tagger-17-tag=%23affected%2Bnutrition%2Bstressed&tagger-18-header=phase3&tagger-18-tag=%23affected%2Bnutrition%2Bcrisis&tagger-19-header=phase4&tagger-19-tag=%23affected%2Bnutrition%2Bemergency&tagger-20-header=phase5&tagger-20-tag=%23affected%2Bnutrition%2Bfamine&tagger-21-header=phase35&tagger-21-tag=%23affected%2Bnutrition%2Btotal_at_risk&tagger-22-header=chtype&tagger-22-tag=%23status&tagger-23-header=exercise_code&tagger-23-tag=%23meta%2Bexercise_code&tagger-24-header=exercise_label&tagger-24-tag=%23meta%2Bexercise_months&tagger-25-header=exercise_year&tagger-25-tag=%23date%2Byear%2Bexercise&tagger-26-header=reference_code&tagger-26-tag=%23meta%2Breference_code&tagger-27-header=reference_label&tagger-27-tag=%23meta%2Breference_months&tagger-28-header=reference_year&tagger-28-tag=%23date%2Byear%2Breference&header-row=1&url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise&name=Cadre+harmonis%C3%A9+%28IPC%29+for+Mali+-+latest+year
+https://beta.proxy.hxlstandard.org/data?name=Humanitarian+Response+Plans&url=http%3A%2F%2Fapi.hpc.tools%2Fv2%2Fpublic%2Fplan&tagger-match-all=on&tagger-02-header=name&tagger-02-tag=%23response%2Bname&tagger-03-header=code&tagger-03-tag=%23response%2Bcode&tagger-04-header=startdate&tagger-04-tag=%23date%2Bstart&tagger-05-header=enddate&tagger-05-tag=%23date%2Bend&tagger-09-header=years&tagger-09-tag=%23date%2Byear&tagger-10-header=locations&tagger-10-tag=%23country%2Bcode%2Blist&tagger-11-header=categories&tagger-11-tag=%23response%2Btype&tagger-12-header=revisedrequirements&tagger-12-tag=%23value%2Brequirements%2Brevised%2Bc_usd&tagger-13-header=origrequirements&tagger-13-tag=%23value%2Brequirements%2Borig%2Bc_usd&header-row=1&filter01=cut&cut-skip-untagged01=on&filter02=clean&clean-date-tags02=%23date&filter03=jsonpath&jsonpath-path03=%24[0].year&jsonpath-patterns03=date%2Byear&filter04=jsonpath&jsonpath-path04=%24[0].name&jsonpath-patterns04=%23response%2Btype&filter05=jsonpath&jsonpath-path05=%24[*].iso3&jsonpath-patterns05=%23country%2Bcode%2Blist&filter06=replace&replace-pattern06=[^A-Z%2C]*&replace-regex06=on&replace-tags06=%23country%2Bcode%2Blist&filter07=replace&replace-pattern07=^%2C%2B&replace-regex07=on&replace-tags07=%23country%2Bcode%2Blist&filter08=replace&replace-pattern08=%2C%2B%24&replace-regex08=on&replace-tags08=country%2Bcode%2Blist&filter09=replace&replace-pattern09=%2C%2B&replace-regex09=on&replace-value09=%2C+&replace-tags09=country%2Bcode%2Blist&filter10=sort&sort-tags10=date%2Bstart&sort-reverse10=on
+https://beta.proxy.hxlstandard.org/data/validate?schema_url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D152581637&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D0
+https://beta.proxy.hxlstandard.org/pcodes/mli-adm1.csv
+https://beta.proxy.hxlstandard.org/api/data-preview.json?url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise
diff --git a/tests/browser-tests/dev-urls.txt b/tests/browser-tests/dev-urls.txt
new file mode 100644
index 00000000..7373a136
--- /dev/null
+++ b/tests/browser-tests/dev-urls.txt
@@ -0,0 +1,9 @@
+https://dev.proxy.hxlstandard.org/about.html
+https://dev.proxy.hxlstandard.org/data/edit?filter_count=7&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ytPD-f4a8CbNKTfMS3EqZOpBo9LWCk_NDKxJCgmpXA8%2Fedit%23gid%3D1101521524&format=html
+https://dev.proxy.hxlstandard.org/data/c17c78
+https://dev.proxy.hxlstandard.org/data/edit?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1_PstA2iB0W1VdA9zhj2hqyQPFkf_7eZJBhOO3MjxNaE%2Fedit%3Fusp%3Dsharing
+https://dev.proxy.hxlstandard.org/data?filter01=select&select-query01-01=%23country%2Bcode%3DMLI&filter02=select&select-query02-01=date%2Byear%2Breference+is+max&filter03=add&add-tag03=%23indicator%2Bnutrition%2Bat_risk%2Bpct&add-value03=%7B%7B+round%28%23affected%2Btotal_at_risk+%2F+%23population+%2A+100%29+%7D%7D&add-header03=Percentage+at+risk&add-before03=on&force=on&tagger-match-all=on&tagger-01-header=adm0_name&tagger-01-tag=%23country%2Bname&tagger-03-header=adm0_pcod3&tagger-03-tag=%23country%2Bcode%2Bv_iso3&tagger-04-header=adm0_pcod2&tagger-04-tag=%23country%2Bcode%2Bv_iso2&tagger-05-header=region&tagger-05-tag=%23region&tagger-06-header=adm1_name&tagger-06-tag=%23adm1%2Bname&tagger-09-header=adm1_pcod2&tagger-09-tag=%23adm1%2Bcode%2Bv_pcode2&tagger-10-header=adm2_name&tagger-10-tag=%23adm2%2Bname&tagger-13-header=adm2_pcod2&tagger-13-tag=%23adm2%2Bcode%2Bv_pcode2&tagger-14-header=population&tagger-14-tag=%23population&tagger-15-header=phase_class&tagger-15-tag=%23meta%2Bphase_class&tagger-16-header=phase1&tagger-16-tag=%23affected%2Bnutrition%2Bminimal&tagger-17-header=phase2&tagger-17-tag=%23affected%2Bnutrition%2Bstressed&tagger-18-header=phase3&tagger-18-tag=%23affected%2Bnutrition%2Bcrisis&tagger-19-header=phase4&tagger-19-tag=%23affected%2Bnutrition%2Bemergency&tagger-20-header=phase5&tagger-20-tag=%23affected%2Bnutrition%2Bfamine&tagger-21-header=phase35&tagger-21-tag=%23affected%2Bnutrition%2Btotal_at_risk&tagger-22-header=chtype&tagger-22-tag=%23status&tagger-23-header=exercise_code&tagger-23-tag=%23meta%2Bexercise_code&tagger-24-header=exercise_label&tagger-24-tag=%23meta%2Bexercise_months&tagger-25-header=exercise_year&tagger-25-tag=%23date%2Byear%2Bexercise&tagger-26-header=reference_code&tagger-26-tag=%23meta%2Breference_code&tagger-27-header=reference_label&tagger-27-tag=%23meta%2Breference_months&tagger-28-header=reference_year&tagger-28-tag=%23date%2Byear%2Breference&header-row=1&url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise&name=Cadre+harmonis%C3%A9+%28IPC%29+for+Mali+-+latest+year
+https://dev.proxy.hxlstandard.org/data?name=Humanitarian+Response+Plans&url=http%3A%2F%2Fapi.hpc.tools%2Fv2%2Fpublic%2Fplan&tagger-match-all=on&tagger-02-header=name&tagger-02-tag=%23response%2Bname&tagger-03-header=code&tagger-03-tag=%23response%2Bcode&tagger-04-header=startdate&tagger-04-tag=%23date%2Bstart&tagger-05-header=enddate&tagger-05-tag=%23date%2Bend&tagger-09-header=years&tagger-09-tag=%23date%2Byear&tagger-10-header=locations&tagger-10-tag=%23country%2Bcode%2Blist&tagger-11-header=categories&tagger-11-tag=%23response%2Btype&tagger-12-header=revisedrequirements&tagger-12-tag=%23value%2Brequirements%2Brevised%2Bc_usd&tagger-13-header=origrequirements&tagger-13-tag=%23value%2Brequirements%2Borig%2Bc_usd&header-row=1&filter01=cut&cut-skip-untagged01=on&filter02=clean&clean-date-tags02=%23date&filter03=jsonpath&jsonpath-path03=%24[0].year&jsonpath-patterns03=date%2Byear&filter04=jsonpath&jsonpath-path04=%24[0].name&jsonpath-patterns04=%23response%2Btype&filter05=jsonpath&jsonpath-path05=%24[*].iso3&jsonpath-patterns05=%23country%2Bcode%2Blist&filter06=replace&replace-pattern06=[^A-Z%2C]*&replace-regex06=on&replace-tags06=%23country%2Bcode%2Blist&filter07=replace&replace-pattern07=^%2C%2B&replace-regex07=on&replace-tags07=%23country%2Bcode%2Blist&filter08=replace&replace-pattern08=%2C%2B%24&replace-regex08=on&replace-tags08=country%2Bcode%2Blist&filter09=replace&replace-pattern09=%2C%2B&replace-regex09=on&replace-value09=%2C+&replace-tags09=country%2Bcode%2Blist&filter10=sort&sort-tags10=date%2Bstart&sort-reverse10=on
+https://dev.proxy.hxlstandard.org/data/validate?schema_url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D152581637&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D0
+https://dev.proxy.hxlstandard.org/pcodes/mli-adm1.csv
+https://dev.proxy.hxlstandard.org/api/data-preview.json?url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise
diff --git a/tests/browser-tests/prod-urls.txt b/tests/browser-tests/prod-urls.txt
new file mode 100644
index 00000000..3c502e69
--- /dev/null
+++ b/tests/browser-tests/prod-urls.txt
@@ -0,0 +1,9 @@
+https://proxy.hxlstandard.org/about.html
+https://proxy.hxlstandard.org/data/edit?filter_count=7&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ytPD-f4a8CbNKTfMS3EqZOpBo9LWCk_NDKxJCgmpXA8%2Fedit%23gid%3D1101521524&format=html
+https://proxy.hxlstandard.org/data/0ce003
+https://proxy.hxlstandard.org/data/edit?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1_PstA2iB0W1VdA9zhj2hqyQPFkf_7eZJBhOO3MjxNaE%2Fedit%3Fusp%3Dsharing
+https://proxy.hxlstandard.org/data?filter01=select&select-query01-01=%23country%2Bcode%3DMLI&filter02=select&select-query02-01=date%2Byear%2Breference+is+max&filter03=add&add-tag03=%23indicator%2Bnutrition%2Bat_risk%2Bpct&add-value03=%7B%7B+round%28%23affected%2Btotal_at_risk+%2F+%23population+%2A+100%29+%7D%7D&add-header03=Percentage+at+risk&add-before03=on&force=on&tagger-match-all=on&tagger-01-header=adm0_name&tagger-01-tag=%23country%2Bname&tagger-03-header=adm0_pcod3&tagger-03-tag=%23country%2Bcode%2Bv_iso3&tagger-04-header=adm0_pcod2&tagger-04-tag=%23country%2Bcode%2Bv_iso2&tagger-05-header=region&tagger-05-tag=%23region&tagger-06-header=adm1_name&tagger-06-tag=%23adm1%2Bname&tagger-09-header=adm1_pcod2&tagger-09-tag=%23adm1%2Bcode%2Bv_pcode2&tagger-10-header=adm2_name&tagger-10-tag=%23adm2%2Bname&tagger-13-header=adm2_pcod2&tagger-13-tag=%23adm2%2Bcode%2Bv_pcode2&tagger-14-header=population&tagger-14-tag=%23population&tagger-15-header=phase_class&tagger-15-tag=%23meta%2Bphase_class&tagger-16-header=phase1&tagger-16-tag=%23affected%2Bnutrition%2Bminimal&tagger-17-header=phase2&tagger-17-tag=%23affected%2Bnutrition%2Bstressed&tagger-18-header=phase3&tagger-18-tag=%23affected%2Bnutrition%2Bcrisis&tagger-19-header=phase4&tagger-19-tag=%23affected%2Bnutrition%2Bemergency&tagger-20-header=phase5&tagger-20-tag=%23affected%2Bnutrition%2Bfamine&tagger-21-header=phase35&tagger-21-tag=%23affected%2Bnutrition%2Btotal_at_risk&tagger-22-header=chtype&tagger-22-tag=%23status&tagger-23-header=exercise_code&tagger-23-tag=%23meta%2Bexercise_code&tagger-24-header=exercise_label&tagger-24-tag=%23meta%2Bexercise_months&tagger-25-header=exercise_year&tagger-25-tag=%23date%2Byear%2Bexercise&tagger-26-header=reference_code&tagger-26-tag=%23meta%2Breference_code&tagger-27-header=reference_label&tagger-27-tag=%23meta%2Breference_months&tagger-28-header=reference_year&tagger-28-tag=%23date%2Byear%2Breference&header-row=1&url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise&name=Cadre+harmonis%C3%A9+%28IPC%29+for+Mali+-+latest+year
+https://proxy.hxlstandard.org/data?name=Humanitarian+Response+Plans&url=http%3A%2F%2Fapi.hpc.tools%2Fv2%2Fpublic%2Fplan&tagger-match-all=on&tagger-02-header=name&tagger-02-tag=%23response%2Bname&tagger-03-header=code&tagger-03-tag=%23response%2Bcode&tagger-04-header=startdate&tagger-04-tag=%23date%2Bstart&tagger-05-header=enddate&tagger-05-tag=%23date%2Bend&tagger-09-header=years&tagger-09-tag=%23date%2Byear&tagger-10-header=locations&tagger-10-tag=%23country%2Bcode%2Blist&tagger-11-header=categories&tagger-11-tag=%23response%2Btype&tagger-12-header=revisedrequirements&tagger-12-tag=%23value%2Brequirements%2Brevised%2Bc_usd&tagger-13-header=origrequirements&tagger-13-tag=%23value%2Brequirements%2Borig%2Bc_usd&header-row=1&filter01=cut&cut-skip-untagged01=on&filter02=clean&clean-date-tags02=%23date&filter03=jsonpath&jsonpath-path03=%24[0].year&jsonpath-patterns03=date%2Byear&filter04=jsonpath&jsonpath-path04=%24[0].name&jsonpath-patterns04=%23response%2Btype&filter05=jsonpath&jsonpath-path05=%24[*].iso3&jsonpath-patterns05=%23country%2Bcode%2Blist&filter06=replace&replace-pattern06=[^A-Z%2C]*&replace-regex06=on&replace-tags06=%23country%2Bcode%2Blist&filter07=replace&replace-pattern07=^%2C%2B&replace-regex07=on&replace-tags07=%23country%2Bcode%2Blist&filter08=replace&replace-pattern08=%2C%2B%24&replace-regex08=on&replace-tags08=country%2Bcode%2Blist&filter09=replace&replace-pattern09=%2C%2B&replace-regex09=on&replace-value09=%2C+&replace-tags09=country%2Bcode%2Blist&filter10=sort&sort-tags10=date%2Bstart&sort-reverse10=on
+https://proxy.hxlstandard.org/data/validate?schema_url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D152581637&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D0
+https://proxy.hxlstandard.org/pcodes/mli-adm1.csv
+https://proxy.hxlstandard.org/api/data-preview.json?url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise
diff --git a/tests/browser-tests/staging-urls.txt b/tests/browser-tests/staging-urls.txt
new file mode 100644
index 00000000..e648189e
--- /dev/null
+++ b/tests/browser-tests/staging-urls.txt
@@ -0,0 +1,9 @@
+https://dev.proxy-hxlstandard-org.ahconu.org/about.html
+https://dev.proxy-hxlstandard-org.ahconu.org/data/edit?filter_count=7&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ytPD-f4a8CbNKTfMS3EqZOpBo9LWCk_NDKxJCgmpXA8%2Fedit%23gid%3D1101521524&format=html
+https://dev.proxy-hxlstandard-org.ahconu.org/data/df9944
+https://dev.proxy-hxlstandard-org.ahconu.org/data/edit?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1_PstA2iB0W1VdA9zhj2hqyQPFkf_7eZJBhOO3MjxNaE%2Fedit%3Fusp%3Dsharing
+https://dev.proxy-hxlstandard-org.ahconu.org/data?filter01=select&select-query01-01=%23country%2Bcode%3DMLI&filter02=select&select-query02-01=date%2Byear%2Breference+is+max&filter03=add&add-tag03=%23indicator%2Bnutrition%2Bat_risk%2Bpct&add-value03=%7B%7B+round%28%23affected%2Btotal_at_risk+%2F+%23population+%2A+100%29+%7D%7D&add-header03=Percentage+at+risk&add-before03=on&force=on&tagger-match-all=on&tagger-01-header=adm0_name&tagger-01-tag=%23country%2Bname&tagger-03-header=adm0_pcod3&tagger-03-tag=%23country%2Bcode%2Bv_iso3&tagger-04-header=adm0_pcod2&tagger-04-tag=%23country%2Bcode%2Bv_iso2&tagger-05-header=region&tagger-05-tag=%23region&tagger-06-header=adm1_name&tagger-06-tag=%23adm1%2Bname&tagger-09-header=adm1_pcod2&tagger-09-tag=%23adm1%2Bcode%2Bv_pcode2&tagger-10-header=adm2_name&tagger-10-tag=%23adm2%2Bname&tagger-13-header=adm2_pcod2&tagger-13-tag=%23adm2%2Bcode%2Bv_pcode2&tagger-14-header=population&tagger-14-tag=%23population&tagger-15-header=phase_class&tagger-15-tag=%23meta%2Bphase_class&tagger-16-header=phase1&tagger-16-tag=%23affected%2Bnutrition%2Bminimal&tagger-17-header=phase2&tagger-17-tag=%23affected%2Bnutrition%2Bstressed&tagger-18-header=phase3&tagger-18-tag=%23affected%2Bnutrition%2Bcrisis&tagger-19-header=phase4&tagger-19-tag=%23affected%2Bnutrition%2Bemergency&tagger-20-header=phase5&tagger-20-tag=%23affected%2Bnutrition%2Bfamine&tagger-21-header=phase35&tagger-21-tag=%23affected%2Bnutrition%2Btotal_at_risk&tagger-22-header=chtype&tagger-22-tag=%23status&tagger-23-header=exercise_code&tagger-23-tag=%23meta%2Bexercise_code&tagger-24-header=exercise_label&tagger-24-tag=%23meta%2Bexercise_months&tagger-25-header=exercise_year&tagger-25-tag=%23date%2Byear%2Bexercise&tagger-26-header=reference_code&tagger-26-tag=%23meta%2Breference_code&tagger-27-header=reference_label&tagger-27-tag=%23meta%2Breference_months&tagger-28-header=reference_year&tagger-28-tag=%23date%2Byear%2Breference&header-row=1&url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise&name=Cadre+harmonis%C3%A9+%28IPC%29+for+Mali+-+latest+year
+https://dev.proxy-hxlstandard-org.ahconu.org/data?name=Humanitarian+Response+Plans&url=http%3A%2F%2Fapi.hpc.tools%2Fv2%2Fpublic%2Fplan&tagger-match-all=on&tagger-02-header=name&tagger-02-tag=%23response%2Bname&tagger-03-header=code&tagger-03-tag=%23response%2Bcode&tagger-04-header=startdate&tagger-04-tag=%23date%2Bstart&tagger-05-header=enddate&tagger-05-tag=%23date%2Bend&tagger-09-header=years&tagger-09-tag=%23date%2Byear&tagger-10-header=locations&tagger-10-tag=%23country%2Bcode%2Blist&tagger-11-header=categories&tagger-11-tag=%23response%2Btype&tagger-12-header=revisedrequirements&tagger-12-tag=%23value%2Brequirements%2Brevised%2Bc_usd&tagger-13-header=origrequirements&tagger-13-tag=%23value%2Brequirements%2Borig%2Bc_usd&header-row=1&filter01=cut&cut-skip-untagged01=on&filter02=clean&clean-date-tags02=%23date&filter03=jsonpath&jsonpath-path03=%24[0].year&jsonpath-patterns03=date%2Byear&filter04=jsonpath&jsonpath-path04=%24[0].name&jsonpath-patterns04=%23response%2Btype&filter05=jsonpath&jsonpath-path05=%24[*].iso3&jsonpath-patterns05=%23country%2Bcode%2Blist&filter06=replace&replace-pattern06=[^A-Z%2C]*&replace-regex06=on&replace-tags06=%23country%2Bcode%2Blist&filter07=replace&replace-pattern07=^%2C%2B&replace-regex07=on&replace-tags07=%23country%2Bcode%2Blist&filter08=replace&replace-pattern08=%2C%2B%24&replace-regex08=on&replace-tags08=country%2Bcode%2Blist&filter09=replace&replace-pattern09=%2C%2B&replace-regex09=on&replace-value09=%2C+&replace-tags09=country%2Bcode%2Blist&filter10=sort&sort-tags10=date%2Bstart&sort-reverse10=on
+https://dev.proxy-hxlstandard-org.ahconu.org/data/validate?schema_url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D152581637&url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1ebcgNBh_v-IhW1hvrk6bkhIxVM3E6uVkHOBkx-bsbK8%2Fedit%23gid%3D0
+https://dev.proxy-hxlstandard-org.ahconu.org/pcodes/mli-adm1.csv
+https://dev.proxy-hxlstandard-org.ahconu.org/api/data-preview.json?url=https%3A%2F%2Fdata.humdata.org%2Fdataset%2Fcadre-harmonise