diff --git a/Dockerfile b/Dockerfile index 28eeec81b..9787ab90a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,112 @@ -FROM kobotoolbox/kobocat_base:latest +FROM nikolaik/python-nodejs:python3.8-nodejs10 -ENV KOBOCAT_SRC_DIR=/srv/src/kobocat \ +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV VIRTUAL_ENV=/opt/venv + + +ENV KOBOCAT_LOGS_DIR=/srv/logs \ + DJANGO_SETTINGS_MODULE=kobo.settings.prod \ + # The mountpoint of a volume shared with the `nginx` container. Static files will + # be copied there. + NGINX_STATIC_DIR=/srv/static \ + KOBOCAT_SRC_DIR=/srv/src/kobocat \ BACKUPS_DIR=/srv/backups \ - KOBOCAT_LOGS_DIR=/srv/logs - -# Install post-base-image `apt` additions from `apt_requirements.txt`, if modified. -COPY ./apt_requirements.txt "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt" -RUN if ! diff "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt" "${KOBOCAT_TMP_DIR}/base_apt_requirements.txt"; then \ - apt-get update && \ - apt-get install -y $(cat "${KOBOCAT_TMP_DIR}/current_apt_requirements.txt") && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - ; fi - -# Version 8 of pip doesn't really seem to upgrade packages when switching from -# PyPI to editable Git -RUN pip install --upgrade 'pip>=10,<11' - -# Install post-base-image `pip` additions/upgrades from `requirements/base.pip`, if modified. -COPY ./requirements/ "${KOBOCAT_TMP_DIR}/current_requirements/" -# FIXME: Replace this with the much simpler command `pip-sync ${KOBOCAT_TMP_DIR}/current_requirements/base.pip`. -RUN if ! diff "${KOBOCAT_TMP_DIR}/current_requirements/base.pip" "${KOBOCAT_TMP_DIR}/base_requirements/base.pip"; then \ - pip install --src "${PIP_EDITABLE_PACKAGES_DIR}/" -r "${KOBOCAT_TMP_DIR}/current_requirements/base.pip" \ - ; fi - -# Install post-base-image `pip` additions/upgrades from `requirements/s3.pip`, if modified. -RUN if ! diff "${KOBOCAT_TMP_DIR}/current_requirements/s3.pip" "${KOBOCAT_TMP_DIR}/base_requirements/s3.pip"; then \ - pip install --src "${PIP_EDITABLE_PACKAGES_DIR}/" -r "${KOBOCAT_TMP_DIR}/current_requirements/s3.pip" \ - ; fi - -# Uninstall `pip` packages installed in the base image from `requirements/uninstall.pip`, if present. -# FIXME: Replace this with the much simpler `pip-sync` command equivalent. -RUN if [ -e "${KOBOCAT_TMP_DIR}/current_requirements/uninstall.pip" ]; then \ - pip uninstall --yes -r "${KOBOCAT_TMP_DIR}/current_requirements/uninstall.pip" \ - ; fi - -# Wipe out the base image's `kobocat` dir (**including migration files**) and copy over this directory in its current state. -RUN rm -rf "${KOBOCAT_SRC_DIR}" + TMP_PATH=/srv/tmp \ + INIT_PATH=/srv/init + +# Install Dockerize. +ENV DOCKERIZE_VERSION v0.6.1 +RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -P /tmp \ + && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && rm /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + +########################################## +# Create build directories # +########################################## + +RUN mkdir -p "${NGINX_STATIC_DIR}" && \ + mkdir -p "${KOBOCAT_SRC_DIR}" && \ + mkdir -p "${TMP_PATH}" && \ + mkdir -p "${BACKUPS_DIR}" && \ + mkdir -p "${INIT_PATH}" + +########################################## +# Install `apt` packages. # +########################################## +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - + +RUN apt -qq update && \ + apt -qq -y install \ + gdal-bin \ + libproj-dev \ + gettext \ + postgresql-client \ + locales \ + runit-init \ + rsync \ + vim && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +########################### +# Install locales # +########################### + +RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen +RUN locale-gen && dpkg-reconfigure locales -f noninteractive + +########################### +# Copy KoBoCAT directory # +########################### + COPY . "${KOBOCAT_SRC_DIR}" +########################### +# Install `pip` packages. # +########################### + +RUN virtualenv "$VIRTUAL_ENV" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip install --quiet --upgrade pip && \ + pip install --quiet pip-tools +COPY ./dependencies/pip/prod.txt /srv/tmp/pip_dependencies.txt +RUN pip-sync /srv/tmp/pip_dependencies.txt 1>/dev/null && \ + rm -rf ~/.cache/pip + +########################################## +# Persist the log and email directories. # +########################################## + +RUN mkdir -p "${KOBOCAT_LOGS_DIR}/" "${KOBOCAT_SRC_DIR}/emails" && \ + chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \ + chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" + +################################################# +# Handle runtime tasks and create main process. # +################################################# + +# Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides. +RUN echo "export PATH=${PATH}" >> /etc/profile +RUN echo 'source /etc/profile' >> /root/.bashrc + # Prepare for execution. -# TODO: Remove the wrong port warning and related files, say, at the start of 2021 (see kobotoolbox/kobo-docker#301) RUN mkdir -p /etc/service/uwsgi_wrong_port_warning && \ cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" /etc/service/uwsgi_wrong_port_warning/run && \ mkdir -p /etc/service/uwsgi && \ + # Remove getty* services + rm -rf /etc/runit/runsvdir/default/getty-tty* && \ cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \ mkdir -p /etc/service/celery && \ ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \ mkdir -p /etc/service/celery_beat && \ ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \ - cp "${KOBOCAT_SRC_DIR}/docker/init.bash" /etc/my_init.d/10_init_kobocat.bash && \ - cp "${KOBOCAT_SRC_DIR}/docker/sync_static.sh" /etc/my_init.d/11_sync_static.bash && \ - mkdir -p "${KOBOCAT_SRC_DIR}/emails/" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \ - mkdir -p "${BACKUPS_DIR}" && \ - mkdir -p "${KOBOCAT_LOGS_DIR}" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" - -RUN echo "db:*:*:kobo:kobo" > /root/.pgpass && \ - chmod 600 /root/.pgpass - -# Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides. -RUN echo 'source /etc/profile' >> /root/.bashrc - WORKDIR "${KOBOCAT_SRC_DIR}" # TODO: Remove port 8000, say, at the start of 2021 (see kobotoolbox/kobo-docker#301 and wrong port warning above) EXPOSE 8001 8000 + +CMD ["/bin/bash", "-c", "exec ${KOBOCAT_SRC_DIR}/docker/init.bash"] diff --git a/Dockerfile.kobocat_base b/Dockerfile.kobocat_base deleted file mode 100644 index 5f85a779b..000000000 --- a/Dockerfile.kobocat_base +++ /dev/null @@ -1,33 +0,0 @@ -# Base image to take care of installing `apt` and `pip` requirements. - -FROM kobotoolbox/base-kobos:latest - - -ENV KOBOCAT_TMP_DIR=/srv/kobocat_tmp \ - # Store editable packages (pulled from VCS repos) in their own directory. - PIP_EDITABLE_PACKAGES_DIR=/srv/pip_editable_packages \ - UWSGI_USER=wsgi \ - UWSGI_GROUP=wsgi - - -########################### -# Install `apt` packages. # -########################### - -COPY ./apt_requirements.txt ${KOBOCAT_TMP_DIR}/base_apt_requirements.txt -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y $(cat ${KOBOCAT_TMP_DIR}/base_apt_requirements.txt) && \ - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - - -########################### -# Install `pip` packages. # -########################### - -COPY ./requirements/ ${KOBOCAT_TMP_DIR}/base_requirements/ -RUN mkdir -p ${PIP_EDITABLE_PACKAGES_DIR} && \ - pip install --upgrade 'pip>=10,<11' && \ - pip install --src ${PIP_EDITABLE_PACKAGES_DIR}/ -r ${KOBOCAT_TMP_DIR}/base_requirements/base.pip && \ - pip install --src ${PIP_EDITABLE_PACKAGES_DIR}/ -r ${KOBOCAT_TMP_DIR}/base_requirements/s3.pip && \ - rm -rf ~/.cache/pip diff --git a/dependencies/pip/dev.in b/dependencies/pip/dev.in new file mode 100644 index 000000000..8bf348510 --- /dev/null +++ b/dependencies/pip/dev.in @@ -0,0 +1,14 @@ +-r requirements.in + +ipdb +ipython +shell_command +Werkzeug +sqlparse +pytest +pytest-django +pytest-env +mongomock +mock +httmock +django-nose diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt new file mode 100644 index 000000000..bd146cbe4 --- /dev/null +++ b/dependencies/pip/dev.txt @@ -0,0 +1,136 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile dependencies/pip/dev.in +# +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in +amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu +argparse==1.4.0 # via unittest2 +attrs==19.3.0 # via jsonschema, pytest +backcall==0.2.0 # via ipython +backports.csv==1.0.7 # via formpack +begins==0.9 # via formpack +billiard==3.6.3.0 # via celery +celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +cssselect==1.1.0 # via pyquery +decorator==4.4.2 # via ipython, traitlets +defusedxml==0.6.0 # via djangorestframework-xml +dict2xml==1.7.0 # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 # via -r dependencies/pip/requirements.in +django-filter==2.3.0 # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian +django-nose==1.4.6 # via -r dependencies/pip/requirements.in +django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 # via -r dependencies/pip/requirements.in +django-render-block==0.7 # via django-templated-email +django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian +docutils==0.16 # via statistics +dpath==2.0.1 # via -r dependencies/pip/requirements.in +ecdsa==0.15 # via tlslite-ng +elaphe3==0.2.0 # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 # via openpyxl +formencode==1.3.1 # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in +future==0.18.2 # via celery +gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 # via formpack +gitdb==4.0.5 # via gitpython +gitpython==3.1.7 # via transifex-client +httmock==1.3.0 # via -r dependencies/pip/dev.in +idna==2.10 # via requests +importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path, pluggy, pytest +ipdb==0.13.3 # via -r dependencies/pip/dev.in +ipython-genutils==0.2.0 # via traitlets +ipython==7.16.1 # via -r dependencies/pip/dev.in, ipdb +jdcal==1.4.1 # via openpyxl +jedi==0.17.2 # via ipython +jsonfield==3.1.0 # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 # via formpack +kombu==4.6.11 # via celery +linecache2==1.0.0 # via traceback2 +lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery +markdown==3.2.2 # via -r dependencies/pip/requirements.in +mock==4.0.2 # via -r dependencies/pip/dev.in +modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in +mongomock==3.19.0 # via -r dependencies/pip/dev.in +more-itertools==8.4.0 # via pytest +nose==1.3.7 # via django-nose +numpy==1.19.0 # via pandas +oauthlib==3.1.0 # via django-oauth-toolkit +openpyxl==3.0.4 # via -r dependencies/pip/requirements.in +packaging==20.4 # via pytest +pandas==1.0.5 # via -r dependencies/pip/requirements.in +parso==0.7.0 # via jedi +path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path==13.1.0 # via path.py +pexpect==4.8.0 # via ipython +pickleshare==0.7.5 # via ipython +pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 +pluggy==0.13.1 # via pytest +prompt-toolkit==3.0.5 # via ipython +psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in +ptyprocess==0.6.0 # via pexpect +py==1.9.0 # via pytest +pygments==2.6.1 # via ipython +pymongo==3.10.1 # via -r dependencies/pip/requirements.in +pyparsing==2.4.7 # via packaging +pyquery==1.4.1 # via formpack +pyrsistent==0.16.0 # via jsonschema +pytest-django==3.9.0 # via -r dependencies/pip/dev.in +pytest-env==0.6.2 # via -r dependencies/pip/dev.in +pytest==5.4.3 # via -r dependencies/pip/dev.in, pytest-django, pytest-env +python-crontab==2.5.1 # via django-celery-beat +python-dateutil==2.8.1 # via pandas, python-crontab +python-slugify==4.0.1 # via transifex-client +pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas +pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack +raven==6.10.0 # via -r dependencies/pip/requirements.in +redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions +requests==2.24.0 # via django-oauth-toolkit, httmock, transifex-client +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in +sentinels==1.0.0 # via mongomock +shell-command==0.1 # via -r dependencies/pip/dev.in +simplejson==3.17.2 # via -r dependencies/pip/requirements.in +six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, mongomock, packaging, pyrsistent, python-dateutil, ssrf-protect, traitlets, transifex-client, unittest2 +smmap==3.0.4 # via gitdb +sqlparse==0.3.1 # via -r dependencies/pip/dev.in, django +statistics==1.0.3.5 # via formpack +text-unidecode==1.3 # via python-slugify +tlslite-ng==0.7.5 # via gdata-python3 +traceback2==1.4.0 # via unittest2 +traitlets==4.3.3 # via ipython +transifex-client==0.13.11 # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 # via djangorestframework-csv, pyxform +unittest2==1.1.0 # via pyxform +urllib3==1.25.9 # via requests, transifex-client +vine==1.3.0 # via amqp, celery +wcwidth==0.2.5 # via prompt-toolkit, pytest +werkzeug==1.0.1 # via -r dependencies/pip/dev.in +xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform +xlsxwriter==1.2.9 # via formpack +xlwt==1.3.0 # via -r dependencies/pip/requirements.in +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/dependencies/pip/prod.in b/dependencies/pip/prod.in new file mode 100644 index 000000000..bd6f83b94 --- /dev/null +++ b/dependencies/pip/prod.in @@ -0,0 +1,3 @@ +-r requirements.in + +uWSGI diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt new file mode 100644 index 000000000..f58b461bd --- /dev/null +++ b/dependencies/pip/prod.txt @@ -0,0 +1,106 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile dependencies/pip/prod.in +# +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in +amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu +argparse==1.4.0 # via unittest2 +attrs==19.3.0 # via jsonschema +backports.csv==1.0.7 # via formpack +begins==0.9 # via formpack +billiard==3.6.3.0 # via celery +celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +cssselect==1.1.0 # via pyquery +defusedxml==0.6.0 # via djangorestframework-xml +dict2xml==1.7.0 # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 # via -r dependencies/pip/requirements.in +django-filter==2.3.0 # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian +django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 # via -r dependencies/pip/requirements.in +django-render-block==0.7 # via django-templated-email +django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian +docutils==0.16 # via statistics +dpath==2.0.1 # via -r dependencies/pip/requirements.in +ecdsa==0.15 # via tlslite-ng +elaphe3==0.2.0 # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 # via openpyxl +formencode==1.3.1 # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in +future==0.18.2 # via celery +gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 # via formpack +gitdb==4.0.5 # via gitpython +gitpython==3.1.7 # via transifex-client +idna==2.10 # via requests +importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path +jdcal==1.4.1 # via openpyxl +jsonfield==3.1.0 # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 # via formpack +kombu==4.6.11 # via celery +linecache2==1.0.0 # via traceback2 +lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery +markdown==3.2.2 # via -r dependencies/pip/requirements.in +modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in +numpy==1.19.0 # via pandas +oauthlib==3.1.0 # via django-oauth-toolkit +openpyxl==3.0.4 # via -r dependencies/pip/requirements.in +pandas==1.0.5 # via -r dependencies/pip/requirements.in +path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path==13.1.0 # via path.py +pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 +psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in +pymongo==3.10.1 # via -r dependencies/pip/requirements.in +pyquery==1.4.1 # via formpack +pyrsistent==0.16.0 # via jsonschema +python-crontab==2.5.1 # via django-celery-beat +python-dateutil==2.8.1 # via pandas, python-crontab +python-slugify==4.0.1 # via transifex-client +pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas +pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack +raven==6.10.0 # via -r dependencies/pip/requirements.in +redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions +requests==2.24.0 # via django-oauth-toolkit, transifex-client +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in +simplejson==3.17.2 # via -r dependencies/pip/requirements.in +six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2 +smmap==3.0.4 # via gitdb +sqlparse==0.3.1 # via django +statistics==1.0.3.5 # via formpack +text-unidecode==1.3 # via python-slugify +tlslite-ng==0.7.5 # via gdata-python3 +traceback2==1.4.0 # via unittest2 +transifex-client==0.13.11 # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 # via djangorestframework-csv, pyxform +unittest2==1.1.0 # via pyxform +urllib3==1.25.9 # via requests, transifex-client +vine==1.3.0 # via amqp, celery +xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform +xlsxwriter==1.2.9 # via formpack +xlwt==1.3.0 # via -r dependencies/pip/requirements.in +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/py3.pip b/dependencies/pip/requirements.in similarity index 66% rename from requirements/py3.pip rename to dependencies/pip/requirements.in index 811e836e9..6cdb391d5 100644 --- a/requirements/py3.pip +++ b/dependencies/pip/requirements.in @@ -1,14 +1,39 @@ +# File for use with `pip-compile`; see https://github.com/nvie/pip-tools +# https://github.com/bndr/pipreqs is a handy utility, too. + +# More up-to-date version of django-digest than PyPI seems to have. +# Also, python-digest is an unlisted dependency thereof. +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest + +# formpack exports +# ToDo Remove when `kobokitten-remove-ui-CUD-actions-unicode is merged. +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack + +# captcha +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client + +# spss +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter + +# ssrf +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect + +# jnm's django storages +# Necessary to use this fork until the resolution of +# https://github.com/jschneier/django-storages/issues/566 +-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages + +# Regular PyPI packages pytz -Django>=1.11,<2 +Django>=2,<3 dj-database-url -# django-guardian django-registration-redux django-templated-email +dpath gdata-python3 -httplib2 modilabs-python-utils Pillow -# poster psycopg2-binary pymongo lxml @@ -17,27 +42,14 @@ django-reversion<3.0.2 xlrd xlwt openpyxl - celery>=4.0,<5.0 celery[redis] -# django-celery-beat amqp -vine==1.3.0 # Freeze version. v5.0.0 does not work -django-nose raven - -# More up-to-date version of django-digest than PyPI seems to have. -# Also, python-digest is an unlisted dependency thereof. --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest - # new export code relies on pandas>=0.12.0 elaphe3 -# request -# formpack exports -git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack django-pure-pagination # sms support @@ -50,28 +62,15 @@ djangorestframework-jsonp djangorestframework-xml # cors -#django-cors-headers +django-cors-headers Markdown -#django-filter - -# captcha -# recaptcha-client==1.0.6 --e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client - -# unicodecsv==0.9.4 -dpath +django-filter # tagging django-taggit -# oauth2 support -# django-oauth-toolkit - -# spss -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter - # JSON data type support -# jsonfield<1.0 +jsonfield django-db-readonly transifex-client @@ -83,20 +82,15 @@ django-extensions django-redis-sessions redis -# ssrf --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect - - -# Compatibility Django 1.11 -django-guardian==1.5.1 -django-timezone-field==3.0 -django-celery-beat==1.6.0 -django-oauth-toolkit==1.1.1 -jsonfield==2.1.1 -django-render-block==0.6 -django-cors-headers==3.0.0 -django-filter==2.2.0 +# Compatibility Django 2.2 +django-guardian +django-timezone-field +django-celery-beat +django-oauth-toolkit # Added packages simplejson djangorestframework-guardian + +# Django Storage AWS +boto3 diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt new file mode 100644 index 000000000..09d8e92dc --- /dev/null +++ b/dependencies/pip/requirements.txt @@ -0,0 +1,106 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile dependencies/pip/requirements.in +# +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in +amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu +argparse==1.4.0 # via unittest2 +attrs==19.3.0 # via jsonschema +backports.csv==1.0.7 # via formpack +begins==0.9 # via formpack +billiard==3.6.3.0 # via celery +celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +cssselect==1.1.0 # via pyquery +defusedxml==0.6.0 # via djangorestframework-xml +dict2xml==1.7.0 # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 # via -r dependencies/pip/requirements.in +django-filter==2.3.0 # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian +django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 # via -r dependencies/pip/requirements.in +django-render-block==0.7 # via django-templated-email +django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian +docutils==0.16 # via statistics +dpath==2.0.1 # via -r dependencies/pip/requirements.in +ecdsa==0.15 # via tlslite-ng +elaphe3==0.2.0 # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 # via openpyxl +formencode==1.3.1 # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in +future==0.18.2 # via celery +gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 # via formpack +gitdb==4.0.5 # via gitpython +gitpython==3.1.7 # via transifex-client +idna==2.10 # via requests +importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path +jdcal==1.4.1 # via openpyxl +jsonfield==3.1.0 # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 # via formpack +kombu==4.6.11 # via celery +linecache2==1.0.0 # via traceback2 +lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery +markdown==3.2.2 # via -r dependencies/pip/requirements.in +modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in +numpy==1.19.0 # via pandas +oauthlib==3.1.0 # via django-oauth-toolkit +openpyxl==3.0.4 # via -r dependencies/pip/requirements.in +pandas==1.0.5 # via -r dependencies/pip/requirements.in +path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path==13.1.0 # via path.py +pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 +psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in +pymongo==3.10.1 # via -r dependencies/pip/requirements.in +pyquery==1.4.1 # via formpack +pyrsistent==0.16.0 # via jsonschema +python-crontab==2.5.1 # via django-celery-beat +python-dateutil==2.8.1 # via pandas, python-crontab +python-slugify==4.0.1 # via transifex-client +pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas +pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack +raven==6.10.0 # via -r dependencies/pip/requirements.in +redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions +requests==2.24.0 # via django-oauth-toolkit, transifex-client +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in +simplejson==3.17.2 # via -r dependencies/pip/requirements.in +six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2 +smmap==3.0.4 # via gitdb +sqlparse==0.3.1 # via django +statistics==1.0.3.5 # via formpack +text-unidecode==1.3 # via python-slugify +tlslite-ng==0.7.5 # via gdata-python3 +traceback2==1.4.0 # via unittest2 +transifex-client==0.13.11 # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 # via djangorestframework-csv, pyxform +unittest2==1.1.0 # via pyxform +urllib3==1.25.9 # via requests, transifex-client +vine==1.3.0 # via amqp, celery +xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform +xlsxwriter==1.2.9 # via formpack +xlwt==1.3.0 # via -r dependencies/pip/requirements.in +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/docker/init.bash b/docker/init.bash index ff934fac2..48e903295 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -5,16 +5,17 @@ source /etc/profile echo 'KoBoCAT intializing...' -oldpwd=$(pwd) cd "${KOBOCAT_SRC_DIR}" -echo 'Synchronizing database.' -python manage.py syncdb --noinput +if [[ -z $DATABASE_URL ]]; then + echo "DATABASE_URL must be configured to run this server" + echo "example: 'DATABASE_URL=postgres://hostname:5432/dbname'" + exit 1 +fi -echo 'Running migrations.' +echo 'Running migrations...' python manage.py migrate --noinput - rm -f /etc/cron.d/clean_up_tmp cp docker/cron/clean_up_tmp /etc/cron.d/ echo 'KoBoCat tmp clean-up cron installed' @@ -28,12 +29,18 @@ else echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" fi +/bin/bash ${KOBOCAT_SRC_DIR}/docker/sync_static.sh + +# Keep it as is, not tested with Python3 rm -rf /etc/profile.d/pydev_debugger.bash.sh -if [[ -d /srv/pydev_orig && ! -z "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then +if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then echo 'Enabling PyDev remote debugging.' "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash" fi +echo 'Cleaning up Celery PIDs...' +rm -rf /tmp/celery*.pid + echo 'KoBoCAT initialization complete.' -cd $oldpwd +exec /usr/bin/runsvdir /etc/service diff --git a/docker/run_tests.bash b/docker/run_tests.bash deleted file mode 100755 index 3fcce8c02..000000000 --- a/docker/run_tests.bash +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -e -source /etc/profile - -echo "print('Django shell loaded successfully.')" | python manage.py shell diff --git a/docker/run_uwsgi.bash b/docker/run_uwsgi.bash index 3ed7ee7ee..b3ac46c51 100755 --- a/docker/run_uwsgi.bash +++ b/docker/run_uwsgi.bash @@ -10,13 +10,32 @@ if [[ "$(stat -c '%U' ${KOBOCAT_LOGS_DIR})" != "${UWSGI_USER}" ]]; then fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" -uwsgi_command="/sbin/setuser ${UWSGI_USER} /usr/local/bin/uwsgi --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" +uwsgi_command="/sbin/setuser ${UWSGI_USER} $(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" + if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then + cd "${KOBOCAT_SRC_DIR}" + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt") + if [[ -n "$DIFF" ]]; then + echo "Syncing pip dependencies..." + pip-sync dependencies/pip/prod.txt 1>/dev/null + cp "dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt" + fi echo 'Running `kobocat` container with uWSGI application server.' exec ${uwsgi_command} else - echo 'Running `kobocat` container with `runserver_plus` debugging application server.' cd "${KOBOCAT_SRC_DIR}" - pip install werkzeug==0.16.0 ipython + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt") + if [[ -n "$DIFF" ]]; then + echo "Syncing pip dependencies..." + pip-sync dependencies/pip/dev.txt 1>/dev/null + cp "dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt" + fi + + if [[ -n "$RAVEN_DSN" ]]; then + echo "Sentry detected. Installing \`raven\` pip dependency..." + pip install raven + fi + + echo 'Running `kobocat` container with `runserver_plus` debugging application server.' exec python manage.py runserver_plus 0:8001 fi diff --git a/docker/sync_static.sh b/docker/sync_static.sh index efb1b9ebf..c752e0e8d 100755 --- a/docker/sync_static.sh +++ b/docker/sync_static.sh @@ -3,9 +3,6 @@ set -e source /etc/profile -oldpwd=$(pwd) -cd "${KOBOCAT_SRC_DIR}" - mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" echo "Collecting static files..." @@ -26,7 +23,5 @@ echo "% chown -R \"${UWSGI_USER}\" \"${KOBOCAT_SRC_DIR}\"" echo '%%%%%%%%%%%%%%%%%%%%%%' echo "Syncing to nginx folder..." -rsync -aq ${KOBOCAT_SRC_DIR}/onadata/static/* /srv/static/ +rsync -aq --delete --chown=www-data "${KPI_SRC_DIR}/onadata/static/" "${NGINX_STATIC_DIR}/" echo "Done" - -cd $oldpwd diff --git a/kobocat-template/templates/base.html b/kobocat-template/templates/base.html index a5d410982..2869e5d7d 100644 --- a/kobocat-template/templates/base.html +++ b/kobocat-template/templates/base.html @@ -37,8 +37,8 @@ + + @@ -106,7 +106,7 @@ - + diff --git a/kobocat-template/templates/legacy_survey.html b/kobocat-template/templates/legacy_survey.html deleted file mode 100644 index ae99232c0..000000000 --- a/kobocat-template/templates/legacy_survey.html +++ /dev/null @@ -1,295 +0,0 @@ -{% load i18n %} -{% load static %} - -{% with legacy_survey_url="https://ee.kobotoolbox.org/single/XyEzJRoc" %} - - - - - -
-
- - -
-

- - - - - icon - - - - - - - - - - {% trans "The legacy interface will soon disappear." %} - {% trans "Please let us know how and why you are still using it." %} -

- -

{% trans "We are planning to remove some or all of it’s the components from this legacy user interface for viewing project data" %} {% trans "(also known as kobocat). We introduced the new user interface in 2016 which eventually included access to all features that were previously only available in the legacy interface (and many more features). Maintaining the legacy interface functionality and security involves a lot of work and costs for our donors. Today, only a very small number of users still access the legacy user interface — including you." %}

- -

{% trans "We announced a while ago on this page that deploying projects on the legacy interface was highly discouraged and that its user interface will be removed “shortly”. We care deeply about our users, so before we disable the legacy interface completely, we ask you to let us know why you are still using the old user interface." %} {% trans "This should not take more than a minute." %}

- - {% trans "Take survey now" %} -
-
-
- -
-
- {% trans "Take survey now" %} - -

- - - - - icon - - - - - - - - - - {% trans "The legacy interface will soon disappear." %} - {% trans "Please let us know how and why you are still using it." %} -

- -

{% trans "We are planning to remove some or all of it’s the components from this legacy user interface for viewing project data (also known as kobocat). We care deeply about our users, so before we disable the legacy interface completely, we ask you to let us know why you are still using the old user interface." %} {% trans "This should not take more than a minute." %}

-
-
- -{% endwith %} \ No newline at end of file diff --git a/onadata/apps/api/tests/viewsets/test_briefcase_api.py b/onadata/apps/api/tests/viewsets/test_briefcase_api.py index d25a4ff4c..4433d9415 100644 --- a/onadata/apps/api/tests/viewsets/test_briefcase_api.py +++ b/onadata/apps/api/tests/viewsets/test_briefcase_api.py @@ -1,7 +1,7 @@ # coding: utf-8 import os -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.files.storage import get_storage_class from django.utils.encoding import smart_str from django_digest.test import DigestAuth diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_api.py b/onadata/apps/api/tests/viewsets/test_xform_list_api.py index 1f50c06af..733caac60 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_api.py @@ -13,7 +13,7 @@ CAN_ADD_SUBMISSIONS, CAN_VIEW_XFORM ) -from onadata.libs.tests.utils.xml import pyxform_version_agnostic +from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml class TestXFormListApi(TestAbstractViewSet): @@ -222,7 +222,7 @@ def test_get_xform_list_with_formid_parameter(self): with open(path) as f: form_list_xml = f.read().strip() data = {"hash": self.xform.hash, "pk": self.xform.pk} - content = response.render().content + content = response.render().content.decode() self.assertEqual(content, form_list_xml % data) def test_retrieve_xform_xml(self): @@ -252,8 +252,9 @@ def test_retrieve_xform_xml(self): form_xml = f.read().strip() data = {"form_uuid": self.xform.uuid} content = smart_str(response.render().content).strip() - self.assertEqual(pyxform_version_agnostic(content), - pyxform_version_agnostic(form_xml % data)) + + is_equal_xml(pyxform_version_agnostic(content), + pyxform_version_agnostic(form_xml % data)) def _load_metadata(self, xform=None): data_value = "screenshot.png" diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 7757e4968..7fd5e4eb9 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -15,7 +15,7 @@ CAN_VIEW_XFORM ) from onadata.libs.serializers.xform_serializer import XFormSerializer -from onadata.libs.tests.utils.xml import pyxform_version_agnostic +from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml class TestXFormViewSet(TestAbstractViewSet): @@ -170,8 +170,8 @@ def test_form_format(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + is_equal_xml(pyxform_version_agnostic(response_doc.toxml()), + pyxform_version_agnostic(expected_doc.toxml())) def test_form_tags(self): self.publish_xls_form() diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 7f7ed39a8..f5e1001ec 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -188,28 +188,27 @@ def get_media_file_response(metadata): return HttpResponseRedirect(metadata.data_value) -def get_view_name(view_cls, suffix=None): - """ - Override Django REST framework's name for the base API class +def get_view_name(view_obj): + """ + Override Django REST framework's name for the base API class """ # The base API class should inherit directly from APIView. We can't use # issubclass() because ViewSets also inherit (indirectly) from APIView. try: - if inspect.getmro(view_cls)[1] is rest_framework_views.APIView: - return 'KoBo Api' # awkward capitalization for consistency + if inspect.getmro(view_obj.__class__)[1] is rest_framework_views.APIView: + return 'KoBo Api' # awkward capitalization for consistency except KeyError: pass - return rest_framework_views.get_view_name(view_cls, suffix) + return rest_framework_views.get_view_name(view_obj) -def get_view_description(view_cls, html=False): +def get_view_description(view_obj, html=False): """ Replace example.com in Django REST framework's default API description with the domain name of the current site """ domain = Site.objects.get_current().domain - description = rest_framework_views.get_view_description(view_cls, - html) + description = rest_framework_views.get_view_description(view_obj, html) # description might not be a plain string: e.g. it could be a SafeText # to prevent further HTML escaping original_type = type(description) diff --git a/onadata/apps/api/urls.py b/onadata/apps/api/urls.py index 0f2f2409e..d85aad518 100644 --- a/onadata/apps/api/urls.py +++ b/onadata/apps/api/urls.py @@ -1,20 +1,20 @@ # coding: utf-8 -from django.conf.urls import url +from django.urls import re_path from rest_framework import routers from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.urlpatterns import format_suffix_patterns from rest_framework.views import APIView +from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet +from onadata.apps.api.viewsets.briefcase_api import BriefcaseApi from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.api.viewsets.metadata_viewset import MetaDataViewSet from onadata.apps.api.viewsets.note_viewset import NoteViewSet -from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.api.viewsets.xform_list_api import XFormListApi from onadata.apps.api.viewsets.xform_submission_api import XFormSubmissionApi -from onadata.apps.api.viewsets.briefcase_api import BriefcaseApi +from onadata.apps.api.viewsets.xform_viewset import XFormViewSet class MultiLookupRouter(routers.DefaultRouter): @@ -89,42 +89,34 @@ def get_lookup_regexes(self, viewset): def get_lookup_routes(self, viewset): ret = [self.routes[0]] - # Determine any `@action` or `@link` decorated methods on the viewset - dynamic_routes = [] - for methodname in dir(viewset): - attr = getattr(viewset, methodname) - httpmethods = getattr(attr, 'bind_to_methods', None) - if httpmethods: - httpmethods = [method.lower() for method in httpmethods] - dynamic_routes.append((httpmethods, methodname)) - for route in self.lookups_routes: if route.mapping == {'{httpmethod}': '{methodname}'}: - # Dynamic routes (@link or @action decorator) - for httpmethods, methodname in dynamic_routes: + for extra_action in viewset.get_extra_actions(): + # FIXME support `url_path` and `url_name` of `@action` decorator + methodname = extra_action.__name__ + mapping = extra_action.mapping + detail = extra_action.detail initkwargs = route.initkwargs.copy() - initkwargs.update(getattr(viewset, methodname).kwargs) - mapping = dict( - (httpmethod, methodname) for httpmethod in httpmethods) - name = routers.replace_methodname(route.name, methodname) + initkwargs.update(extra_action.kwargs) + name = self.replace_methodname(route.name, methodname) if 'extra_lookup_fields' in initkwargs: uri = route.url[1] - uri = routers.replace_methodname(uri, methodname) - - # FIXME routers.Route() expects `detail` parameter + uri = self.replace_methodname(uri, methodname) ret.append(routers.Route( url=uri, mapping=mapping, - name='%s-extra' % name, + name=f'{name}-extra', initkwargs=initkwargs, + detail=detail )) - uri = routers.replace_methodname(route.url[0], methodname) - # FIXME routers.Route() expects `detail` parameter + + uri = self.replace_methodname(route.url[0], methodname) ret.append(routers.Route( url=uri, mapping=mapping, name=name, initkwargs=initkwargs, + detail=detail )) else: # Standard route @@ -321,8 +313,8 @@ def get_urls(self): ret = [] if self.include_root_view: - root_url = url(r'^$', self.get_api_root_view(), - name=self.root_view_name) + root_url = re_path(r'^$', self.get_api_root_view(), + name=self.root_view_name) ret.append(root_url) for prefix, viewset, basename in self.registry: lookup = self.get_lookup_regex(viewset) @@ -347,11 +339,29 @@ def get_urls(self): ) view = viewset.as_view(mapping, **route.initkwargs) name = route.name.format(basename=basename) - ret.append(url(regex, view, name=name)) + ret.append(re_path(regex, view, name=name)) if self.include_format_suffixes: ret = format_suffix_patterns(ret, allowed=['[a-z0-9]+']) return ret + @staticmethod + def replace_methodname(format_string, methodname): + """ + Taken from old version of DRF for retro-compatibility. + (AFAIK, this method has been dropped in DRF 3.8) + + Partially format a format_string, swapping out any + '{methodname}' or '{methodnamehyphen}' components. + + @ToDo If DRF got rid of it, we should too? Let's find a better way to + achieve the same goal. + """ + methodnamehyphen = methodname.replace('_', '-') + ret = format_string + ret = ret.replace('{methodname}', methodname) + ret = ret.replace('{methodnamehyphen}', methodnamehyphen) + return ret + class MultiLookupRouterWithPatchList(MultiLookupRouter): """ @@ -385,3 +395,6 @@ def make_routes(template_text): router_with_patch_list = MultiLookupRouterWithPatchList(trailing_slash=False) router_with_patch_list.register(r'data', DataViewSet, basename='data') + +#router_api_v1 = ExtendedDefaultRouter() +#data_routes = router_api_v1.register(r'data', DataViewSet, basename='data') diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index 443c10825..2e54f0d20 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -7,7 +7,7 @@ from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin from onadata.libs.serializers.user_profile_serializer import ( UserProfileWithTokenSerializer) -from onadata.settings.common import DEFAULT_SESSION_EXPIRY_TIME +from onadata.settings.base import DEFAULT_SESSION_EXPIRY_TIME class ConnectViewSet(ObjectLookupMixin, viewsets.GenericViewSet): diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index f9e804383..ad19eb9b5 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -400,12 +400,7 @@ def bulk_delete(self, request, *args, **kwargs): postgres_query, mongo_query = self.__build_db_queries(xform, payload) # Delete Postgres & Mongo - updated_records_count = Instance.objects.filter(**postgres_query).count() - - # Since Django 1.9, `.delete()` returns an dict with number of rows - # deleted per object. - # FixMe remove `.count()` query and use that dict instance - Instance.objects.filter(**postgres_query).delete() + updated_records_count = Instance.objects.filter(**postgres_query).delete() ParsedInstance.bulk_delete(mongo_query) return Response({ 'detail': _('{} submissions have been deleted').format( diff --git a/onadata/apps/api/viewsets/note_viewset.py b/onadata/apps/api/viewsets/note_viewset.py index b46f7955b..d9d4713cd 100644 --- a/onadata/apps/api/viewsets/note_viewset.py +++ b/onadata/apps/api/viewsets/note_viewset.py @@ -65,11 +65,9 @@ def get_queryset(self): CAN_VIEW_XFORM, XForm, accept_global_perms=False) - viewable_notes = Note.objects.filter( - Q(instance__xform=viewable_xforms) | Q(instance__xform__shared_data=True) + Q(instance__xform__in=viewable_xforms) | Q(instance__xform__shared_data=True) ) - return viewable_notes # This used to be post_save. Part of it is here, permissions validation diff --git a/onadata/apps/api/viewsets/xform_list_api.py b/onadata/apps/api/viewsets/xform_list_api.py index 17984939d..7a60c390f 100644 --- a/onadata/apps/api/viewsets/xform_list_api.py +++ b/onadata/apps/api/viewsets/xform_list_api.py @@ -85,7 +85,7 @@ def filter_queryset(self, queryset): if profile.require_auth: # The specified has user ticked "Require authentication to see # forms and submit data"; reject anonymous requests - if self.request.user.is_anonymous(): + if self.request.user.is_anonymous: # raises a permission denied exception, forces # authentication self.permission_denied(self.request) diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 3f08bec53..f4cdee4d7 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -24,7 +24,7 @@ def handle(self, *args, **kwargs): "Expected %s as permission") % ' or '.join(permissions)) try: - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() + s3 = get_storage_class('storages.backends.s3boto3.S3Boto3Storage')() except: print(_("Missing necessary libraries. Try running: pip install " "-r requirements-s3.pip")) diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index 53fab96a5..83b4c2edc 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -22,7 +22,7 @@ def handle(self, *args, **kwargs): try: fs = get_storage_class( 'django.core.files.storage.FileSystemStorage')() - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() + s3 = get_storage_class('storages.backends.s3boto3.S3Boto3Storage')() except: print(_("Missing necessary libraries. Try running: pip install -r" "requirements/s3.pip")) diff --git a/onadata/apps/logger/migrations/0001_initial.py b/onadata/apps/logger/migrations/0001_initial.py index 3c70f9cc6..94acea2a8 100644 --- a/onadata/apps/logger/migrations/0001_initial.py +++ b/onadata/apps/logger/migrations/0001_initial.py @@ -22,7 +22,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('media_file', models.FileField(upload_to=onadata.apps.logger.models.attachment.upload_to)), - ('mimetype', models.CharField(default=b'', max_length=50, blank=True)), + ('mimetype', models.CharField(default='', max_length=50, blank=True)), ], ), migrations.CreateModel( @@ -84,7 +84,7 @@ class Migration(migrations.Migration): ('downloadable', models.BooleanField(default=True)), ('allows_sms', models.BooleanField(default=False)), ('encrypted', models.BooleanField(default=False)), - ('sms_id_string', models.SlugField(default=b'', verbose_name='SMS ID', max_length=100, editable=False)), + ('sms_id_string', models.SlugField(default='', verbose_name='SMS ID', max_length=100, editable=False)), ('id_string', models.SlugField(verbose_name='ID', max_length=100, editable=False)), ('title', models.CharField(max_length=255, editable=False)), ('date_created', models.DateTimeField(auto_now_add=True)), diff --git a/onadata/apps/logger/migrations/0004_increase-length-of-attachment-mimetype-field.py b/onadata/apps/logger/migrations/0004_increase-length-of-attachment-mimetype-field.py index fc05dffb1..479551a48 100644 --- a/onadata/apps/logger/migrations/0004_increase-length-of-attachment-mimetype-field.py +++ b/onadata/apps/logger/migrations/0004_increase-length-of-attachment-mimetype-field.py @@ -12,6 +12,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='attachment', name='mimetype', - field=models.CharField(default=b'', max_length=100, blank=True), + field=models.CharField(default='', max_length=100, blank=True), ), ] diff --git a/onadata/apps/logger/migrations/0007_add_validate_permission_on_xform.py b/onadata/apps/logger/migrations/0007_add_validate_permission_on_xform.py index e863ac6a4..6e9e17d42 100644 --- a/onadata/apps/logger/migrations/0007_add_validate_permission_on_xform.py +++ b/onadata/apps/logger/migrations/0007_add_validate_permission_on_xform.py @@ -11,6 +11,6 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( name='xform', - options={'ordering': ('id_string',), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms', 'permissions': (('view_xform', 'Can view associated data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('validate_xform', 'Can validate submissions.'))}, + options={'ordering': ('id_string',), 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms', 'permissions': (('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership.'), ('validate_xform', 'Can validate submissions.'))}, ), ] diff --git a/onadata/apps/logger/migrations/0014_attachment_add_media_file_size.py b/onadata/apps/logger/migrations/0014_attachment_add_media_file_size.py index d3220ee8f..5de4e887f 100644 --- a/onadata/apps/logger/migrations/0014_attachment_add_media_file_size.py +++ b/onadata/apps/logger/migrations/0014_attachment_add_media_file_size.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - +# coding: utf-8 from django.db import migrations, models diff --git a/onadata/apps/logger/migrations/0015_add_delete_data_permission.py b/onadata/apps/logger/migrations/0015_add_delete_data_permission.py index da6c73b85..7bbc56f0b 100644 --- a/onadata/apps/logger/migrations/0015_add_delete_data_permission.py +++ b/onadata/apps/logger/migrations/0015_add_delete_data_permission.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - +# coding: utf-8 import sys from django.db import migrations @@ -135,6 +133,15 @@ def reverse_func(apps, schema_editor): class Migration(migrations.Migration): + """ + This migration has changed between KoBoCAT 1.0 and KoBoCAT 2.0. + Permissions on Note model are altered in this migration in this branch. + With Django 2.2 "view" permission is included by default. + So `view_xform` and `view_notes` must be removed here because they were + added by previous migrations in Django 1.8. + It avoids an IntegrityError when Django tries to add `view_xform`|`view_notes` + twice. + """ dependencies = [ ('logger', '0014_attachment_add_media_file_size'), @@ -148,7 +155,6 @@ class Migration(migrations.Migration): 'verbose_name': 'XForm', 'verbose_name_plural': 'XForms', 'permissions': ( - ('view_xform', 'Can view associated data'), ('report_xform', 'Can make submissions to the form'), ('move_xform', 'Can move form between projects'), ('transfer_xform', 'Can transfer form ownership'), @@ -157,5 +163,9 @@ class Migration(migrations.Migration): ), }, ), + migrations.AlterModelOptions( + name='note', + options={'permissions': ()}, + ), migrations.RunPython(forwards_func, reverse_func), ] diff --git a/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py b/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py new file mode 100644 index 000000000..d6a0f80ce --- /dev/null +++ b/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py @@ -0,0 +1,35 @@ +# coding: utf-8 +from django.db import migrations, models + + +class Migration(migrations.Migration): + """ + Re-apply parts of `0015_add_delete_data_permission.py` for upgrade + of existing installations. + See `0015_add_delete_data_permission.py` for detailsf + """ + + dependencies = [ + ('logger', '0015_add_delete_data_permission'), + ] + + operations = [ + migrations.AlterModelOptions( + name='xform', + options={ + 'ordering': ('id_string',), + 'verbose_name': 'XForm', + 'verbose_name_plural': 'XForms', + 'permissions': ( + ('report_xform', 'Can make submissions to the form'), + ('transfer_xform', 'Can transfer form ownership.'), + ('validate_xform', 'Can validate submissions.'), + ('delete_data_xform', 'Can delete submissions'), + ), + }, + ), + migrations.AlterModelOptions( + name='note', + options={'permissions': ()}, + ), + ] diff --git a/onadata/apps/logger/models/note.py b/onadata/apps/logger/models/note.py index 40ab9059e..bc0b77742 100644 --- a/onadata/apps/logger/models/note.py +++ b/onadata/apps/logger/models/note.py @@ -11,6 +11,3 @@ class Note(models.Model): class Meta: app_label = 'logger' - permissions = ( - ('view_note', 'View note'), - ) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index ec7f2229d..743475edd 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -2,7 +2,6 @@ import json import os import re -from io import StringIO from hashlib import md5 from xml.sax import saxutils @@ -10,7 +9,7 @@ from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist from django.core.files.storage import get_storage_class -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.db.models.signals import post_save, post_delete from django.utils.encoding import smart_text @@ -103,7 +102,6 @@ class Meta: verbose_name_plural = ugettext_lazy("XForms") ordering = ("id_string",) permissions = ( - (CAN_VIEW_XFORM, _('Can view associated data')), (CAN_ADD_SUBMISSIONS, _('Can make submissions to the form')), (CAN_TRANSFER_OWNERSHIP, _('Can transfer form ownership.')), (CAN_VALIDATE_XFORM, _('Can validate submissions')), @@ -271,7 +269,7 @@ def _xls_file_io(self): if file_path.endswith('.csv'): return convert_csv_to_xls(ff.read()) else: - return StringIO(ff.read()) + return BytesIO(ff.read()) @property def settings(self): diff --git a/onadata/apps/logger/south_migrations/0012_add_permission_view_xform.py b/onadata/apps/logger/south_migrations/0012_add_permission_view_xform.py index 9cea13c6f..5cb79542f 100644 --- a/onadata/apps/logger/south_migrations/0012_add_permission_view_xform.py +++ b/onadata/apps/logger/south_migrations/0012_add_permission_view_xform.py @@ -21,7 +21,7 @@ def forwards(self, orm): ct = ContentType.objects.get(model='xform', app_label='odk_logger') Permission.objects.get(content_type=ct, codename='can_view').delete() # add new permission label - perm, created = Permission.objects.get_or_create(content_type=ct, codename='view_xform', name='Can view associated data') + perm, created = Permission.objects.get_or_create(content_type=ct, codename='view_xform', name='Can view associated data!') except (ContentType.DoesNotExist, Permission.DoesNotExist): pass diff --git a/onadata/apps/logger/tests/test_briefcase_api.py b/onadata/apps/logger/tests/test_briefcase_api.py index 8d4959bce..2213a73aa 100644 --- a/onadata/apps/logger/tests/test_briefcase_api.py +++ b/onadata/apps/logger/tests/test_briefcase_api.py @@ -1,7 +1,7 @@ # coding: utf-8 import os -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.files.storage import get_storage_class from django_digest.test import DigestAuth from rest_framework.test import APIRequestFactory diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 0879be34a..0e059822d 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -7,7 +7,7 @@ from django.contrib.auth import authenticate from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import UploadedFile -from django.core.urlresolvers import reverse +from django.urls import reverse from django.test import RequestFactory from django_digest.test import Client as DigestClient from httmock import urlmatch, HTTMock diff --git a/onadata/apps/logger/tests/test_encrypted_submissions.py b/onadata/apps/logger/tests/test_encrypted_submissions.py index a3e86777d..897de7c28 100644 --- a/onadata/apps/logger/tests/test_encrypted_submissions.py +++ b/onadata/apps/logger/tests/test_encrypted_submissions.py @@ -1,7 +1,7 @@ # coding: utf-8 import os -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth import authenticate from rest_framework.test import APIRequestFactory diff --git a/onadata/apps/logger/tests/test_importing_database.py b/onadata/apps/logger/tests/test_importing_database.py index dda4f3b15..109e651ca 100644 --- a/onadata/apps/logger/tests/test_importing_database.py +++ b/onadata/apps/logger/tests/test_importing_database.py @@ -2,7 +2,7 @@ import os from django.core.files.storage import get_storage_class -from django.core.urlresolvers import reverse +from django.urls import reverse from django.conf import settings from onadata.apps.main.tests.test_base import TestBase diff --git a/onadata/apps/logger/tests/test_webforms.py b/onadata/apps/logger/tests/test_webforms.py index c35c24a33..d987c2819 100644 --- a/onadata/apps/logger/tests/test_webforms.py +++ b/onadata/apps/logger/tests/test_webforms.py @@ -1,22 +1,11 @@ # coding: utf-8 import os -import requests from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.xform_instance_parser import get_uuid_from_xml from onadata.libs.utils.logger_tools import inject_instanceid -from httmock import urlmatch, HTTMock - - -@urlmatch(netloc=r'(.*\.)?enketo\.formhub\.org$') -def enketo_edit_mock(url, request): - response = requests.Response() - response.status_code = 201 - response._content = '{"edit_url": "https://hmh2a.enketo.formhub.org"}' - return response - class TestWebforms(TestBase): def setUp(self): diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 3fde67e39..6063f8e52 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -24,7 +24,6 @@ from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.template import loader -from django.template import RequestContext from django.utils.six import string_types, text_type from django.utils.translation import ugettext as _ from django.views.decorators.http import require_GET, require_POST diff --git a/onadata/apps/main/context_processors.py b/onadata/apps/main/context_processors.py index 169da275a..2e2f35b5b 100644 --- a/onadata/apps/main/context_processors.py +++ b/onadata/apps/main/context_processors.py @@ -24,6 +24,7 @@ def site_name(request): site_name = site.name return {'SITE_NAME': site_name} + def base_url(request): """ Return a BASE_URL template context for the current request. @@ -33,4 +34,4 @@ def base_url(request): else: scheme = 'http://' - return {'BASE_URL': scheme + request.get_host(),} + return {'BASE_URL': scheme + request.get_host(), } diff --git a/onadata/apps/main/migrations/0001_initial.py b/onadata/apps/main/migrations/0001_initial.py index d6974409b..1d9c9425f 100644 --- a/onadata/apps/main/migrations/0001_initial.py +++ b/onadata/apps/main/migrations/0001_initial.py @@ -39,7 +39,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(max_length=255, blank=True)), ('city', models.CharField(max_length=255, blank=True)), - ('country', models.CharField(blank=True, max_length=2, choices=[(b'AF', 'Afghanistan'), (b'AL', 'Albania'), (b'DZ', 'Algeria'), (b'AS', 'American Samoa'), (b'AD', 'Andorra'), (b'AO', 'Angola'), (b'AI', 'Anguilla'), (b'AQ', 'Antarctica'), (b'AG', 'Antigua and Barbuda'), (b'AR', 'Argentina'), (b'AM', 'Armenia'), (b'AW', 'Aruba'), (b'AU', 'Australia'), (b'AT', 'Austria'), (b'AZ', 'Azerbaijan'), (b'BS', 'Bahamas'), (b'BH', 'Bahrain'), (b'BD', 'Bangladesh'), (b'BB', 'Barbados'), (b'BY', 'Belarus'), (b'BE', 'Belgium'), (b'BZ', 'Belize'), (b'BJ', 'Benin'), (b'BM', 'Bermuda'), (b'BT', 'Bhutan'), (b'BO', 'Bolivia'), (b'BQ', 'Bonaire, Sint Eustatius and Saba'), (b'BA', 'Bosnia and Herzegovina'), (b'BW', 'Botswana'), (b'BR', 'Brazil'), (b'IO', 'British Indian Ocean Territory'), (b'BN', 'Brunei Darussalam'), (b'BG', 'Bulgaria'), (b'BF', 'Burkina Faso'), (b'BI', 'Burundi'), (b'KH', 'Cambodia'), (b'CM', 'Cameroon'), (b'CA', 'Canada'), (b'CV', 'Cape Verde'), (b'KY', 'Cayman Islands'), (b'CF', 'Central African Republic'), (b'TD', 'Chad'), (b'CL', 'Chile'), (b'CN', 'China'), (b'CX', 'Christmas Island'), (b'CC', 'Cocos (Keeling) Islands'), (b'CO', 'Colombia'), (b'KM', 'Comoros'), (b'CG', 'Congo'), (b'CD', 'Congo, The Democratic Republic of the'), (b'CK', 'Cook Islands'), (b'CR', 'Costa Rica'), (b'CI', 'Ivory Coast'), (b'HR', 'Croatia'), (b'CU', 'Cuba'), (b'CW', 'Curacao'), (b'CY', 'Cyprus'), (b'CZ', 'Czech Republic'), (b'DK', 'Denmark'), (b'DJ', 'Djibouti'), (b'DM', 'Dominica'), (b'DO', 'Dominican Republic'), (b'EC', 'Ecuador'), (b'EG', 'Egypt'), (b'SV', 'El Salvador'), (b'GQ', 'Equatorial Guinea'), (b'ER', 'Eritrea'), (b'EE', 'Estonia'), (b'ET', 'Ethiopia'), (b'FK', 'Falkland Islands (Malvinas)'), (b'FO', 'Faroe Islands'), (b'FJ', 'Fiji'), (b'FI', 'Finland'), (b'FR', 'France'), (b'GF', 'French Guiana'), (b'PF', 'French Polynesia'), (b'TF', 'French Southern Territories'), (b'GA', 'Gabon'), (b'GM', 'Gambia'), (b'GE', 'Georgia'), (b'DE', 'Germany'), (b'GH', 'Ghana'), (b'GI', 'Gibraltar'), (b'GR', 'Greece'), (b'GL', 'Greenland'), (b'GD', 'Grenada'), (b'GP', 'Guadeloupe'), (b'GU', 'Guam'), (b'GT', 'Guatemala'), (b'GG', 'Guernsey'), (b'GN', 'Guinea'), (b'GW', 'Guinea-Bissau'), (b'GY', 'Guyana'), (b'HT', 'Haiti'), (b'HM', 'Heard Island and McDonald Islands'), (b'VA', 'Holy See (Vatican City State)'), (b'HN', 'Honduras'), (b'HK', 'Hong Kong'), (b'HU', 'Hungary'), (b'IS', 'Iceland'), (b'IN', 'India'), (b'ID', 'Indonesia'), (b'XZ', 'Installations in International Waters'), (b'IR', 'Iran, Islamic Republic of'), (b'IQ', 'Iraq'), (b'IE', 'Ireland'), (b'IM', 'Isle of Man'), (b'IL', 'Israel'), (b'IT', 'Italy'), (b'JM', 'Jamaica'), (b'JP', 'Japan'), (b'JE', 'Jersey'), (b'JO', 'Jordan'), (b'KZ', 'Kazakhstan'), (b'KE', 'Kenya'), (b'KI', 'Kiribati'), (b'KP', "Korea, Democratic People's Republic of"), (b'KR', 'Korea, Republic of'), (b'XK', 'Kosovo'), (b'KW', 'Kuwait'), (b'KG', 'Kyrgyzstan'), (b'LA', "Lao People's Democratic Republic"), (b'LV', 'Latvia'), (b'LB', 'Lebanon'), (b'LS', 'Lesotho'), (b'LR', 'Liberia'), (b'LY', 'Libyan Arab Jamahiriya'), (b'LI', 'Liechtenstein'), (b'LT', 'Lithuania'), (b'LU', 'Luxembourg'), (b'MO', 'Macao'), (b'MK', 'Macedonia, The former Yugoslav Republic of'), (b'MG', 'Madagascar'), (b'MW', 'Malawi'), (b'MY', 'Malaysia'), (b'MV', 'Maldives'), (b'ML', 'Mali'), (b'MT', 'Malta'), (b'MH', 'Marshall Islands'), (b'MQ', 'Martinique'), (b'MR', 'Mauritania'), (b'MU', 'Mauritius'), (b'YT', 'Mayotte'), (b'MX', 'Mexico'), (b'FM', 'Micronesia, Federated States of'), (b'MD', 'Moldova, Republic of'), (b'MC', 'Monaco'), (b'MN', 'Mongolia'), (b'ME', 'Montenegro'), (b'MS', 'Montserrat'), (b'MA', 'Morocco'), (b'MZ', 'Mozambique'), (b'MM', 'Myanmar'), (b'NA', 'Namibia'), (b'NR', 'Nauru'), (b'NP', 'Nepal'), (b'NL', 'Netherlands'), (b'NC', 'New Caledonia'), (b'NZ', 'New Zealand'), (b'NI', 'Nicaragua'), (b'NE', 'Niger'), (b'NG', 'Nigeria'), (b'NU', 'Niue'), (b'NF', 'Norfolk Island'), (b'MP', 'Northern Mariana Islands'), (b'NO', 'Norway'), (b'OM', 'Oman'), (b'PK', 'Pakistan'), (b'PW', 'Palau'), (b'PS', 'Palestinian Territory, Occupied'), (b'PA', 'Panama'), (b'PG', 'Papua New Guinea'), (b'PY', 'Paraguay'), (b'PE', 'Peru'), (b'PH', 'Philippines'), (b'PN', 'Pitcairn'), (b'PL', 'Poland'), (b'PT', 'Portugal'), (b'PR', 'Puerto Rico'), (b'QA', 'Qatar'), (b'RE', 'Reunion'), (b'RO', 'Romania'), (b'RU', 'Russian Federation'), (b'RW', 'Rwanda'), (b'SH', 'Saint Helena'), (b'KN', 'Saint Kitts and Nevis'), (b'LC', 'Saint Lucia'), (b'PM', 'Saint Pierre and Miquelon'), (b'VC', 'Saint Vincent and the Grenadines'), (b'WS', 'Samoa'), (b'SM', 'San Marino'), (b'ST', 'Sao Tome and Principe'), (b'SA', 'Saudi Arabia'), (b'SN', 'Senegal'), (b'RS', 'Serbia'), (b'SC', 'Seychelles'), (b'SL', 'Sierra Leone'), (b'SG', 'Singapore'), (b'SX', 'Sint Maarten (Dutch Part)'), (b'SK', 'Slovakia'), (b'SI', 'Slovenia'), (b'SB', 'Solomon Islands'), (b'SO', 'Somalia'), (b'ZA', 'South Africa'), (b'GS', 'South Georgia and the South Sandwich Islands'), (b'SS', 'South Sudan'), (b'ES', 'Spain'), (b'LK', 'Sri Lanka'), (b'SD', 'Sudan'), (b'SR', 'Suriname'), (b'SJ', 'Svalbard and Jan Mayen'), (b'SZ', 'Swaziland'), (b'SE', 'Sweden'), (b'CH', 'Switzerland'), (b'SY', 'Syrian Arab Republic'), (b'TW', 'Taiwan, Province of China'), (b'TJ', 'Tajikistan'), (b'TZ', 'Tanzania, United Republic of'), (b'TH', 'Thailand'), (b'TL', 'Timor-Leste'), (b'TG', 'Togo'), (b'TK', 'Tokelau'), (b'TO', 'Tonga'), (b'TT', 'Trinidad and Tobago'), (b'TN', 'Tunisia'), (b'TR', 'Turkey'), (b'TM', 'Turkmenistan'), (b'TC', 'Turks and Caicos Islands'), (b'TV', 'Tuvalu'), (b'UG', 'Uganda'), (b'UA', 'Ukraine'), (b'AE', 'United Arab Emirates'), (b'GB', 'United Kingdom'), (b'US', 'United States'), (b'UM', 'United States Minor Outlying Islands'), (b'UY', 'Uruguay'), (b'UZ', 'Uzbekistan'), (b'VU', 'Vanuatu'), (b'VE', 'Venezuela'), (b'VN', 'Viet Nam'), (b'VG', 'Virgin Islands, British'), (b'VI', 'Virgin Islands, U.S.'), (b'WF', 'Wallis and Futuna'), (b'EH', 'Western Sahara'), (b'YE', 'Yemen'), (b'ZM', 'Zambia'), (b'ZW', 'Zimbabwe'), (b'ZZ', 'Unknown or unspecified country')])), + ('country', models.CharField(blank=True, max_length=2, choices=[('AF', 'Afghanistan'), ('AL', 'Albania'), ('DZ', 'Algeria'), ('AS', 'American Samoa'), ('AD', 'Andorra'), ('AO', 'Angola'), ('AI', 'Anguilla'), ('AQ', 'Antarctica'), ('AG', 'Antigua and Barbuda'), ('AR', 'Argentina'), ('AM', 'Armenia'), ('AW', 'Aruba'), ('AU', 'Australia'), ('AT', 'Austria'), ('AZ', 'Azerbaijan'), ('BS', 'Bahamas'), ('BH', 'Bahrain'), ('BD', 'Bangladesh'), ('B', 'Barbados'), ('BY', 'Belarus'), ('BE', 'Belgium'), ('BZ', 'Belize'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BT', 'Bhutan'), ('BO', 'Bolivia'), ('BQ', 'Bonaire, Sint Eustatius and Saba'), ('BA', 'Bosnia and Herzegovina'), ('BW', 'Botswana'), ('BR', 'Brazil'), ('IO', 'British Indian Ocean Territory'), ('BN', 'Brunei Darussalam'), ('BG', 'Bulgaria'), ('BF', 'Burkina Faso'), ('BI', 'Burundi'), ('KH', 'Cambodia'), ('CM', 'Cameroon'), ('CA', 'Canada'), ('CV', 'Cape Verde'), ('KY', 'Cayman Islands'), ('CF', 'Central African Republic'), ('TD', 'Chad'), ('CL', 'Chile'), ('CN', 'China'), ('CX', 'Christmas Island'), ('CC', 'Cocos (Keeling) Islands'), ('CO', 'Colombia'), ('KM', 'Comoros'), ('CG', 'Congo'), ('CD', 'Congo, The Democratic Republic of the'), ('CK', 'Cook Islands'), ('CR', 'Costa Rica'), ('CI', 'Ivory Coast'), ('HR', 'Croatia'), ('CU', 'Cuba'), ('CW', 'Curacao'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DK', 'Denmark'), ('DJ', 'Djibouti'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('EC', 'Ecuador'), ('EG', 'Egypt'), ('SV', 'El Salvador'), ('GQ', 'Equatorial Guinea'), ('ER', 'Eritrea'), ('EE', 'Estonia'), ('ET', 'Ethiopia'), ('FK', 'Falkland Islands (Malvinas)'), ('FO', 'Faroe Islands'), ('FJ', 'Fiji'), ('FI', 'Finland'), ('FR', 'France'), ('GF', 'French Guiana'), ('PF', 'French Polynesia'), ('TF', 'French Southern Territories'), ('GA', 'Gabon'), ('GM', 'Gambia'), ('GE', 'Georgia'), ('DE', 'Germany'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GR', 'Greece'), ('GL', 'Greenland'), ('GD', 'Grenada'), ('GP', 'Guadeloupe'), ('GU', 'Guam'), ('GT', 'Guatemala'), ('GG', 'Guernsey'), ('GN', 'Guinea'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HT', 'Haiti'), ('HM', 'Heard Island and McDonald Islands'), ('VA', 'Holy See (Vatican City State)'), ('HN', 'Honduras'), ('HK', 'Hong Kong'), ('HU', 'Hungary'), ('IS', 'Iceland'), ('IN', 'India'), ('ID', 'Indonesia'), ('XZ', 'Installations in International Waters'), ('IR', 'Iran, Islamic Republic of'), ('IQ', 'Iraq'), ('IE', 'Ireland'), ('IM', 'Isle of Man'), ('IL', 'Israel'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JP', 'Japan'), ('JE', 'Jersey'), ('JO', 'Jordan'), ('KZ', 'Kazakhstan'), ('KE', 'Kenya'), ('KI', 'Kiribati'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('XK', 'Kosovo'), ('KW', 'Kuwait'), ('KG', 'Kyrgyzstan'), ('LA', "Lao People's Democratic Republic"), ('LV', 'Latvia'), ('L', 'Lebanon'), ('LS', 'Lesotho'), ('LR', 'Liberia'), ('LY', 'Libyan Arab Jamahiriya'), ('LI', 'Liechtenstein'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('MO', 'Macao'), ('MK', 'Macedonia, The former Yugoslav Republic of'), ('MG', 'Madagascar'), ('MW', 'Malawi'), ('MY', 'Malaysia'), ('MV', 'Maldives'), ('ML', 'Mali'), ('MT', 'Malta'), ('MH', 'Marshall Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MU', 'Mauritius'), ('YT', 'Mayotte'), ('MX', 'Mexico'), ('FM', 'Micronesia, Federated States of'), ('MD', 'Moldova, Republic of'), ('MC', 'Monaco'), ('MN', 'Mongolia'), ('ME', 'Montenegro'), ('MS', 'Montserrat'), ('MA', 'Morocco'), ('MZ', 'Mozambique'), ('MM', 'Myanmar'), ('NA', 'Namibia'), ('NR', 'Nauru'), ('NP', 'Nepal'), ('NL', 'Netherlands'), ('NC', 'New Caledonia'), ('NZ', 'New Zealand'), ('NI', 'Nicaragua'), ('NE', 'Niger'), ('NG', 'Nigeria'), ('NU', 'Niue'), ('NF', 'Norfolk Island'), ('MP', 'Northern Mariana Islands'), ('NO', 'Norway'), ('OM', 'Oman'), ('PK', 'Pakistan'), ('PW', 'Palau'), ('PS', 'Palestinian Territory, Occupied'), ('PA', 'Panama'), ('PG', 'Papua New Guinea'), ('PY', 'Paraguay'), ('PE', 'Peru'), ('PH', 'Philippines'), ('PN', 'Pitcairn'), ('PL', 'Poland'), ('PT', 'Portugal'), ('PR', 'Puerto Rico'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SH', 'Saint Helena'), ('KN', 'Saint Kitts and Nevis'), ('LC', 'Saint Lucia'), ('PM', 'Saint Pierre and Miquelon'), ('VC', 'Saint Vincent and the Grenadines'), ('WS', 'Samoa'), ('SM', 'San Marino'), ('ST', 'Sao Tome and Principe'), ('SA', 'Saudi Arabia'), ('SN', 'Senegal'), ('RS', 'Serbia'), ('SC', 'Seychelles'), ('SL', 'Sierra Leone'), ('SG', 'Singapore'), ('SX', 'Sint Maarten (Dutch Part)'), ('SK', 'Slovakia'), ('SI', 'Slovenia'), ('S', 'Solomon Islands'), ('SO', 'Somalia'), ('ZA', 'South Africa'), ('GS', 'South Georgia and the South Sandwich Islands'), ('SS', 'South Sudan'), ('ES', 'Spain'), ('LK', 'Sri Lanka'), ('SD', 'Sudan'), ('SR', 'Suriname'), ('SJ', 'Svalbard and Jan Mayen'), ('SZ', 'Swaziland'), ('SE', 'Sweden'), ('CH', 'Switzerland'), ('SY', 'Syrian Arab Republic'), ('TW', 'Taiwan, Province of China'), ('TJ', 'Tajikistan'), ('TZ', 'Tanzania, United Republic of'), ('TH', 'Thailand'), ('TL', 'Timor-Leste'), ('TG', 'Togo'), ('TK', 'Tokelau'), ('TO', 'Tonga'), ('TT', 'Trinidad and Tobago'), ('TN', 'Tunisia'), ('TR', 'Turkey'), ('TM', 'Turkmenistan'), ('TC', 'Turks and Caicos Islands'), ('TV', 'Tuvalu'), ('UG', 'Uganda'), ('UA', 'Ukraine'), ('AE', 'United Arab Emirates'), ('G', 'United Kingdom'), ('US', 'United States'), ('UM', 'United States Minor Outlying Islands'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VU', 'Vanuatu'), ('VE', 'Venezuela'), ('VN', 'Viet Nam'), ('VG', 'Virgin Islands, British'), ('VI', 'Virgin Islands, U.S.'), ('WF', 'Wallis and Futuna'), ('EH', 'Western Sahara'), ('YE', 'Yemen'), ('ZM', 'Zambia'), ('ZW', 'Zimbabwe'), ('ZZ', 'Unknown or unspecified country')])), ('organization', models.CharField(max_length=255, blank=True)), ('home_page', models.CharField(max_length=255, blank=True)), ('twitter', models.CharField(max_length=255, blank=True)), diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index 3a9d91254..1430cdf23 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -148,7 +148,7 @@ def _submit_transport_instance_w_attachment(self, survey_at=0): os.path.join(self.this_directory, 'fixtures', 'transportation', 'instances', s, media_file)) self.attachment = Attachment.objects.all().reverse()[0] - self.attachment_media_file = self.attachment.media_file + self.attachment_media_file = str(self.attachment.media_file) def _publish_transportation_form_and_submit_instance(self): self._publish_transportation_form() diff --git a/onadata/apps/main/tests/test_form_auth.py b/onadata/apps/main/tests/test_form_auth.py index c7e6fc6c2..0704c0183 100644 --- a/onadata/apps/main/tests/test_form_auth.py +++ b/onadata/apps/main/tests/test_form_auth.py @@ -1,5 +1,5 @@ # coding: utf-8 -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.views import login_redirect from .test_base import TestBase diff --git a/onadata/apps/main/tests/test_form_errors.py b/onadata/apps/main/tests/test_form_errors.py index 0ef81294c..f94f743cb 100644 --- a/onadata/apps/main/tests/test_form_errors.py +++ b/onadata/apps/main/tests/test_form_errors.py @@ -2,7 +2,7 @@ import os from unittest import skip -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.files.storage import get_storage_class from onadata.apps.main.views import show diff --git a/onadata/apps/main/tests/test_form_exports.py b/onadata/apps/main/tests/test_form_exports.py index 2b0c91055..383579714 100644 --- a/onadata/apps/main/tests/test_form_exports.py +++ b/onadata/apps/main/tests/test_form_exports.py @@ -4,7 +4,7 @@ import csv import tempfile -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.files.storage import get_storage_class, FileSystemStorage from django.utils import timezone from xlrd import open_workbook diff --git a/onadata/apps/main/tests/test_form_metadata.py b/onadata/apps/main/tests/test_form_metadata.py index 56b9edb55..328788749 100644 --- a/onadata/apps/main/tests/test_form_metadata.py +++ b/onadata/apps/main/tests/test_form_metadata.py @@ -4,7 +4,7 @@ from django.core.files.base import File from django.core.files.uploadedfile import InMemoryUploadedFile -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.models import MetaData from onadata.apps.main.views import ( @@ -114,7 +114,7 @@ def test_windows_csv_file_upload(self): media_file = os.path.join( self.this_directory, 'fixtures', 'transportation', 'transportation.csv') - f = InMemoryUploadedFile(open(media_file), + f = InMemoryUploadedFile(open(media_file, 'rb'), 'media', 'transportation.csv', 'application/octet-stream', diff --git a/onadata/apps/main/tests/test_form_show.py b/onadata/apps/main/tests/test_form_show.py index 70f922cbe..12db05dad 100644 --- a/onadata/apps/main/tests/test_form_show.py +++ b/onadata/apps/main/tests/test_form_show.py @@ -3,7 +3,7 @@ from unittest import skip from django.core.files.base import ContentFile -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.views import show, form_photos, \ show_form_settings diff --git a/onadata/apps/main/tests/test_http_auth.py b/onadata/apps/main/tests/test_http_auth.py index 30c4f79f8..4866e83b7 100644 --- a/onadata/apps/main/tests/test_http_auth.py +++ b/onadata/apps/main/tests/test_http_auth.py @@ -1,5 +1,5 @@ # coding: utf-8 -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.tests.test_base import TestBase diff --git a/onadata/apps/main/tests/test_permissions.py b/onadata/apps/main/tests/test_permissions.py index a230d38c1..ce4cdd384 100644 --- a/onadata/apps/main/tests/test_permissions.py +++ b/onadata/apps/main/tests/test_permissions.py @@ -1,14 +1,10 @@ +# coding: utf-8 import os -import unittest -from django.core.urlresolvers import reverse +from django.urls import reverse from guardian.shortcuts import assign_perm, remove_perm -from onadata.apps.logger.models import XForm -#from onadata.apps.viewer.views import map_view -from onadata.apps.main.views import show, edit, profile -from onadata.apps.main.models import MetaData - +from onadata.apps.main.views import show from .test_base import TestBase diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 2cb5f721f..aa4b317ab 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -7,7 +7,7 @@ import re import unittest -from django.core.urlresolvers import reverse +from django.urls import reverse from django.conf import settings from django_digest.test import Client as DigestClient from django.core.files.uploadedfile import UploadedFile @@ -19,7 +19,7 @@ from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.libs.tests.utils.xml import pyxform_version_agnostic +from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME from .test_base import TestBase @@ -178,8 +178,8 @@ def _download_xform(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + is_equal_xml(pyxform_version_agnostic(response_doc.toxml()), + pyxform_version_agnostic(expected_doc.toxml())) def _check_csv_export(self): self._check_data_dictionary() diff --git a/onadata/apps/main/tests/test_user_profile.py b/onadata/apps/main/tests/test_user_profile.py index b4f98e3e7..330bf014d 100644 --- a/onadata/apps/main/tests/test_user_profile.py +++ b/onadata/apps/main/tests/test_user_profile.py @@ -4,7 +4,7 @@ from django.test import TestCase from django.test.client import Client from django.contrib.auth.models import User -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.views import profile diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index 2e71243e5..01c03df21 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -1,17 +1,16 @@ # coding: utf-8 -from django.conf.urls import include, url from django.conf import settings from django.contrib import admin + +from django.urls import include, re_path from django.views.generic import RedirectView from django.views.i18n import JavaScriptCatalog - -from onadata.apps.api.urls import router, router_with_patch_list +from onadata.apps.api.urls import BriefcaseApi from onadata.apps.api.urls import XFormListApi from onadata.apps.api.urls import XFormSubmissionApi -from onadata.apps.api.urls import BriefcaseApi +from onadata.apps.api.urls import router, router_with_patch_list from onadata.apps.main.service_health import service_health - from onadata.apps.main.views import ( # main website views home, @@ -46,7 +45,6 @@ download_jsonform, ) - # Statistics for superusers. The username is irrelevant, but leave it as # the first part of the path to avoid collisions from onadata.apps.logger.views import ( @@ -57,142 +55,139 @@ admin.autodiscover() urlpatterns = [ - # url('') # Same as `url(r'^$', home)`? # change Language - url(r'^i18n/', include('django.conf.urls.i18n')), - url('^api/v1/', include(router.urls)), - url('^api/v1/', include(router_with_patch_list.urls)), - url(r'^service_health/$', service_health), - url(r'^api-docs/', RedirectView.as_view(url='/api/v1/')), - url(r'^api/', RedirectView.as_view(url='/api/v1/')), - url(r'^api/v1', RedirectView.as_view(url='/api/v1/')), + re_path(r'^i18n/', include('django.conf.urls.i18n')), + re_path('^api/v1/', include(router.urls)), + re_path('^api/v1/', include(router_with_patch_list.urls)), + re_path(r'^service_health/$', service_health), + re_path(r'^api-docs/', RedirectView.as_view(url='/api/v1/')), + re_path(r'^api/', RedirectView.as_view(url='/api/v1/')), + re_path(r'^api/v1', RedirectView.as_view(url='/api/v1/')), # django default stuff - url(r'^accounts/', include('registration.auth_urls')), - url(r'^admin/', admin.site.urls), - url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + re_path(r'^accounts/', include('registration.auth_urls')), + re_path(r'^admin/', admin.site.urls), + re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), # oath2_provider - url(r'^o/', include('oauth2_provider.urls', namespace='oauth2_provider')), + re_path(r'^o/', include('oauth2_provider.urls', namespace='oauth2_provider')), # main website views - url(r'^$', home), - url(r'^forms/(?P[^/]+)$', show, name='show_form'), - url(r'^login_redirect/$', login_redirect), - + re_path(r'^$', home), + re_path(r'^forms/(?P[^/]+)$', show, name='show_form'), + re_path(r'^login_redirect/$', login_redirect), # Bring back old url because it's still used by `kpi` # ToDo Remove when `kpi#gallery-2` is merged into master - url(r"^attachment/$", attachment_url, name='attachment_url'), - url(r"^attachment/(?P[^/]+)$", - attachment_url, name='attachment_url'), - url(r"^{}$".format(settings.MEDIA_URL.lstrip('/')), attachment_url, name='attachment_url'), - url(r"^{}(?P[^/]+)$".format(settings.MEDIA_URL.lstrip('/')), - attachment_url, name='attachment_url'), - url(r'^jsi18n/$', JavaScriptCatalog.as_view(), - {'packages': ('main', 'viewer',)}, - name='javascript-catalog'), - url(r'^(?P[^/]+)/$', - profile, name='user_profile'), - url(r'^(?P[^/]+)/api-token$', - api_token, name='api_token'), + re_path(r"^attachment/$", attachment_url, name='attachment_url'), + re_path(r"^attachment/(?P[^/]+)$", + attachment_url, name='attachment_url'), + re_path(r"^{}$".format(settings.MEDIA_URL.lstrip('/')), attachment_url, name='attachment_url'), + re_path(r"^{}(?P[^/]+)$".format(settings.MEDIA_URL.lstrip('/')), + attachment_url, name='attachment_url'), + re_path(r'^jsi18n/$', JavaScriptCatalog.as_view(packages=['onadata.apps.main', 'onadata.apps.viewer']), + name='javascript-catalog'), + re_path(r'^(?P[^/]+)/$', + profile, name='user_profile'), + re_path(r'^(?P[^/]+)/api-token$', + api_token, name='api_token'), # form specific - url(r'^(?P[^/]+)/forms/(?P[^/]+)$', - show, name='show_form'), - url(r'^(?P[^/]+)/forms/(?P[^/]+)/edit$', - edit, name='edit_form'), - url(r'^(?P[^/]+)/forms/(?P[^/]+)/photos', - form_photos, name='form_photos'), - url(r'^(?P[^/]+)/forms/(?P[^/]+)/formid-media/' - r'(?P\d+)', download_media_data, name='download_media_data'), - url(r'^(?P[^/]+)/forms/(?P[^/]+)/form_settings$', - show_form_settings, name='show_form_settings'), + re_path(r'^(?P[^/]+)/forms/(?P[^/]+)$', + show, name='show_form'), + re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/edit$', + edit, name='edit_form'), + re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/photos', + form_photos, name='form_photos'), + re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/formid-media/' + r'(?P\d+)', download_media_data, name='download_media_data'), + re_path(r'^(?P[^/]+)/forms/(?P[^/]+)/form_settings$', + show_form_settings, name='show_form_settings'), # briefcase api urls - url(r"^(?P\w+)/view/submissionList$", - BriefcaseApi.as_view({'get': 'list', 'head': 'list'}), - name='view-submission-list'), - url(r"^(?P\w+)/view/downloadSubmission$", - BriefcaseApi.as_view({'get': 'retrieve', 'head': 'retrieve'}), - name='view-download-submission'), - url(r"^(?P\w+)/formUpload$", - BriefcaseApi.as_view({'post': 'create', 'head': 'create'}), - name='form-upload'), - url(r"^(?P\w+)/upload$", - BriefcaseApi.as_view({'post': 'create', 'head': 'create'}), - name='upload'), + re_path(r"^(?P\w+)/view/submissionList$", + BriefcaseApi.as_view({'get': 'list', 'head': 'list'}), + name='view-submission-list'), + re_path(r"^(?P\w+)/view/downloadSubmission$", + BriefcaseApi.as_view({'get': 'retrieve', 'head': 'retrieve'}), + name='view-download-submission'), + re_path(r"^(?P\w+)/formUpload$", + BriefcaseApi.as_view({'post': 'create', 'head': 'create'}), + name='form-upload'), + re_path(r"^(?P\w+)/upload$", + BriefcaseApi.as_view({'post': 'create', 'head': 'create'}), + name='upload'), # exporting stuff - url(r"^(?P\w+)/forms/(?P[^/]+)/data\.csv$", - data_export, name='csv_export', - kwargs={'export_type': 'csv'}), - url(r"^(?P\w+)/forms/(?P[^/]+)/data\.xls", - data_export, name='xls_export', - kwargs={'export_type': 'xls'}), - url(r"^(?P\w+)/forms/(?P[^/]+)/data\.kml$", - kml_export), - url(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" - r"/new$", create_export, name='create_export'), - url(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" - r"/delete$", delete_export, name='delete_export'), - url(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" - r"/progress$", export_progress, name='export_progress'), - url(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" - r"/$", export_list, name='export_list'), - url(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" - "/(?P[^/]+)$", - export_download, name='export_download'), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/data\.csv$", + data_export, name='csv_export', + kwargs={'export_type': 'csv'}), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/data\.xls", + data_export, name='xls_export', + kwargs={'export_type': 'xls'}), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/data\.kml$", + kml_export), + re_path(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" + r"/new$", create_export, name='create_export'), + re_path(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" + r"/delete$", delete_export, name='delete_export'), + re_path(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" + r"/progress$", export_progress, name='export_progress'), + re_path(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" + r"/$", export_list, name='export_list'), + re_path(r"^(?P\w+)/exports/(?P[^/]+)/(?P\w+)" + "/(?P[^/]+)$", + export_download, name='export_download'), # odk data urls - url(r"^submission$", - XFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), - url(r"^formList$", - XFormListApi.as_view({'get': 'list'}), name='form-list'), - url(r"^(?P\w+)/formList$", - XFormListApi.as_view({'get': 'list'}), name='form-list'), - url(r"^(?P\w+)/xformsManifest/(?P[\d+^/]+)$", - XFormListApi.as_view({'get': 'manifest'}), - name='manifest-url'), - url(r"^xformsManifest/(?P[\d+^/]+)$", - XFormListApi.as_view({'get': 'manifest'}), - name='manifest-url'), - url(r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)" - r"/(?P[\d+^/.]+)$", - XFormListApi.as_view({'get': 'media'}), name='xform-media'), - url(r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)" - r"/(?P[\d+^/.]+)\.(?P[a-z0-9]+)$", - XFormListApi.as_view({'get': 'media'}), name='xform-media'), - url(r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", - XFormListApi.as_view({'get': 'media'}), name='xform-media'), - url(r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\." - r"(?P[a-z0-9]+)$", - XFormListApi.as_view({'get': 'media'}), name='xform-media'), - url(r"^(?P\w+)/submission$", - XFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}), - name='submissions'), - url(r"^(?P\w+)/bulk-submission$", - bulksubmission), - url(r"^(?P\w+)/bulk-submission-form$", - bulksubmission_form), - url(r"^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$", - XFormListApi.as_view({'get': 'retrieve'}), - name="download_xform"), - url(r"^(?P\w+)/forms/(?P[^/]+)/form\.xml$", - download_xform, name="download_xform"), - url(r"^(?P\w+)/forms/(?P[^/]+)/form\.xls$", - download_xlsform, - name="download_xlsform"), - url(r"^(?P\w+)/forms/(?P[^/]+)/form\.json", - download_jsonform, - name="download_jsonform"), - url(r'^favicon\.ico', - RedirectView.as_view(url='/static/images/favicon.ico')), + re_path(r"^submission$", + XFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}), + name='submissions'), + re_path(r"^formList$", + XFormListApi.as_view({'get': 'list'}), name='form-list'), + re_path(r"^(?P\w+)/formList$", + XFormListApi.as_view({'get': 'list'}), name='form-list'), + re_path(r"^(?P\w+)/xformsManifest/(?P[\d+^/]+)$", + XFormListApi.as_view({'get': 'manifest'}), + name='manifest-url'), + re_path(r"^xformsManifest/(?P[\d+^/]+)$", + XFormListApi.as_view({'get': 'manifest'}), + name='manifest-url'), + re_path(r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)" + r"/(?P[\d+^/.]+)$", + XFormListApi.as_view({'get': 'media'}), name='xform-media'), + re_path(r"^(?P\w+)/xformsMedia/(?P[\d+^/]+)" + r"/(?P[\d+^/.]+)\.(?P[a-z0-9]+)$", + XFormListApi.as_view({'get': 'media'}), name='xform-media'), + re_path(r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)$", + XFormListApi.as_view({'get': 'media'}), name='xform-media'), + re_path(r"^xformsMedia/(?P[\d+^/]+)/(?P[\d+^/.]+)\." + r"(?P[a-z0-9]+)$", + XFormListApi.as_view({'get': 'media'}), name='xform-media'), + re_path(r"^(?P\w+)/submission$", + XFormSubmissionApi.as_view({'post': 'create', 'head': 'create'}), + name='submissions'), + re_path(r"^(?P\w+)/bulk-submission$", + bulksubmission), + re_path(r"^(?P\w+)/bulk-submission-form$", + bulksubmission_form), + re_path(r"^(?P\w+)/forms/(?P[\d+^/]+)/form\.xml$", + XFormListApi.as_view({'get': 'retrieve'}), + name="download_xform"), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/form\.xml$", + download_xform, name="download_xform"), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/form\.xls$", + download_xlsform, + name="download_xlsform"), + re_path(r"^(?P\w+)/forms/(?P[^/]+)/form\.json", + download_jsonform, + name="download_jsonform"), + re_path(r'^favicon\.ico', + RedirectView.as_view(url='/static/images/favicon.ico')), # Statistics for superusers. The username is irrelevant, but leave it as # the first part of the path to avoid collisions - url(r"^(?P[^/]+)/superuser_stats/$", - superuser_stats), - url(r"^(?P[^/]+)/superuser_stats/(?P[^/]+)$", - retrieve_superuser_stats), + re_path(r"^(?P[^/]+)/superuser_stats/$", + superuser_stats), + re_path(r"^(?P[^/]+)/superuser_stats/(?P[^/]+)$", + retrieve_superuser_stats), ] diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 05aa91f48..c81e3b111 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -6,7 +6,7 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.core.files.storage import get_storage_class -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponse from django.http import HttpResponseForbidden from django.http import HttpResponseNotFound diff --git a/onadata/apps/viewer/migrations/0001_initial.py b/onadata/apps/viewer/migrations/0001_initial.py index a654dfed4..60901004c 100644 --- a/onadata/apps/viewer/migrations/0001_initial.py +++ b/onadata/apps/viewer/migrations/0001_initial.py @@ -26,7 +26,7 @@ class Migration(migrations.Migration): ('created_on', models.DateTimeField(auto_now_add=True)), ('filename', models.CharField(max_length=255, null=True, blank=True)), ('filedir', models.CharField(max_length=255, null=True, blank=True)), - ('export_type', models.CharField(default=b'xls', max_length=10, choices=[(b'xls', b'Excel'), (b'csv', b'CSV'), (b'gdoc', b'GDOC'), (b'zip', b'ZIP'), (b'kml', b'kml'), (b'csv_zip', b'CSV ZIP'), (b'sav_zip', b'SAV ZIP'), (b'sav', b'SAV'), (b'external', b'Excel')])), + ('export_type', models.CharField(default='xls', max_length=10, choices=[('xls', 'Excel'), ('csv', 'CSV'), ('gdoc', 'GDOC'), ('zip', 'ZIP'), ('kml', 'kml'), ('csv_zip', 'CSV ZIP'), ('sav_zip', 'SAV ZIP'), ('sav', 'SAV'), ('external', 'Excel')])), ('task_id', models.CharField(max_length=255, null=True, blank=True)), ('time_of_last_submission', models.DateTimeField(default=None, null=True)), ('internal_status', models.SmallIntegerField(default=0, max_length=1)), diff --git a/onadata/apps/viewer/migrations/0003_auto_20171123_1521.py b/onadata/apps/viewer/migrations/0003_auto_20171123_1521.py index bc0a35e02..742a89839 100644 --- a/onadata/apps/viewer/migrations/0003_auto_20171123_1521.py +++ b/onadata/apps/viewer/migrations/0003_auto_20171123_1521.py @@ -12,6 +12,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='export', name='export_type', - field=models.CharField(default=b'xls', max_length=10, choices=[(b'xls', b'Excel'), (b'csv', b'CSV'), (b'gdoc', b'GDOC'), (b'zip', b'ZIP'), (b'kml', b'kml'), (b'csv_zip', b'CSV ZIP'), (b'sav_zip', b'SAV ZIP'), (b'sav', b'SAV'), (b'external', b'Excel'), (b'analyser', b'Analyser')]), + field=models.CharField(default='xls', max_length=10, choices=[('xls', 'Excel'), ('csv', 'CSV'), ('gdoc', 'GDOC'), ('zip', 'ZIP'), ('kml', 'kml'), ('csv_zip', 'CSV ZIP'), ('sav_zip', 'SAV ZIP'), ('sav', 'SAV'), ('external', 'Excel'), ('analyser', 'Analyser')]), ), ] diff --git a/onadata/apps/viewer/migrations/0004_update_meta_data_export_types.py b/onadata/apps/viewer/migrations/0004_update_meta_data_export_types.py index 6615b59af..9093458c0 100644 --- a/onadata/apps/viewer/migrations/0004_update_meta_data_export_types.py +++ b/onadata/apps/viewer/migrations/0004_update_meta_data_export_types.py @@ -14,6 +14,15 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='export', name='export_type', - field=models.CharField(default=b'xls', max_length=10, choices=[(b'xls', b'Excel'), (b'csv', b'CSV'), (b'zip', b'ZIP'), (b'kml', b'kml')]), + field=models.CharField( + default='xls', + max_length=10, + choices=[ + ('xls', 'Excel'), + ('csv', 'CSV'), + ('zip', 'ZIP'), + ('kml', 'kml'), + ], + ), ), ] diff --git a/onadata/apps/viewer/tests/test_attachment_url.py b/onadata/apps/viewer/tests/test_attachment_url.py index ba06e1994..0bdea5a1f 100644 --- a/onadata/apps/viewer/tests/test_attachment_url.py +++ b/onadata/apps/viewer/tests/test_attachment_url.py @@ -1,5 +1,5 @@ # coding: utf-8 -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import Attachment @@ -16,7 +16,7 @@ def setUp(self): self._publish_transportation_form() self._submit_transport_instance_w_attachment() self.url = reverse( - attachment_url, kwargs={'size': 'original'}) + 'attachment_url', kwargs={'size': 'original'}) def test_attachment_url(self): self.assertEqual( diff --git a/onadata/apps/viewer/tests/test_export_list.py b/onadata/apps/viewer/tests/test_export_list.py index cc2f534dc..be68e207e 100644 --- a/onadata/apps/viewer/tests/test_export_list.py +++ b/onadata/apps/viewer/tests/test_export_list.py @@ -2,7 +2,7 @@ import os import unittest -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.export import Export diff --git a/onadata/apps/viewer/tests/test_exports.py b/onadata/apps/viewer/tests/test_exports.py index 34e03a340..db4ea6f3b 100644 --- a/onadata/apps/viewer/tests/test_exports.py +++ b/onadata/apps/viewer/tests/test_exports.py @@ -9,7 +9,7 @@ from django.conf import settings from django.core.files.storage import get_storage_class, FileSystemStorage -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.dateparse import parse_datetime from xlrd import open_workbook diff --git a/onadata/apps/viewer/tests/test_kml_export.py b/onadata/apps/viewer/tests/test_kml_export.py index d81de3441..d6a963900 100644 --- a/onadata/apps/viewer/tests/test_kml_export.py +++ b/onadata/apps/viewer/tests/test_kml_export.py @@ -2,7 +2,7 @@ import os import unittest -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.instance import Instance diff --git a/onadata/apps/viewer/tests/test_map_view.py b/onadata/apps/viewer/tests/test_map_view.py index 32d24107a..80a8154b4 100644 --- a/onadata/apps/viewer/tests/test_map_view.py +++ b/onadata/apps/viewer/tests/test_map_view.py @@ -1,6 +1,6 @@ # coding: utf-8 from django.test import TestCase -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.logger.views import download_jsonform diff --git a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py index 244f9a7ea..f5e0ed066 100644 --- a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py +++ b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py @@ -4,7 +4,7 @@ from tempfile import NamedTemporaryFile from django.utils.dateparse import parse_datetime -from django.core.urlresolvers import reverse +from django.urls import reverse from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.xform import XForm diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 755be0241..a4d7650a7 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -11,7 +11,7 @@ from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage from django.core.files.storage import get_storage_class -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db.models import Q from django.http import ( HttpResponseForbidden, HttpResponseRedirect, HttpResponseNotFound, @@ -362,7 +362,7 @@ def kml_export(request, username, id_string): def attachment_url(request, size='medium'): media_file = request.GET.get('media_file') - # TODO: how to make sure we have the right media file, + # this assumes duplicates are the same file. # # Django seems to already handle that. It appends datetime to the filename. diff --git a/onadata/koboform/redirect_middleware.py b/onadata/koboform/redirect_middleware.py index 231c5bcc8..f5a3607ef 100644 --- a/onadata/koboform/redirect_middleware.py +++ b/onadata/koboform/redirect_middleware.py @@ -1,6 +1,6 @@ # coding: utf-8 from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse +from django.urls import reverse from django.utils.deprecation import MiddlewareMixin from onadata import koboform @@ -35,11 +35,11 @@ def process_view(self, request, view, args, kwargs): if koboform.active and koboform.autoredirect: login_url = koboform.redirect_url(login_url) - if view_name is 'login': + if view_name == 'login': return HttpResponseRedirect(koboform.login_url(next_kobocat_url='/')) - if view_name is 'logout': + if view_name == 'logout': return HttpResponseRedirect(koboform.redirect_url('/accounts/logout/')) - if view_name is 'FHRegistrationView': + if view_name == 'FHRegistrationView': return HttpResponseRedirect(koboform.redirect_url('/accounts/register/')) if view_name in DISABLED_VIEWS: diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index 0d53dfd8c..81e101581 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -35,6 +35,8 @@ class Meta: 'data_type', 'data_file', 'data_file_type', + 'file_hash', + 'url', 'from_kpi', ) diff --git a/onadata/libs/tests/serializers/test_attachment_serializer.py b/onadata/libs/tests/serializers/test_attachment_serializer.py index d922c85ff..a3b8d4410 100644 --- a/onadata/libs/tests/serializers/test_attachment_serializer.py +++ b/onadata/libs/tests/serializers/test_attachment_serializer.py @@ -1,14 +1,14 @@ # coding: utf-8 -from django.test import TransactionTestCase +from django.test import TestCase from onadata.libs.serializers.attachment_serializer import get_path -class TestAttachmentSerializer(TransactionTestCase): +class TestAttachmentSerializer(TestCase): def setUp(self): """ - self.data is a json represenatation of an xform + self.data is a json representation of an xform """ self.data = { "name": "photo_in_group", diff --git a/onadata/libs/tests/utils/xml.py b/onadata/libs/tests/utils/xml.py index ded1ce9c8..d0e296ec9 100644 --- a/onadata/libs/tests/utils/xml.py +++ b/onadata/libs/tests/utils/xml.py @@ -1,4 +1,5 @@ # coding: utf-8 +from xml.dom import minidom PYXFORM_CHANGED_STRINGS = [ ' jr:preload="uid"', @@ -12,3 +13,34 @@ def pyxform_version_agnostic(xml): for str_ in PYXFORM_CHANGED_STRINGS: xml = xml.replace(str_, '') return xml + + +def is_equal_xml(source: str, target: str) -> bool: + """ + Validates if `source` and `target` are identical whatever the order + of node attributes + + See: https://stackoverflow.com/a/321941/1141214 + """ + + def is_equal_element(a, b): + if a.tagName != b.tagName: + return False + if sorted(a.attributes.items()) != sorted(b.attributes.items()): + return False + if len(a.childNodes) != len(b.childNodes): + return False + for ac, bc in zip(a.childNodes, b.childNodes): + if ac.nodeType != bc.nodeType: + return False + if ac.nodeType == ac.TEXT_NODE and ac.data != bc.data: + return False + if ( + ac.nodeType == ac.ELEMENT_NODE + and not is_equal_element(ac, bc) + ): + return False + return True + + da, db = minidom.parseString(source), minidom.parseString(target) + return is_equal_element(da.documentElement, db.documentElement) diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 17de45dcd..9a5f635c0 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -19,7 +19,7 @@ END_TIME = "end_time" END = "end" -# value of INSTANCE_DOC_NAME that indicates a regisration form +# value of INSTANCE_DOC_NAME that indicates a registration form REGISTRATION = "registration" # keys that we'll look for in the registration form NAME = "name" diff --git a/onadata/libs/utils/image_tools.py b/onadata/libs/utils/image_tools.py index 4ee8700a8..b3d9f09db 100644 --- a/onadata/libs/utils/image_tools.py +++ b/onadata/libs/utils/image_tools.py @@ -1,13 +1,13 @@ # coding: utf-8 -from io import StringIO +from io import BytesIO from tempfile import NamedTemporaryFile -from PIL import Image import requests - from django.conf import settings from django.core.files.storage import get_storage_class from django.core.files.base import ContentFile +from PIL import Image + from onadata.libs.utils.viewer_tools import get_path @@ -81,7 +81,7 @@ def resize(filename): original_path = filename req = requests.get(path) if req.status_code == 200: - im = StringIO(req.content) + im = BytesIO(req.content) image = Image.open(im) if image: diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 48e20eeaa..5582cafae 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -561,7 +561,7 @@ def __init__(self, *args, **kwargs): self.content = ''' %s -''' % self.content +''' % smart_str(self.content) class OpenRosaResponseNotFound(OpenRosaResponse): diff --git a/onadata/libs/utils/middleware.py b/onadata/libs/utils/middleware.py index a66073f5d..c60725c5f 100644 --- a/onadata/libs/utils/middleware.py +++ b/onadata/libs/utils/middleware.py @@ -57,21 +57,6 @@ def process_response(self, request, response): return response -class BrokenClientMiddleware(MiddlewareMixin): - """ - ODK Collect sends HTTP-violating localized date strings, e.g. - 'mar., 25 ao\xfbt 2015 07:11:56 GMT+00:00', which wreak havoc on oauthlib. - This middleware detects and discards HTTP_DATE headers that contain invalid - characters. - """ - def process_request(self, request): - if 'HTTP_DATE' in request.META: - try: - request.META['HTTP_DATE'].decode() - except UnicodeDecodeError: - del request.META['HTTP_DATE'] - - class UsernameInResponseHeaderMiddleware(MiddlewareMixin): """ Record the authenticated user (if any) in the `X-KoBoNaUt` HTTP header diff --git a/onadata/settings/common.py b/onadata/settings/base.py similarity index 98% rename from onadata/settings/common.py rename to onadata/settings/base.py index 7ca2c4df0..bcda94a5a 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/base.py @@ -22,6 +22,7 @@ from pymongo import MongoClient from onadata.libs.utils.redis_helper import RedisHelper +from pyxform.xform2json import logger BASE_DIR = os.path.dirname(os.path.dirname(__file__)) ONADATA_DIR = BASE_DIR @@ -148,12 +149,9 @@ 'django.middleware.locale.LocaleMiddleware', 'django.middleware.common.CommonMiddleware', 'onadata.libs.utils.middleware.LocaleMiddlewareWithTweaks', - # BrokenClientMiddleware must come before AuthenticationMiddleware - 'onadata.libs.utils.middleware.BrokenClientMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'corsheaders.middleware.CorsMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'onadata.libs.utils.middleware.HTTPResponseNotAllowedMiddleware', 'readonly.middleware.DatabaseReadOnlyMiddleware', @@ -217,7 +215,7 @@ # needed by guardian ANONYMOUS_USER_ID = -1 -INSTALLED_APPS = ( +INSTALLED_APPS = [ 'django.contrib.contenttypes', # Always put `contenttypes` before `auth`; see # https://code.djangoproject.com/ticket/10827 @@ -250,7 +248,7 @@ 'pure_pagination', 'django_celery_beat', 'django_extensions', -) +] OAUTH2_PROVIDER = { # this is the list of available scopes @@ -307,7 +305,7 @@ CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_WHITELIST = ( - 'dev.ona.io', + 'http://kc.kobo.local', ) USE_THOUSAND_SEPARATOR = True @@ -393,7 +391,7 @@ def skip_suspicious_operations(record): 'class': 'onadata.libs.utils.log.AuditLogHandler', 'formatter': 'verbose', 'model': 'onadata.apps.main.models.audit.AuditLog' - }, + } }, 'loggers': { 'django.request': { @@ -527,7 +525,7 @@ def skip_suspicious_operations(record): from django.db import migrations except ImportError: # Native migrations unavailable; use South instead - INSTALLED_APPS += ('south',) + INSTALLED_APPS.append['south'] SOUTH_MIGRATION_MODULES = { 'taggit': 'taggit.south_migrations', @@ -591,3 +589,7 @@ def skip_suspicious_operations(record): # The maximum size (in bytes) that an upload will be before it gets streamed to the file system FILE_UPLOAD_MAX_MEMORY_SIZE = 10485760 + +# Monkey Patch PyXForm. @ToDo remove after upgrading to v1.1.0 +logger.removeHandler(logging.NullHandler) +logger.addHandler(logging.NullHandler()) diff --git a/onadata/settings/dev.py b/onadata/settings/dev.py index d5558eff6..330a720a8 100644 --- a/onadata/settings/dev.py +++ b/onadata/settings/dev.py @@ -1,11 +1,13 @@ # coding: utf-8 -from .kc_environ import * - +from .prod import * LOGGING['handlers']['console'] = { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'verbose' + 'formatter': 'verbose', + 'stream': sys.stdout, +} +LOGGING['root'] = { + 'handlers': ['console'], + 'level': 'DEBUG' } - -sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) diff --git a/onadata/settings/kc_environ.py b/onadata/settings/prod.py similarity index 97% rename from onadata/settings/kc_environ.py rename to onadata/settings/prod.py index 2a13b9f03..cd5e72f15 100644 --- a/onadata/settings/kc_environ.py +++ b/onadata/settings/prod.py @@ -7,7 +7,7 @@ from celery.signals import after_setup_logger from django.utils.six.moves.urllib.parse import quote_plus -from onadata.settings.common import * +from onadata.settings.base import * def celery_logger_setup_handler(logger, **kwargs): @@ -106,9 +106,8 @@ def celery_logger_setup_handler(logger, **kwargs): KOBOCAT_URL = os.environ.get("KOBOCAT_URL", "http://kc.kobo.local") -TEMPLATE_CONTEXT_PROCESSORS = ( - 'onadata.koboform.context_processors.koboform_integration', -) + TEMPLATE_CONTEXT_PROCESSORS +TEMPLATES[0]['OPTIONS']['context_processors'].insert( + 0, 'onadata.koboform.context_processors.koboform_integration') MIDDLEWARE.insert(0, 'onadata.koboform.redirect_middleware.ConditionalRedirects') @@ -124,7 +123,7 @@ def celery_logger_setup_handler(logger, **kwargs): # print "SECRET_KEY=%s" % SECRET_KEY # print "CSRF_COOKIE_DOMAIN=%s " % CSRF_COOKIE_DOMAIN -# MongoDB - moved here from common.py +# MongoDB - moved here from base.py if MONGO_DATABASE.get('USER') and MONGO_DATABASE.get('PASSWORD'): MONGO_CONNECTION_URL = "mongodb://{user}:{password}@{host}:{port}/{db_name}".\ format( @@ -187,9 +186,7 @@ def celery_logger_setup_handler(logger, **kwargs): except ImportError: print('Please install Raven to enable Sentry logging.') else: - INSTALLED_APPS = INSTALLED_APPS + ( - 'raven.contrib.django.raven_compat', - ) + INSTALLED_APPS.append('raven.contrib.django.raven_compat') RAVEN_CONFIG = { 'dsn': os.environ['RAVEN_DSN'], } diff --git a/onadata/settings/test_environ.py b/onadata/settings/testing.py similarity index 97% rename from onadata/settings/test_environ.py rename to onadata/settings/testing.py index ac5fa5ea3..01306185a 100644 --- a/onadata/settings/test_environ.py +++ b/onadata/settings/testing.py @@ -3,7 +3,7 @@ from mongomock import MongoClient as MockMongoClient -from onadata.settings.common import * +from onadata.settings.base import * DEBUG = os.environ.get('DJANGO_DEBUG', 'True') == 'True' TEMPLATE_DEBUG = os.environ.get('TEMPLATE_DEBUG', 'True') == 'True' @@ -112,9 +112,7 @@ except ImportError: print('Please install Raven to enable Sentry logging.') else: - INSTALLED_APPS = INSTALLED_APPS + ( - 'raven.contrib.django.raven_compat', - ) + INSTALLED_APPS.append('raven.contrib.django.raven_compat') RAVEN_CONFIG = { 'dsn': os.environ['RAVEN_DSN'], } diff --git a/pytest.ini b/pytest.ini index 659936031..8e44c882c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] testpaths = onadata env = - DJANGO_SETTINGS_MODULE=onadata.settings.test_environ + DJANGO_SETTINGS_MODULE=onadata.settings.testing DJANGO_SECRET_KEY=bEW2NgYfhXdRnqkF7J9zPaZkqmTaKN