diff --git a/.github/workflows/ci-cd-workflow.yml b/.github/workflows/ci-cd-workflow.yml index e3a7106..37d7796 100644 --- a/.github/workflows/ci-cd-workflow.yml +++ b/.github/workflows/ci-cd-workflow.yml @@ -25,6 +25,7 @@ jobs: - name: Formatting and linters run: | black --check src scripts tests setup.py --exclude doc/conf.py + black-nb --check notebooks isort --check-only --quiet src scripts tests setup.py doc/conf.py flake8 src scripts tests setup.py doc/conf.py diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index f695271..5c198e7 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -31,7 +31,6 @@ jobs: mkdir -p bin/magicc/magicc-v7.5.3 wget -O "bin/magicc/magicc-v7.5.3.tar.gz" "${{ secrets.MAGICC_LINK_FROM_MAGICC_DOT_ORG }}" tar -xf bin/magicc/magicc-v7.5.3.tar.gz -C bin/magicc/magicc-v7.5.3 - cp -r bin/magicc/magicc-v7.5.3/run/defaults/* bin/magicc/magicc-v7.5.3/run/ MAGICC_RUN_DIR=bin/magicc/magicc-v7.5.3/run/ python scripts/generate-magicc-sr15-input-files.py mkdir -p data/magicc-drawnsets wget -O "data/magicc-drawnsets/magicc-ar6-0fd0f62-f023edb-drawnset.tar.gz" "${{ secrets.MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG }}" diff --git a/.github/workflows/wg3.yaml b/.github/workflows/wg3.yaml index 3690355..76c1204 100644 --- a/.github/workflows/wg3.yaml +++ b/.github/workflows/wg3.yaml @@ -1,6 +1,6 @@ name: Exact reproduction of WG3 database on: - # Uncomment these two lines for debugging, but leave them commented on 'main' + # # Uncomment these two lines for debugging, but leave them commented on 'main' # pull_request: # branches: [ main ] push: @@ -24,13 +24,27 @@ jobs: - name: Install dev dependencies run: | pip install --upgrade pip wheel - pip install -e .[tests] + pip install -e .[tests,notebooks] - - name: Test with pytest + - name: Get infiller database env: SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} - run: pytest tests/nightly/test_wg3_reproduction_ciceroscm.py -m wg3 -r a -vv + INFILLER_DATABASE_NAME: "1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv" + INFILLER_DATABASE_DIR: "data" + INFILLER_HASH: "30fae0530d76cbcb144f134e9ed0051f" + run: | + python -c 'import os.path; from climate_assessment.testing import _get_infiller_download_link, _file_available_or_downloaded; infiller_link = _get_infiller_download_link(os.environ.get("INFILLER_DATABASE_NAME")); _file_available_or_downloaded(os.path.join(os.environ.get("INFILLER_DATABASE_DIR"), os.environ.get("INFILLER_DATABASE_NAME")), os.environ.get("INFILLER_HASH"), infiller_link)' + + - name: Test with pytest + # scenario explorer stuff needed here too so we don't skip the run with + # pytest + env: + SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} + SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} + run: | + pytest notebooks/run-example-ciceroscm.ipynb -r a --nbval-lax --no-cov + pytest tests/nightly/test_wg3_reproduction_ciceroscm.py -m wg3 -r a -vv fair: runs-on: ubuntu-latest @@ -46,13 +60,27 @@ jobs: - name: Install dev dependencies run: | pip install --upgrade pip wheel - pip install -e .[tests] + pip install -e .[tests,notebooks] + + - name: Get infiller database + env: + SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} + SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} + INFILLER_DATABASE_NAME: "1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv" + INFILLER_DATABASE_DIR: "data" + INFILLER_HASH: "30fae0530d76cbcb144f134e9ed0051f" + run: + python -c 'import os.path; from climate_assessment.testing import _get_infiller_download_link, _file_available_or_downloaded; infiller_link = _get_infiller_download_link(os.environ.get("INFILLER_DATABASE_NAME")); _file_available_or_downloaded(os.path.join(os.environ.get("INFILLER_DATABASE_DIR"), os.environ.get("INFILLER_DATABASE_NAME")), os.environ.get("INFILLER_HASH"), infiller_link)' - - name: Test with pytest + - name: Test with pytest + # scenario explorer stuff needed here too so we don't skip the run with + # pytest env: SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} - run: pytest tests/nightly/test_wg3_reproduction_fair.py -m wg3 -r a -vv + run: | + pytest notebooks/run-example-fair.ipynb -r a --nbval-lax --no-cov + pytest tests/nightly/test_wg3_reproduction_fair.py -m wg3 -r a -vv magicc: runs-on: ubuntu-latest @@ -68,7 +96,17 @@ jobs: - name: Install dev dependencies run: | pip install --upgrade pip wheel - pip install -e .[tests] + pip install -e .[tests,notebooks] + + - name: Get infiller database + env: + SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} + SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} + INFILLER_DATABASE_NAME: "1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv" + INFILLER_DATABASE_DIR: "data" + INFILLER_HASH: "30fae0530d76cbcb144f134e9ed0051f" + run: + python -c 'import os.path; from climate_assessment.testing import _get_infiller_download_link, _file_available_or_downloaded; infiller_link = _get_infiller_download_link(os.environ.get("INFILLER_DATABASE_NAME")); _file_available_or_downloaded(os.path.join(os.environ.get("INFILLER_DATABASE_DIR"), os.environ.get("INFILLER_DATABASE_NAME")), os.environ.get("INFILLER_HASH"), infiller_link)' - name: Download MAGICC env: @@ -77,18 +115,22 @@ jobs: mkdir -p bin/magicc/magicc-v7.5.3 wget -O "bin/magicc/magicc-v7.5.3.tar.gz" "${{ secrets.MAGICC_LINK_FROM_MAGICC_DOT_ORG }}" tar -xf bin/magicc/magicc-v7.5.3.tar.gz -C bin/magicc/magicc-v7.5.3 - cp -r bin/magicc/magicc-v7.5.3/run/defaults/* bin/magicc/magicc-v7.5.3/run/ - MAGICC_RUN_DIR=bin/magicc/magicc-v7.5.3/run/ python scripts/generate-magicc-sr15-input-files.py mkdir -p data/magicc-drawnsets wget -O "data/magicc-drawnsets/magicc-ar6-0fd0f62-f023edb-drawnset.tar.gz" "${{ secrets.MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG }}" tar -xf "data/magicc-drawnsets/magicc-ar6-0fd0f62-f023edb-drawnset.tar.gz" -C data/magicc-drawnsets - name: Test with pytest + # scenario explorer stuff needed here too so we don't skip the run with + # pytest env: + SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} + SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} MAGICC_EXECUTABLE_7: /home/runner/work/climate-assessment/climate-assessment/bin/magicc/magicc-v7.5.3/bin/magicc + MAGICC_LINK_FROM_MAGICC_DOT_ORG: ${{ secrets.MAGICC_LINK_FROM_MAGICC_DOT_ORG }} + MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG: ${{ secrets.MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG }} MAGICC_WORKER_NUMBER: 4 MAGICC_WORKER_ROOT_DIR: /tmp MAGICC_PROBABILISTIC_FILE: data/magicc-drawnsets/0fd0f62-derived-metrics-id-f023edb-drawnset.json - SCENARIO_EXPLORER_USER: ${{ secrets.SCENARIO_EXPLORER_USER }} - SCENARIO_EXPLORER_PASSWORD: ${{ secrets.SCENARIO_EXPLORER_PASSWORD }} - run: pytest tests/nightly/test_wg3_reproduction_magicc.py -m wg3 -r a -vv + run: | + pytest notebooks/run-example-magicc.ipynb -r a --nbval-lax --no-cov + pytest tests/nightly/test_wg3_reproduction_magicc.py -m wg3 -r a -vv diff --git a/.gitignore b/.gitignore index 50dc8de..d8bf187 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv tests/test-data/fair-1.6.2-wg3-params*.json tests/test-data/rcmip-emissions-annual-means-v5-1-0.csv +magicc-files scripts/*cluster* 20211003* diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 60cea12..d6d1420 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,7 @@ master Added ~~~~~ +- (`#6 `_) Added example run notebooks and tests thereof - (`#1 `_) Added :func:`climate_assessment.cli.run_workflow` v0.1.0 - 2022-06-08 diff --git a/Makefile b/Makefile index c4c14ae..8b775db 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ VENV_DIR ?= ./venv DATA_DIR ?= ./data SCRIPTS_DIR ?= ./scripts +NOTEBOOKS_DIR ?= ./notebooks FILES_TO_FORMAT_PYTHON=scripts src tests setup.py doc/conf.py @@ -40,6 +41,15 @@ checks: $(VENV_DIR) ## run all the checks echo "\n\n=== flake8 ==="; $(VENV_DIR)/bin/flake8 $(FILES_TO_FORMAT_PYTHON) || echo "--- flake8 failed ---" >&2; \ echo +.PHONY: format-notebooks +format-notebooks: $(VENV_DIR) ## format the notebooks + @status=$$(git status --porcelain $(NOTEBOOKS_DIR)); \ + if test ${FORCE} || test "x$${status}" = x; then \ + $(VENV_DIR)/bin/black-nb $(NOTEBOOKS_DIR); \ + else \ + echo Not trying any formatting. Working directory is dirty ... >&2; \ + fi; + .PHONY: format format: ## re-format files make isort diff --git a/notebooks/run-example-ciceroscm.ipynb b/notebooks/run-example-ciceroscm.ipynb new file mode 100644 index 0000000..f8a421a --- /dev/null +++ b/notebooks/run-example-ciceroscm.ipynb @@ -0,0 +1,371 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "529d03cc", + "metadata": {}, + "source": [ + "# Example run: CICERO-SCM\n", + "\n", + "Here we demonstrate how to run with CICERO-SCM. We run with the IPs from WG3 and demonstrate that the results in the database can be reproduced. However, other input data could be used for custom scenario runs.\n", + "\n", + "For more information, see the docs associated with the CICERO-SCM model." + ] + }, + { + "cell_type": "markdown", + "id": "98357272", + "metadata": {}, + "source": [ + "## Setup logging\n", + "\n", + "Pyam does its own logging stuff, so we have to configure logging before that import." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59eecbfd", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "# Increase the level to reduce the number of log messages\n", + "LOGGING_LEVEL = logging.INFO\n", + "\n", + "LOGGER = logging.getLogger(\"pipeline\")\n", + "LOGGER.setLevel(LOGGING_LEVEL)\n", + "# have to set root logger too to get messages to come through\n", + "logging.getLogger().setLevel(LOGGING_LEVEL)\n", + "\n", + "logFormatter = logging.Formatter(\n", + " \"%(asctime)s %(name)s %(threadName)s - %(levelname)s: %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\",\n", + ")\n", + "stdoutHandler = logging.StreamHandler()\n", + "stdoutHandler.setFormatter(logFormatter)\n", + "\n", + "logging.getLogger().addHandler(stdoutHandler)" + ] + }, + { + "cell_type": "markdown", + "id": "c61735ad", + "metadata": {}, + "source": [ + "## Other imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acaff9ea", + "metadata": {}, + "outputs": [], + "source": [ + "import os.path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import pandas.testing as pdt\n", + "import pooch\n", + "import pyam\n", + "import tempfile\n", + "\n", + "from climate_assessment.cli import run_workflow" + ] + }, + { + "cell_type": "markdown", + "id": "a596e20e", + "metadata": {}, + "source": [ + "## Configuration of input data" + ] + }, + { + "cell_type": "markdown", + "id": "a8bdd363", + "metadata": {}, + "source": [ + "### Set model input paths\n", + "\n", + "Here we tell the pipeline where to look for CICERO-SCM and where to create parallel workers on disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74129df4", + "metadata": {}, + "outputs": [], + "source": [ + "# How many cores should be used when running CICERO-SCM in parallel?\n", + "os.environ[\"CICEROSCM_WORKER_NUMBER\"] = \"4\"\n", + "\n", + "# Where should Cicero-SCM workers be located on the filesystem\n", + "os.environ[\"CICEROSCM_WORKER_ROOT_DIR\"] = tempfile.gettempdir()" + ] + }, + { + "cell_type": "markdown", + "id": "a22bdbac", + "metadata": {}, + "source": [ + "### Set general input arguments and options to the climate assessment workflow\n", + "\n", + "The values we've set below will let you run CICERO-SCM and reproduce the AR6 WG3 results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1079bd63", + "metadata": {}, + "outputs": [], + "source": [ + "model = \"ciceroscm\"\n", + "model_version = \"v2019vCH4\"\n", + "probabilistic_file = os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"cicero\",\n", + " \"subset_cscm_configfile.json\",\n", + ")\n", + "\n", + "# Use fewer (e.g. 10) if you just want to do a test run but note that this breaks\n", + "# the stats of the probabilistic ensemble\n", + "num_cfgs = 600\n", + "# Set to True if you're not using the full CICERO-SCM ensemble\n", + "test_run = False\n", + "# How many scenarios do you want to run in one go?\n", + "scenario_batch_size = 20\n", + "\n", + "# Where should the output be saved?\n", + "outdir = os.path.join(\"..\", \"data\", \"output-ciceroscm-example-notebook\")" + ] + }, + { + "cell_type": "markdown", + "id": "cbbed3b6", + "metadata": {}, + "source": [ + "### Choose input emissions pathway file\n", + "\n", + "By default, we use 2 of the AR6 IPs. You could use a different file (formatted the same way) if you wish." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41d0741c", + "metadata": {}, + "outputs": [], + "source": [ + "EMISSIONS_DATA_DIR = os.path.join(\"..\", \"tests\", \"test-data\")\n", + "EMISSIONS_INPUT_FILE = \"ar6_IPs_emissions.csv\"\n", + "\n", + "input_emissions_file = os.path.join(EMISSIONS_DATA_DIR, EMISSIONS_INPUT_FILE)" + ] + }, + { + "cell_type": "markdown", + "id": "f92038e8", + "metadata": {}, + "source": [ + "### Choose infiller database file\n", + "\n", + "We run using the infiller database that was used in CMIP6. As a result of the licensing of the scenario data, this file has to be downloaded by hand (see documentation under \"Installation\", section \"Infiller database\"). Make sure that the variable `infilling_database_file` points to where you saved this file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b27e25d", + "metadata": {}, + "outputs": [], + "source": [ + "infilling_database_file = os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f7e8686c", + "metadata": {}, + "source": [ + "## Run the climate assessment workflow" + ] + }, + { + "cell_type": "markdown", + "id": "b1302596", + "metadata": {}, + "source": [ + "*N.B. the log with information and some warnings will be quite long - but that is nothing to worry about!*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74a44671", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "run_workflow(\n", + " input_emissions_file,\n", + " outdir,\n", + " model=model,\n", + " model_version=model_version,\n", + " probabilistic_file=probabilistic_file,\n", + " num_cfgs=num_cfgs,\n", + " infilling_database=infilling_database_file,\n", + " scenario_batch_size=scenario_batch_size,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "35346bc7", + "metadata": {}, + "source": [ + "### Load results\n", + "\n", + "*N.B The filename will likely have changed if you have run your own scenarios.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e7e20bf", + "metadata": {}, + "outputs": [], + "source": [ + "output = pyam.IamDataFrame(os.path.join(outdir, \"ar6_IPs_emissions_alloutput.xlsx\"))\n", + "output" + ] + }, + { + "cell_type": "markdown", + "id": "ce7ed444", + "metadata": {}, + "source": [ + "### Compare with database results\n", + "\n", + "These would normally need to be downloaded, but we include a set in the repository for testing. Here we check that we have reproduced the database results. Obviously, this should be skipped if you have run a custom scenario." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f25a77c", + "metadata": {}, + "outputs": [], + "source": [ + "expected_output_file = os.path.join(\n", + " \"..\", \"tests\", \"test-data\", \"expected-output-wg3/two_ips_climate_cicero.xlsx\"\n", + ")\n", + "\n", + "expected_db_output = pyam.IamDataFrame(expected_output_file).timeseries()\n", + "\n", + "# The database does not necessarily include all the outputs we have\n", + "test_output = output.timeseries().loc[expected_db_output.index, :]\n", + "\n", + "# Check that we reproduce values\n", + "pdt.assert_frame_equal(\n", + " test_output,\n", + " expected_db_output,\n", + " rtol=1e-3,\n", + " atol=1e-3,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cca3f359", + "metadata": {}, + "source": [ + "### Some basic exploration" + ] + }, + { + "cell_type": "markdown", + "id": "d617ae5e", + "metadata": {}, + "source": [ + "Look at the scenario categories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82238f8b", + "metadata": {}, + "outputs": [], + "source": [ + "output.meta[\"Category\"]" + ] + }, + { + "cell_type": "markdown", + "id": "287f742c", + "metadata": {}, + "source": [ + "Make a plot of median warming." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c381213b", + "metadata": {}, + "outputs": [], + "source": [ + "ax = output.filter(variable=\"*|Surface Temperature (GSAT)|*|50.0th Percentile\").plot(\n", + " color=\"scenario\"\n", + ")\n", + "plt.title(\"Global warming above the 1850-1900 mean\")\n", + "ax.set_xlim([1995, 2100])" + ] + }, + { + "cell_type": "markdown", + "id": "3b9b79e7", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "That's it! You just ran a a full climate assessment workflow going from emissions to temperature (and more) using the functionality from the climate-assessment package, and then visualised the results. \n", + "\n", + "It is also possible to run from the command line, and build more elaborate workflows. For that, please see the extended documentation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/run-example-custom.ipynb b/notebooks/run-example-custom.ipynb new file mode 100644 index 0000000..8c6d528 --- /dev/null +++ b/notebooks/run-example-custom.ipynb @@ -0,0 +1,499 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "529d03cc", + "metadata": {}, + "source": [ + "# Example run: custom scenario\n", + "\n", + "Here we demonstrate how to run with a custom scenario. In order to run this notebook, you will need to add in the code required to run your climate model of choice. See the other example notebooks for how to do this." + ] + }, + { + "cell_type": "markdown", + "id": "98357272", + "metadata": {}, + "source": [ + "## Setup logging\n", + "\n", + "Pyam does its own logging stuff, so we have to configure logging before that import." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59eecbfd", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "# Increase the level to reduce the number of log messages\n", + "LOGGING_LEVEL = logging.INFO\n", + "\n", + "LOGGER = logging.getLogger(\"pipeline\")\n", + "LOGGER.setLevel(LOGGING_LEVEL)\n", + "# have to set root logger too to get messages to come through\n", + "logging.getLogger().setLevel(LOGGING_LEVEL)\n", + "\n", + "logFormatter = logging.Formatter(\n", + " \"%(asctime)s %(name)s %(threadName)s - %(levelname)s: %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\",\n", + ")\n", + "stdoutHandler = logging.StreamHandler()\n", + "stdoutHandler.setFormatter(logFormatter)\n", + "\n", + "logging.getLogger().addHandler(stdoutHandler)" + ] + }, + { + "cell_type": "markdown", + "id": "c61735ad", + "metadata": {}, + "source": [ + "## Other imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acaff9ea", + "metadata": {}, + "outputs": [], + "source": [ + "import os.path\n", + "import tempfile\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pandas.testing as pdt\n", + "import pooch\n", + "import pyam\n", + "\n", + "from climate_assessment.cli import run_workflow" + ] + }, + { + "cell_type": "markdown", + "id": "a596e20e", + "metadata": {}, + "source": [ + "## Configuration of input data" + ] + }, + { + "cell_type": "markdown", + "id": "a22bdbac", + "metadata": {}, + "source": [ + "### Choice of climate model\n", + "\n", + "See the individual run example notebooks for information about how to get setup with each model." + ] + }, + { + "cell_type": "markdown", + "id": "20ebd766", + "metadata": {}, + "source": [ + "#### CICERO-SCM" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92ae35f4", + "metadata": {}, + "outputs": [], + "source": [ + "climate_model_kwargs = dict(\n", + " model=\"ciceroscm\",\n", + " model_version=\"v2019vCH4\",\n", + " probabilistic_file=os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"cicero\",\n", + " \"subset_cscm_configfile.json\",\n", + " ),\n", + " # num_cfgs=600,\n", + " num_cfgs=1,\n", + " test_run=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c54d98f0", + "metadata": {}, + "source": [ + "#### MAGICC" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ccf9ba8f", + "metadata": {}, + "outputs": [], + "source": [ + "# Where is the binary?\n", + "os.environ[\"MAGICC_EXECUTABLE_7\"] = os.path.join(\n", + " \"path\", \"to\", \"magicc-v7.5.3\", \"bin\", \"magicc\"\n", + ")\n", + "\n", + "# How many MAGICC workers can run in parallel?\n", + "os.environ[\"MAGICC_WORKER_NUMBER\"] = \"4\"\n", + "\n", + "# Where should the MAGICC workers be located on the filesystem (you need about\n", + "# 500Mb space per worker at the moment, they're removed after use)\n", + "os.environ[\"MAGICC_WORKER_ROOT_DIR\"] = tempfile.gettempdir()\n", + "\n", + "magicc_data_dir = os.path.join(\"..\", \"data\", \"magicc\")\n", + "\n", + "climate_model_kwargs = dict(\n", + " model=\"magicc\",\n", + " model_version=\"v7.5.3\",\n", + " probabilistic_file=os.path.join(\n", + " \"path\",\n", + " \"to\",\n", + " \"magicc-ar6-0fd0f62-f023edb-drawnset\",\n", + " \"0fd0f62-derived-metrics-id-f023edb-drawnset.json\",\n", + " ),\n", + " num_cfgs=600,\n", + " test_run=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "5c68abab", + "metadata": {}, + "source": [ + "#### FaIR" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9965f5cc", + "metadata": {}, + "outputs": [], + "source": [ + "fair_data_dir = os.path.join(\"..\", \"data\", \"fair\")\n", + "\n", + "climate_model_kwargs = dict(\n", + " model=\"fair\",\n", + " model_version=\"1.6.2\",\n", + " probabilistic_file=os.path.join(fair_data_dir, \"fair-1.6.2-wg3-params-slim.json\"),\n", + " fair_extra_config=os.path.join(fair_data_dir, \"fair-1.6.2-wg3-params-common.json\"),\n", + " num_cfgs=2237,\n", + " test_run=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e10c3910", + "metadata": {}, + "source": [ + "### Other config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b0aa2ec", + "metadata": {}, + "outputs": [], + "source": [ + "# Where should the output be saved?\n", + "outdir = os.path.join(\"..\", \"data\", \"output-custom-example-notebook\")\n", + "\n", + "# How many scenarios do you want to run in one go?\n", + "scenario_batch_size = 20" + ] + }, + { + "cell_type": "markdown", + "id": "87165dab", + "metadata": {}, + "source": [ + "### Create input emissions\n", + "\n", + "This could be taken from a pre-configured file. Here we create the file on the fly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed3cfaf3", + "metadata": {}, + "outputs": [], + "source": [ + "idx = pd.MultiIndex.from_arrays(\n", + " [\n", + " [\"stylised\"] * 4,\n", + " [\"example\"] * 4,\n", + " [\n", + " \"Emissions|CO2|Energy and Industrial Processes\",\n", + " \"Emissions|CO2|AFOLU\",\n", + " \"Emissions|CH4\",\n", + " \"Emissions|N2O\",\n", + " ],\n", + " [\"Mt CO2/yr\", \"Mt CO2/yr\", \"Mt CH4/yr\", \"kt N2O/yr\"],\n", + " [\"World\"] * 4,\n", + " ],\n", + " names=[\"model\", \"scenario\", \"variable\", \"unit\", \"region\"],\n", + ")\n", + "years = np.arange(2010, 2100 + 1, 5)\n", + "\n", + "\n", + "def sigmoid(k, x, x0):\n", + " return 1 / (1 + np.exp(-k * (x - x0)))\n", + "\n", + "\n", + "base = pd.DataFrame(\n", + " data=[\n", + " 35000 * sigmoid(-1 / 8, years, 2050),\n", + " 4500 * sigmoid(-1 / 8, years, 2050),\n", + " 375 / 2 * (1 + sigmoid(-1 / 8, years, 2040)),\n", + " 11500 * 2 * sigmoid(-1 / 300, years, 2015),\n", + " ],\n", + " columns=years,\n", + " index=idx,\n", + ")\n", + "shuffle = base.reset_index()\n", + "shuffle[\"scenario\"] = \"example_shuffle\"\n", + "shuffle = shuffle.set_index(base.index.names)\n", + "shuffle.loc[\n", + " shuffle.index.get_level_values(\"variable\").isin(\n", + " [\"Emissions|CO2|Energy and Industrial Processes\"]\n", + " ),\n", + " :,\n", + "] = 35000 * sigmoid(-1 / 5, years, 2040)\n", + "\n", + "inp = pyam.IamDataFrame(pd.concat([base, shuffle]))\n", + "\n", + "ax = inp.filter(variable=\"Emissions|CH4\").plot()\n", + "ax.set_ylim(ymin=0)\n", + "ax = inp.filter(variable=\"Emissions|N2O\").plot()\n", + "ax.set_ylim(ymin=0)\n", + "ax = inp.filter(variable=\"Emissions|CO2|Energy and Industrial Processes\").plot()\n", + "ax.set_ylim(ymin=0)\n", + "ax = inp.filter(variable=\"Emissions|CO2|AFOLU\").plot()\n", + "ax.set_ylim(ymin=0)\n", + "\n", + "inp" + ] + }, + { + "cell_type": "markdown", + "id": "fe1f6aac", + "metadata": {}, + "source": [ + "Write file to disk (yes, unfortunately our API currently expects a file on disk, PRs welcome)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af15002f", + "metadata": {}, + "outputs": [], + "source": [ + "input_emissions_file = \"input-emissions.csv\"\n", + "inp.to_csv(input_emissions_file)" + ] + }, + { + "cell_type": "markdown", + "id": "f92038e8", + "metadata": {}, + "source": [ + "### Choose infiller database file\n", + "\n", + "We run using the infiller database that was used in CMIP6. As a result of the licensing of the scenario data, this file has to be downloaded by hand (see documentation under \"Installation\", section \"Infiller database\"). Make sure that the variable `infilling_database_file` points to where you saved this file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b27e25d", + "metadata": {}, + "outputs": [], + "source": [ + "infilling_database_file = os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f7e8686c", + "metadata": {}, + "source": [ + "## Run the climate assessment workflow" + ] + }, + { + "cell_type": "markdown", + "id": "b1302596", + "metadata": {}, + "source": [ + "*N.B. the log with information and some warnings will be quite long - but that is nothing to worry about!*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74a44671", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "run_workflow(\n", + " input_emissions_file,\n", + " outdir,\n", + " infilling_database=infilling_database_file,\n", + " scenario_batch_size=scenario_batch_size,\n", + " **climate_model_kwargs,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "35346bc7", + "metadata": {}, + "source": [ + "### Load results\n", + "\n", + "*N.B The filename will likely have changed if you have run your own scenarios.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e7e20bf", + "metadata": {}, + "outputs": [], + "source": [ + "output = pyam.IamDataFrame(os.path.join(outdir, \"input-emissions_alloutput.xlsx\"))\n", + "output" + ] + }, + { + "cell_type": "markdown", + "id": "cca3f359", + "metadata": {}, + "source": [ + "### Some basic exploration" + ] + }, + { + "cell_type": "markdown", + "id": "d617ae5e", + "metadata": {}, + "source": [ + "Look at the scenario categories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82238f8b", + "metadata": {}, + "outputs": [], + "source": [ + "output.meta[\"Category\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "997db63f", + "metadata": {}, + "outputs": [], + "source": [ + "# A hacky way to examine harmonisation\n", + "v = \"Emissions|CO2|Energy and Industrial Processes\"\n", + "v = \"Emissions|CO2|AFOLU\"\n", + "# v = \"Emissions|CH4\"\n", + "\n", + "ax = output.filter(variable=f\"*Infilled|{v}\").plot(color=\"scenario\")\n", + "inp.filter(variable=v).plot(color=\"scenario\", ax=ax)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b76b6f1d", + "metadata": {}, + "outputs": [], + "source": [ + "ax = output.filter(variable=f\"*Infilled|Emissions|Sulfur\").plot(color=\"scenario\")\n", + "ax.set_ylim(ymin=0)" + ] + }, + { + "cell_type": "markdown", + "id": "287f742c", + "metadata": {}, + "source": [ + "Make a plot of median warming." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c381213b", + "metadata": {}, + "outputs": [], + "source": [ + "ax = output.filter(variable=\"*|Surface Temperature (GSAT)|*|50.0th Percentile\").plot(\n", + " color=\"scenario\"\n", + ")\n", + "plt.title(\"Global warming above the 1850-1900 mean\")\n", + "ax.set_xlim([1995, 2100])" + ] + }, + { + "cell_type": "markdown", + "id": "3b9b79e7", + "metadata": {}, + "source": [ + "## Conclusion\n", + "That's it! You just ran a a full climate assessment workflow going from emissions to temperature (and more) using the functionality from the climate-assessment package, and then visualised the results. \n", + "\n", + "Naturally, similar workflows can be constructed using CICERO-SCM (on Linux) and MAGICC (on Windows and Linux)!\n", + "\n", + "It is also possible to run from the command line, and build more elaborate workflows. For that, please see the extended documentation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/run-example-fair.ipynb b/notebooks/run-example-fair.ipynb index efd38ca..b846eb5 100644 --- a/notebooks/run-example-fair.ipynb +++ b/notebooks/run-example-fair.ipynb @@ -1,38 +1,78 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "529d03cc", + "metadata": {}, + "source": [ + "# Example run: FaIR\n", + "\n", + "Here we demonstrate how to run with FaIR. We run with the IPs from WG3 and demonstrate that the results in the database can be reproduced. However, other input data could be used for custom scenario runs.\n", + "\n", + "For more information, see the docs associated with the FaIR model." + ] + }, + { + "cell_type": "markdown", + "id": "98357272", + "metadata": {}, + "source": [ + "## Setup logging\n", + "\n", + "Pyam does its own logging stuff, so we have to configure logging before that import." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "acaff9ea", + "id": "59eecbfd", "metadata": {}, "outputs": [], "source": [ "import logging\n", - "import os.path\n", - "import pooch\n", "\n", - "import matplotlib.pyplot as plt\n", - "import pyam\n", + "# Increase the level to reduce the number of log messages\n", + "LOGGING_LEVEL = logging.INFO\n", + "\n", + "LOGGER = logging.getLogger(\"pipeline\")\n", + "LOGGER.setLevel(LOGGING_LEVEL)\n", + "# have to set root logger too to get messages to come through\n", + "logging.getLogger().setLevel(LOGGING_LEVEL)\n", "\n", - "from climate_assessment.cli import (\n", - " _get_key_string_and_log_outdir,\n", - " _harmonize_and_infill,\n", - " _load_emissions_convert_to_basic,\n", - " _setup_logging,\n", + "logFormatter = logging.Formatter(\n", + " \"%(asctime)s %(name)s %(threadName)s - %(levelname)s: %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\",\n", ")\n", - "from climate_assessment.climate import climate_assessment\n", - "from climate_assessment.postprocess import do_postprocess" + "stdoutHandler = logging.StreamHandler()\n", + "stdoutHandler.setFormatter(logFormatter)\n", + "\n", + "logging.getLogger().addHandler(stdoutHandler)" + ] + }, + { + "cell_type": "markdown", + "id": "c61735ad", + "metadata": {}, + "source": [ + "## Other imports" ] }, { "cell_type": "code", "execution_count": null, - "id": "59eecbfd", + "id": "acaff9ea", "metadata": {}, "outputs": [], "source": [ - "LOGGER = logging.getLogger(\"pipeline\")\n", - "_setup_logging(LOGGER)" + "import os.path\n", + "import pooch\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import pandas.testing as pdt\n", + "import pyam\n", + "\n", + "from climate_assessment.cli import run_workflow" ] }, { @@ -48,7 +88,7 @@ "id": "2f8c995d", "metadata": {}, "source": [ - "### Climate emulator (JSON) configuration data download (FaIR)" + "### Download configuration" ] }, { @@ -56,13 +96,9 @@ "id": "2345c0e4", "metadata": {}, "source": [ - "Our FaIR setup uses two files, because it reduces the size of the configuration input compared to using only one file.\n", - "\n", - "You can download these files manually from \"https://zenodo.org/record/6601980\", or you can download it interactively using `pooch`, which we'll showcase here. \n", - "\n", - "1. First, we specify where we want to put the downloaded cached data (`fair_config_data_dir`)\n", + "The FaIR setup uses two files, because it reduces the size of the configuration input compared to using only one file.\n", "\n", - "2. Then, we specify what config files we want, and download them using `pooch.retrieve`" + "You can download these files manually from \"https://zenodo.org/record/6601980\", or you can download it interactively using `pooch` as demonstrated here. " ] }, { @@ -72,7 +108,8 @@ "metadata": {}, "outputs": [], "source": [ - "fair_config_data_dir = os.path.join(\"..\", \"data\", \"fair\")" + "fair_data_dir = os.path.join(\"..\", \"data\", \"fair\")\n", + "os.makedirs(fair_data_dir, exist_ok=True)" ] }, { @@ -91,7 +128,7 @@ "id": "922e25a2", "metadata": {}, "source": [ - "**N.B.** you only need to run the two cells below this the first time, after that you can also skip and run without internet connection." + "**N.B.** you only need to run the two cells below this the first time, after that you can also skip and run without internet connection. Of course, pooch caches all the files so running them again won't cost you any time." ] }, { @@ -101,15 +138,17 @@ "metadata": {}, "outputs": [], "source": [ - "fair_slim_url = \"https://zenodo.org/record/6601980/files/fair-1.6.2-wg3-params-slim.json?download=1\"\n", + "fair_slim_url = (\n", + " \"https://zenodo.org/record/6601980/files/fair-1.6.2-wg3-params-slim.json?download=1\"\n", + ")\n", "fair_slim_hash = \"c071ca619c0ae37a6abdeb79c0cece7b\"\n", - " \n", + "\n", "pooch.retrieve(\n", - " url=fair_slim_url,\n", - " known_hash=f\"md5:{fair_slim_hash}\",\n", - " path=fair_config_data_dir,\n", - " fname=fair_slim_filename,\n", - " )" + " url=fair_slim_url,\n", + " known_hash=f\"md5:{fair_slim_hash}\",\n", + " path=fair_data_dir,\n", + " fname=fair_slim_filename,\n", + ")" ] }, { @@ -123,11 +162,11 @@ "fair_common_hash = \"42ccaffcd3dea88edfca77da0cd5789b\"\n", "\n", "pooch.retrieve(\n", - " url=fair_common_url,\n", - " known_hash=f\"md5:{fair_common_hash}\",\n", - " path=fair_config_data_dir,\n", - " fname=fair_common_filename,\n", - " )" + " url=fair_common_url,\n", + " known_hash=f\"md5:{fair_common_hash}\",\n", + " path=fair_data_dir,\n", + " fname=fair_common_filename,\n", + ")" ] }, { @@ -135,7 +174,9 @@ "id": "a22bdbac", "metadata": {}, "source": [ - "### Set general input arguments and options to the climate assessment workflow" + "### Set general input arguments and options to the climate assessment workflow\n", + "\n", + "The values we've set below will let you run FaIR and reproduce the AR6 WG3 results." ] }, { @@ -147,16 +188,19 @@ "source": [ "model = \"fair\"\n", "model_version = \"1.6.2\"\n", - "fair_extra_config = os.path.join(fair_config_data_dir, fair_common_filename)\n", - "probabilistic_file = os.path.join(fair_config_data_dir, fair_slim_filename)\n", - "num_cfgs = 10 # only 10 configurations (out of 2237 for FaIR) for this example\n", - "test_run = True # this option must be True because we don't run with all 2237 configurations and thus cannot be certain the output temperature data will be correct\n", - "scenario_batch_size = 20 # how many scenarios do you want to run in one go?\n", - "prefix = \"AR6 climate diagnostics\" # string prefix \n", - "inputcheck = True # make sure the input data is in the right format\n", + "fair_extra_config = os.path.join(fair_data_dir, fair_common_filename)\n", + "probabilistic_file = os.path.join(fair_data_dir, fair_slim_filename)\n", + "\n", + "# Use fewer (e.g. 10) if you just want to do a test run but note that this breaks\n", + "# the stats of the probabilistic ensemble\n", + "num_cfgs = 2237\n", + "# Set to True if you're not using the full FaIR ensemble\n", + "test_run = False\n", + "# How many scenarios do you want to run in one go?\n", + "scenario_batch_size = 20\n", "\n", - "os.environ[\"ROOT_DIR\"] = os.path.join(\"..\") # set root directory\n", - "outdir = \"output\" # what folder will the climate output be found after the run?\n" + "# Where should the output be saved?\n", + "outdir = os.path.join(\"..\", \"data\", \"output-fair-example-notebook\")" ] }, { @@ -164,7 +208,9 @@ "id": "cbbed3b6", "metadata": {}, "source": [ - "### Choose input emissions pathway file" + "### Choose input emissions pathway file\n", + "\n", + "By default, we use 2 of the AR6 IPs. You could use a different file (formatted the same way) if you wish." ] }, { @@ -174,13 +220,10 @@ "metadata": {}, "outputs": [], "source": [ - "EMISSIONS_DATA_DIR = os.path.join(\n", - " \"..\", \"tests\", \"test-data\"\n", - ") \n", + "EMISSIONS_DATA_DIR = os.path.join(\"..\", \"tests\", \"test-data\")\n", "EMISSIONS_INPUT_FILE = \"ar6_IPs_emissions.csv\"\n", - "input_emissions_file = os.path.join(\n", - " EMISSIONS_DATA_DIR, EMISSIONS_INPUT_FILE\n", - ")" + "\n", + "input_emissions_file = os.path.join(EMISSIONS_DATA_DIR, EMISSIONS_INPUT_FILE)" ] }, { @@ -188,7 +231,9 @@ "id": "f92038e8", "metadata": {}, "source": [ - "### Choose infiller database file" + "### Choose infiller database file\n", + "\n", + "We run using the infiller database that was used in CMIP6. As a result of the licensing of the scenario data, this file has to be downloaded by hand (see documentation under \"Installation\", section \"Infiller database\"). Make sure that the variable `infilling_database_file` points to where you saved this file." ] }, { @@ -198,30 +243,10 @@ "metadata": {}, "outputs": [], "source": [ - "infilling_database = os.path.join(\n", - " \"..\", \"src\", \"climate_assessment\", \"infilling\", \"cmip6-ssps-workflow-emissions.csv\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "0fc44196", - "metadata": {}, - "source": [ - "> **_NOTE:_** The cell above selects the \"cmip6-ssps\" infiller database, which is directly provided with the climate-assessment package. However, to reproduce the AR6 temperature projections and for most applications, using the AR6 infiller database is strongly recommended (see documentation under \"Installation\", section \"Infiller database\"). " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "029018d1", - "metadata": {}, - "outputs": [], - "source": [ - "key_string = _get_key_string_and_log_outdir(\n", - " os.path.join(EMISSIONS_DATA_DIR, EMISSIONS_INPUT_FILE),\n", - " outdir,\n", - " LOGGER,\n", + "infilling_database_file = os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv\",\n", ")" ] }, @@ -233,201 +258,122 @@ "## Run the climate assessment workflow" ] }, - { - "cell_type": "markdown", - "id": "7af6bebb", - "metadata": {}, - "source": [ - "### Load input emissions file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca1c5908", - "metadata": {}, - "outputs": [], - "source": [ - "input_df = _load_emissions_convert_to_basic(input_emissions_file, LOGGER)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "327f1ac4", - "metadata": {}, - "outputs": [], - "source": [ - "input_df.timeseries()" - ] - }, - { - "cell_type": "markdown", - "id": "80dbb908", - "metadata": {}, - "source": [ - "### Harmonize and infill the scenarios" - ] - }, { "cell_type": "markdown", "id": "b1302596", "metadata": {}, "source": [ - "*N.B. watch out, the log with information and some warnings will be quite long - but that is nothing to worry about!*" + "*N.B. the log with information and some warnings will be quite long - but that is nothing to worry about!*" ] }, { "cell_type": "code", "execution_count": null, - "id": "f965d0af", + "id": "74a44671", "metadata": {}, "outputs": [], "source": [ - "_harmonize_and_infill(\n", - " input_df,\n", - " inputcheck,\n", - " key_string,\n", + "run_workflow(\n", + " input_emissions_file,\n", " outdir,\n", - " infilling_database,\n", - " harmonize=True,\n", - " prefix=prefix,\n", - " harmonization_instance=\"ar6\",\n", + " model=model,\n", + " model_version=model_version,\n", + " probabilistic_file=probabilistic_file,\n", + " fair_extra_config=fair_extra_config,\n", + " num_cfgs=num_cfgs,\n", + " infilling_database=infilling_database_file,\n", + " scenario_batch_size=scenario_batch_size,\n", ")" ] }, { "cell_type": "markdown", - "id": "1f3c5bfb", - "metadata": {}, - "source": [ - "### Load back in emissions data after infilling was successful" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51082a8f", + "id": "35346bc7", "metadata": {}, - "outputs": [], "source": [ - "# read in infilled database\n", - "infilled_emissions = os.path.join(\n", - " outdir, \"{}_harmonized_infilled.csv\".format(key_string)\n", - ")\n", - "LOGGER.info(\"Reading in infilled scenarios from: %s\", infilled_emissions)\n", - "df_infilled = pyam.IamDataFrame(infilled_emissions)" + "### Load results\n", + "\n", + "*N.B The filename will likely have changed if you have run your own scenarios.*" ] }, { "cell_type": "code", "execution_count": null, - "id": "ed1e9638", + "id": "4e7e20bf", "metadata": {}, "outputs": [], "source": [ - "df_infilled.timeseries()" - ] - }, - { - "cell_type": "markdown", - "id": "bf3ea7a0", - "metadata": {}, - "source": [ - "### Run climate emulator FaIR" + "output = pyam.IamDataFrame(os.path.join(outdir, \"ar6_IPs_emissions_alloutput.xlsx\"))\n", + "output" ] }, { "cell_type": "markdown", - "id": "8f574b5d", + "id": "ce7ed444", "metadata": {}, "source": [ - "*N.B. prepare for another info log*" + "### Compare with database results\n", + "\n", + "These would normally need to be downloaded, but we include a set in the repository for testing. Here we check that we have reproduced the database results. Obviously, this should be skipped if you have run a custom scenario." ] }, { "cell_type": "code", "execution_count": null, - "id": "87eee27e", + "id": "9f25a77c", "metadata": {}, "outputs": [], "source": [ - "df_climate = climate_assessment(\n", - " df_infilled,\n", - " key_string,\n", - " outdir,\n", - " fair_extra_config=fair_extra_config,\n", - " model=model,\n", - " model_version=model_version,\n", - " probabilistic_file=probabilistic_file,\n", - " num_cfgs=num_cfgs,\n", - " historical_warming=0.85,\n", - " historical_warming_reference_period=\"1850-1900\",\n", - " historical_warming_evaluation_period=\"1995-2014\",\n", - " test_run=test_run,\n", - " scenario_batch_size=scenario_batch_size,\n", - " save_raw_output=False,\n", - " co2_and_non_co2_warming=False,\n", - " prefix=prefix,\n", + "expected_output_file = os.path.join(\n", + " \"..\", \"tests\", \"test-data\", \"expected-output-wg3/two_ips_climate_fair.xlsx\"\n", + ")\n", + "\n", + "expected_db_output = pyam.IamDataFrame(expected_output_file).timeseries()\n", + "\n", + "# The database does not necessarily include all the outputs we have\n", + "test_output = output.timeseries().loc[expected_db_output.index, :]\n", + "\n", + "# Check that we reproduce values\n", + "pdt.assert_frame_equal(\n", + " test_output,\n", + " expected_db_output,\n", + " rtol=1e-5,\n", + " atol=1e-6,\n", ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e76657b", - "metadata": {}, - "outputs": [], - "source": [ - "df_climate.timeseries()" - ] - }, { "cell_type": "markdown", - "id": "b0c2c685", + "id": "cca3f359", "metadata": {}, "source": [ - "### Combine output and do post-processing" + "### Some basic exploration" ] }, { "cell_type": "markdown", - "id": "2afcce04", + "id": "d617ae5e", "metadata": {}, "source": [ - "*N.B. prepare yourself for a very long log and a bit of scrolling this time, there will be a lot of (duplicate) information logged for each climate run.*" + "Look at the scenario categories." ] }, { "cell_type": "code", "execution_count": null, - "id": "415357d0", + "id": "82238f8b", "metadata": {}, "outputs": [], "source": [ - "results = pyam.concat([df_infilled, df_climate])\n", - "output = pyam.concat(\n", - " [input_df.filter(variable=results.variable, keep=False), results]\n", - ")\n", - "do_postprocess(\n", - " output,\n", - " outdir=outdir,\n", - " key_string=key_string,\n", - " categorisation=True,\n", - " reporting_completeness_categorisation=False,\n", - " prefix=prefix,\n", - " gwp=True,\n", - " model_version=model_version,\n", - " model=model,\n", - ")" + "output.meta[\"Category\"]" ] }, { "cell_type": "markdown", - "id": "35346bc7", + "id": "287f742c", "metadata": {}, "source": [ - "### Visualise results" + "Make a plot of median warming." ] }, { @@ -437,10 +383,11 @@ "metadata": {}, "outputs": [], "source": [ - "output.filter(variable=\"*|Surface Temperature (GSAT)|*|50.0th Percentile\").plot(color=\"scenario\", fill_between=True, final_ranges=dict(linewidth=5))\n", - "plt.tight_layout()\n", + "ax = output.filter(variable=\"*|Surface Temperature (GSAT)|*|50.0th Percentile\").plot(\n", + " color=\"scenario\"\n", + ")\n", "plt.title(\"Global warming above the 1850-1900 mean\")\n", - "plt.show()" + "ax.set_xlim([1995, 2100])" ] }, { @@ -449,9 +396,8 @@ "metadata": {}, "source": [ "## Conclusion\n", - "That's it! You just ran a a full climate assessment workflow going from emissions to temperature (and more) using the functionality from the climate-assessment package, and then visualised the results. \n", "\n", - "Naturally, similar workflows can be constructed using CICERO-SCM (on Linux) and MAGICC (on Windows and Linux)!\n", + "That's it! You just ran a a full climate assessment workflow going from emissions to temperature (and more) using the functionality from the climate-assessment package, and then visualised the results.\n", "\n", "It is also possible to run from the command line, and build more elaborate workflows. For that, please see the extended documentation." ] @@ -459,9 +405,9 @@ ], "metadata": { "kernelspec": { - "display_name": "ar6_env", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "ar6_env" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -473,7 +419,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.11" + "version": "3.7.5" } }, "nbformat": 4, diff --git a/notebooks/run-example-magicc.ipynb b/notebooks/run-example-magicc.ipynb new file mode 100644 index 0000000..2d175db --- /dev/null +++ b/notebooks/run-example-magicc.ipynb @@ -0,0 +1,447 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "529d03cc", + "metadata": {}, + "source": [ + "# Example run: MAGICC\n", + "\n", + "Here we demonstrate how to run with MAGICC. We run with the IPs from WG3 and demonstrate that the results in the database can be reproduced. However, other input data could be used for custom scenario runs.\n", + "\n", + "For more information, see the docs associated with the MAGICC model." + ] + }, + { + "cell_type": "markdown", + "id": "98357272", + "metadata": {}, + "source": [ + "## Setup logging\n", + "\n", + "Pyam does its own logging stuff, so we have to configure logging before that import." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59eecbfd", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "# Increase the level to reduce the number of log messages\n", + "LOGGING_LEVEL = logging.INFO\n", + "\n", + "LOGGER = logging.getLogger(\"pipeline\")\n", + "LOGGER.setLevel(LOGGING_LEVEL)\n", + "# have to set root logger too to get messages to come through\n", + "logging.getLogger().setLevel(LOGGING_LEVEL)\n", + "\n", + "logFormatter = logging.Formatter(\n", + " \"%(asctime)s %(name)s %(threadName)s - %(levelname)s: %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\",\n", + ")\n", + "stdoutHandler = logging.StreamHandler()\n", + "stdoutHandler.setFormatter(logFormatter)\n", + "\n", + "logging.getLogger().addHandler(stdoutHandler)" + ] + }, + { + "cell_type": "markdown", + "id": "c61735ad", + "metadata": {}, + "source": [ + "## Other imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acaff9ea", + "metadata": {}, + "outputs": [], + "source": [ + "import os.path\n", + "\n", + "import dotenv\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import pandas.testing as pdt\n", + "import pooch\n", + "import pyam\n", + "import tempfile\n", + "\n", + "from climate_assessment.cli import run_workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e319cda5", + "metadata": {}, + "outputs": [], + "source": [ + "dotenv.load_dotenv()" + ] + }, + { + "cell_type": "markdown", + "id": "a596e20e", + "metadata": {}, + "source": [ + "## Configuration of input data" + ] + }, + { + "cell_type": "markdown", + "id": "2f8c995d", + "metadata": {}, + "source": [ + "### Download model and configuration" + ] + }, + { + "cell_type": "markdown", + "id": "e0b76b47", + "metadata": {}, + "source": [ + "The MAGICC model and configuration is available at [magicc.org](https://magicc.org/), where you can also download the parameter set used in AR6. Please read the license and expectations carefully, we rely on users to act in a way which brings both new scientific outcomes but also acknowledges the work put into the MAGICC AR6 setup.\n", + "\n", + "After you have downloaded the tar files, please extract it (we typically extract MAGICC and the probabilistic distribution into `magicc-files` in the root directory). \n", + "\n", + "The set of commands on a unix-like system is given below (PRs to add the Windows equivalent are welcome)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "269bfcc7", + "metadata": {}, + "outputs": [], + "source": [ + "# The variables can be taken from an environment (.env) file or simply set here\n", + "# Just replace the links with the ones you receive from www.magicc.org\n", + "# os.environ[\"MAGICC_LINK_FROM_MAGICC_DOT_ORG\"] = \"www.magicc.org/api/download?type=MAGICC7&token=tok\"\n", + "# os.environ[\"MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG\"] = \"www.magicc.org/api/download?type=AR6_PROBABILISTIC&token=tok\"\n", + "os.environ[\"MAGICC_ROOT_FILES_DIR\"] = os.path.join(\"..\", \"magicc-files\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2395d78", + "metadata": {}, + "source": [ + "The cell below downloads and sets up MAGICC on a linux system. It can be skipped once it has been run once (and will need to be modified for a windows system)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c2453ea", + "metadata": {}, + "outputs": [], + "source": [ + "# MAGICC binary\n", + "!mkdir -p \"${MAGICC_ROOT_FILES_DIR}\"/magicc-v7.5.3\n", + "!wget -O \"${MAGICC_ROOT_FILES_DIR}/magicc-v7.5.3.tar.gz\" \"${MAGICC_LINK_FROM_MAGICC_DOT_ORG}\"\n", + "!tar -xf \"${MAGICC_ROOT_FILES_DIR}/magicc-v7.5.3.tar.gz\" -C \"${MAGICC_ROOT_FILES_DIR}/magicc-v7.5.3\"\n", + "\n", + "# Probabilistic distribution\n", + "!mkdir -p \"${MAGICC_ROOT_FILES_DIR}/magicc-ar6-0fd0f62-f023edb-drawnset\"\n", + "!wget -O \"${MAGICC_ROOT_FILES_DIR}/magicc-ar6-0fd0f62-f023edb-drawnset.tar.gz\" \"${MAGICC_PROB_DISTRIBUTION_LINK_FROM_MAGICC_DOT_ORG}\"\n", + "!tar -xf \"${MAGICC_ROOT_FILES_DIR}/magicc-ar6-0fd0f62-f023edb-drawnset.tar.gz\" -C \"${MAGICC_ROOT_FILES_DIR}/magicc-ar6-0fd0f62-f023edb-drawnset\"" + ] + }, + { + "cell_type": "markdown", + "id": "a8bdd363", + "metadata": {}, + "source": [ + "### Set model input paths\n", + "\n", + "Here we tell the pipeline where to look for MAGICC and where to create parallel workers on disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74129df4", + "metadata": {}, + "outputs": [], + "source": [ + "# Where is the binary\n", + "os.environ[\"MAGICC_EXECUTABLE_7\"] = os.path.join(\n", + " os.environ[\"MAGICC_ROOT_FILES_DIR\"], \"magicc-v7.5.3\", \"bin\", \"magicc\"\n", + ")\n", + "\n", + "# How many MAGICC workers can run in parallel?\n", + "os.environ[\"MAGICC_WORKER_NUMBER\"] = \"4\"\n", + "\n", + "# Where should the MAGICC workers be located on the filesystem (you need about\n", + "# 500Mb space per worker at the moment, they're removed after use)\n", + "os.environ[\"MAGICC_WORKER_ROOT_DIR\"] = tempfile.gettempdir()" + ] + }, + { + "cell_type": "markdown", + "id": "a22bdbac", + "metadata": {}, + "source": [ + "### Set general input arguments and options to the climate assessment workflow\n", + "\n", + "The values we've set below will let you run MAGICC and reproduce the AR6 WG3 results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1079bd63", + "metadata": {}, + "outputs": [], + "source": [ + "model = \"magicc\"\n", + "model_version = \"v7.5.3\"\n", + "probabilistic_file = os.path.join(\n", + " os.environ[\"MAGICC_ROOT_FILES_DIR\"],\n", + " \"magicc-ar6-0fd0f62-f023edb-drawnset\",\n", + " \"0fd0f62-derived-metrics-id-f023edb-drawnset.json\",\n", + ")\n", + "\n", + "# Use fewer (e.g. 10) if you just want to do a test run but note that this breaks\n", + "# the stats of the probabilistic ensemble\n", + "num_cfgs = 600\n", + "# Set to True if you're not using the full MAGICC ensemble\n", + "test_run = False\n", + "# How many scenarios do you want to run in one go?\n", + "scenario_batch_size = 20\n", + "\n", + "# Where should the output be saved?\n", + "outdir = os.path.join(\"..\", \"data\", \"output-magicc-example-notebook\")" + ] + }, + { + "cell_type": "markdown", + "id": "cbbed3b6", + "metadata": {}, + "source": [ + "### Choose input emissions pathway file\n", + "\n", + "By default, we use 2 of the AR6 IPs. You could use a different file (formatted the same way) if you wish." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41d0741c", + "metadata": {}, + "outputs": [], + "source": [ + "EMISSIONS_DATA_DIR = os.path.join(\"..\", \"tests\", \"test-data\")\n", + "EMISSIONS_INPUT_FILE = \"ar6_IPs_emissions.csv\"\n", + "\n", + "input_emissions_file = os.path.join(EMISSIONS_DATA_DIR, EMISSIONS_INPUT_FILE)" + ] + }, + { + "cell_type": "markdown", + "id": "f92038e8", + "metadata": {}, + "source": [ + "### Choose infiller database file\n", + "\n", + "We run using the infiller database that was used in CMIP6. As a result of the licensing of the scenario data, this file has to be downloaded by hand (see documentation under \"Installation\", section \"Infiller database\"). Make sure that the variable `infilling_database_file` points to where you saved this file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b27e25d", + "metadata": {}, + "outputs": [], + "source": [ + "infilling_database_file = os.path.join(\n", + " \"..\",\n", + " \"data\",\n", + " \"1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f7e8686c", + "metadata": {}, + "source": [ + "## Run the climate assessment workflow" + ] + }, + { + "cell_type": "markdown", + "id": "b1302596", + "metadata": {}, + "source": [ + "*N.B. the log with information and some warnings will be quite long - but that is nothing to worry about!*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74a44671", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "run_workflow(\n", + " input_emissions_file,\n", + " outdir,\n", + " model=model,\n", + " model_version=model_version,\n", + " probabilistic_file=probabilistic_file,\n", + " num_cfgs=num_cfgs,\n", + " infilling_database=infilling_database_file,\n", + " scenario_batch_size=scenario_batch_size,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "35346bc7", + "metadata": {}, + "source": [ + "### Load results\n", + "\n", + "*N.B The filename will likely have changed if you have run your own scenarios.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e7e20bf", + "metadata": {}, + "outputs": [], + "source": [ + "output = pyam.IamDataFrame(os.path.join(outdir, \"ar6_IPs_emissions_alloutput.xlsx\"))\n", + "output" + ] + }, + { + "cell_type": "markdown", + "id": "ce7ed444", + "metadata": {}, + "source": [ + "### Compare with database results\n", + "\n", + "These would normally need to be downloaded, but we include a set in the repository for testing. Here we check that we have reproduced the database results. Obviously, this should be skipped if you have run a custom scenario." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f25a77c", + "metadata": {}, + "outputs": [], + "source": [ + "expected_output_file = os.path.join(\n", + " \"..\", \"tests\", \"test-data\", \"expected-output-wg3/two_ips_climate_magicc.xlsx\"\n", + ")\n", + "\n", + "expected_db_output = pyam.IamDataFrame(expected_output_file).timeseries()\n", + "\n", + "# The database does not necessarily include all the outputs we have\n", + "test_output = output.timeseries().loc[expected_db_output.index, :]\n", + "\n", + "# Check that we reproduce values\n", + "pdt.assert_frame_equal(\n", + " test_output,\n", + " expected_db_output,\n", + " rtol=1e-3,\n", + " atol=1e-3,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cca3f359", + "metadata": {}, + "source": [ + "### Some basic exploration" + ] + }, + { + "cell_type": "markdown", + "id": "d617ae5e", + "metadata": {}, + "source": [ + "Look at the scenario categories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82238f8b", + "metadata": {}, + "outputs": [], + "source": [ + "output.meta[\"Category\"]" + ] + }, + { + "cell_type": "markdown", + "id": "287f742c", + "metadata": {}, + "source": [ + "Make a plot of median warming." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c381213b", + "metadata": {}, + "outputs": [], + "source": [ + "ax = output.filter(variable=\"*|Surface Temperature (GSAT)|*|50.0th Percentile\").plot(\n", + " color=\"scenario\"\n", + ")\n", + "plt.title(\"Global warming above the 1850-1900 mean\")\n", + "ax.set_xlim([1995, 2100])" + ] + }, + { + "cell_type": "markdown", + "id": "3b9b79e7", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "That's it! You just ran a a full climate assessment workflow going from emissions to temperature (and more) using the functionality from the climate-assessment package, and then visualised the results.\n", + "\n", + "It is also possible to run from the command line, and build more elaborate workflows. For that, please see the extended documentation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/setup.cfg b/setup.cfg index 9d2a502..6e3a5a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,6 +64,7 @@ docs = tests = codecov + nbval pytest-cov pytest-console-scripts pytest>=4.0 @@ -77,6 +78,7 @@ deploy = linter = bandit black + black-nb flake8 isort diff --git a/src/climate_assessment/checks.py b/src/climate_assessment/checks.py index d246d4e..28f5b0c 100644 --- a/src/climate_assessment/checks.py +++ b/src/climate_assessment/checks.py @@ -1043,8 +1043,7 @@ def _helper(out_kyoto): # are smaller then the Kyoto gases of harmonized emissions if (kyoto_fit_infilled.values < kyoto_fit_harmonized.values).any(): kyoto_wrong = kyoto_fit_infilled.loc[ - lambda kyoto_fit_infilled: kyoto_fit_infilled.values - < kyoto_fit_harmonized.values + kyoto_fit_infilled.values < kyoto_fit_harmonized.values ] raise ValueError( f"The Kyoto gases of infilled emissions data of {[ind for ind in kyoto_wrong.index]} " diff --git a/src/climate_assessment/testing.py b/src/climate_assessment/testing.py index 90e9bd3..668cfdb 100644 --- a/src/climate_assessment/testing.py +++ b/src/climate_assessment/testing.py @@ -1,5 +1,78 @@ +import os.path import traceback +import pooch +import pyam +import requests + def _format_traceback_and_stdout_from_click_result(result): return "{}\n\n{}".format(traceback.print_exception(*result.exc_info), result.stdout) + + +def _get_infiller_download_link(filename): + """ + Get infiller download link, intended only for use in CI + """ + pyam.iiasa.set_config( + os.environ.get("SCENARIO_EXPLORER_USER"), + os.environ.get("SCENARIO_EXPLORER_PASSWORD"), + "iiasa_creds.yaml", + ) + try: + conn = pyam.iiasa.Connection( + creds="iiasa_creds.yaml", + auth_url="https://db1.ene.iiasa.ac.at/EneAuth/config/v1", + ) + finally: + # remove the yaml cred file + os.remove("iiasa_creds.yaml") + + infiller_url = ( + "https://db1.ene.iiasa.ac.at/ar6-public-api/rest/v2.1/files/" + f"{filename}?redirect=false" + ) + + return requests.get( + infiller_url, + headers={"Authorization": f"Bearer {conn._token}"}, + ).json()["directLink"] + + +def _file_available_or_downloaded(filepath, hash_exp, url): + """ + Check if file exists (and matches expected hash) or can be downloaded + + Only intended for use in testing, but might provide inspiration for + how to do this for others + + Parameters + ---------- + filepath : str + Path to file + + hash_exp : str + Expected md5 hash + + url : str + URL from which to download the file if it doesn't exist + + Returns + ------- + bool + Is the file available (or has it been downloaded hence is now + available)? + """ + try: + pooch.retrieve( + url=url, + known_hash=f"md5:{hash_exp}", + path=os.path.dirname(filepath), + fname=os.path.basename(filepath), + ) + except Exception as exc: + # probably better ways to do this, can iterate as we use + print(str(exc)) + return False + + return True diff --git a/tests/conftest.py b/tests/conftest.py index d8dde85..e744747 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,12 +6,15 @@ import pandas as pd import pandas.testing as pdt -import pooch import pyam import pytest -import requests import scmdata +from climate_assessment.testing import ( + _file_available_or_downloaded, + _get_infiller_download_link, +) + TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "test-data") DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data") @@ -66,7 +69,7 @@ def rcmip_emissions(): hash_exp = "4044106f55ca65b094670e7577eaf9b3" rcmip_emms_url = "https://rcmip-protocols-au.s3-ap-southeast-2.amazonaws.com/v5.1.0/rcmip-emissions-annual-means-v5-1-0.csv" - if not file_available_or_downloaded( + if not _file_available_or_downloaded( rcmip_emms_filepath, hash_exp, rcmip_emms_url, @@ -363,67 +366,6 @@ def check_func( return check_func -def get_infiller_download_link(filename): - pyam.iiasa.set_config( - os.environ.get("SCENARIO_EXPLORER_USER"), - os.environ.get("SCENARIO_EXPLORER_PASSWORD"), - "iiasa_creds.yaml", - ) - try: - conn = pyam.iiasa.Connection( - creds="iiasa_creds.yaml", - auth_url="https://db1.ene.iiasa.ac.at/EneAuth/config/v1", - ) - finally: - # remove the yaml cred file - os.remove("iiasa_creds.yaml") - - infiller_url = ( - "https://db1.ene.iiasa.ac.at/ar6-public-api/rest/v2.1/files/" - f"{filename}?redirect=false" - ) - return requests.get( - infiller_url, - headers={"Authorization": f"Bearer {conn._token}"}, - ).json()["directLink"] - - -def file_available_or_downloaded(filepath, hash_exp, url): - """ - Check if file exists (and matches expected hash) or can be downloaded - - Parameters - ---------- - filepath : str - Path to file - - hash_exp : str - Expected md5 hash - - url : str - URL from which to download the file if it doesn't exist - - Returns - ------- - bool - Is the file available (or has it been downloaded hence is now - available)? - """ - try: - pooch.retrieve( - url=url, - known_hash=f"md5:{hash_exp}", - path=os.path.dirname(filepath), - fname=os.path.basename(filepath), - ) - except Exception as exc: - # probably better ways to do this, can iterate as we use - print(str(exc)) - return False - - return True - - @pytest.fixture(scope="session") def infiller_database_filepath(): INFILLER_DATABASE_NAME = ( @@ -450,7 +392,7 @@ def infiller_database_filepath(): ) try: - INFILLER_DATABASE_DOWNLOAD_URL = get_infiller_download_link( + INFILLER_DATABASE_DOWNLOAD_URL = _get_infiller_download_link( INFILLER_DATABASE_NAME ) except RuntimeError as exc: @@ -458,7 +400,7 @@ def infiller_database_filepath(): f"{skip_msg}. Retrieving infiller database download url failed with error message: {exc}" ) - if not file_available_or_downloaded( + if not _file_available_or_downloaded( INFILLER_DATABASE_FILEPATH, INFILLER_HASH, INFILLER_DATABASE_DOWNLOAD_URL, @@ -475,7 +417,7 @@ def fair_slim_configs_filepath(): FAIR_SLIM_CONFIGS_DOWNLOAD_URL = "https://zenodo.org/record/6601980/files/fair-1.6.2-wg3-params-slim.json?download=1" FAIR_SLIM_CONFIGS_FILEPATH = os.path.join(TEST_DATA_DIR, FAIR_SLIM_CONFIGS_FILENAME) - if not file_available_or_downloaded( + if not _file_available_or_downloaded( FAIR_SLIM_CONFIGS_FILEPATH, FAIR_SLIM_CONFIGS_HASH, FAIR_SLIM_CONFIGS_DOWNLOAD_URL, @@ -494,7 +436,7 @@ def fair_common_configs_filepath(): TEST_DATA_DIR, FAIR_COMMON_CONFIGS_FILENAME ) - if not file_available_or_downloaded( + if not _file_available_or_downloaded( FAIR_COMMON_CONFIGS_FILEPATH, FAIR_COMMON_CONFIGS_HASH, FAIR_COMMON_CONFIGS_DOWNLOAD_URL,