diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 401abb05c..155b4bb0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,6 +157,7 @@ jobs: - name: Test (py) run: doit test:py:* - name: Upload (reports) + if: always() uses: actions/upload-artifact@v2 with: name: | @@ -170,6 +171,7 @@ jobs: runs-on: ubuntu-latest env: DOCS_IN_CI: 1 + FORCE_PYODIDE: 1 steps: - name: Checkout uses: actions/checkout@v2 @@ -223,6 +225,17 @@ jobs: run: doit docs:sphinx - name: Check Built Artifacts run: doit check + - name: Test (py) + run: doit test:py:jupyterlite + - name: Upload (reports) + if: always() + uses: actions/upload-artifact@v2 + with: + name: | + jupyterlite reports ${{ github.run_number }} ${{ matrix.os }} ${{ matrix.python-version }} + path: | + build/htmlcov + build/pytest - name: Upload (sphinx logs) if: always() uses: actions/upload-artifact@v2 diff --git a/dodo.py b/dodo.py index 84c152540..490f742cd 100644 --- a/dodo.py +++ b/dodo.py @@ -512,14 +512,18 @@ def task_docs(): docs_app_targets = [B.DOCS_APP_WHEEL_INDEX, B.DOCS_APP_JS_BUNDLE] + uptodate = [] + if C.FORCE_PYODIDE: docs_app_targets += [B.DOCS_APP_PYODIDE_JS] + uptodate = [doit.tools.config_changed(dict(pyodide_url=C.PYODIDE_URL))] yield U.ok( B.OK_DOCS_APP, name="app:build", doc="use the jupyterlite CLI to (pre-)build the docs app", task_dep=[f"dev:py:{C.NAME}"], + uptodate=uptodate, actions=[(U.docs_app, [])], file_dep=app_build_deps, targets=docs_app_targets, @@ -528,6 +532,7 @@ def task_docs(): yield dict( name="app:pack", doc="build the as-deployed app archive", + uptodate=uptodate, file_dep=[B.OK_DOCS_APP], actions=[(U.docs_app, ["archive"])], targets=[B.DOCS_APP_ARCHIVE], @@ -628,6 +633,12 @@ def task_test(): if C.LINTING_IN_CI: return + env = dict(os.environ) + + if P.PYODIDE_ARCHIVE_CACHE.exists(): + # this makes some tests e.g. archive _very_ slow + env["TEST_JUPYTERLITE_PYODIDE_URL"] = str(P.PYODIDE_ARCHIVE_CACHE) + pytest_args = [ *C.PYM, "pytest", @@ -679,6 +690,7 @@ def task_test(): f"--html={html_index}", "--self-contained-html", *pkg_args, + env=env, cwd=cwd, ) ], @@ -722,8 +734,10 @@ class C: PYODIDE_DOWNLOAD = f"{PYODIDE_GH}/releases/download" PYODIDE_VERSION = "0.18.1" PYODIDE_JS = "pyodide.js" - PYODIDE_URL = ( - f"{PYODIDE_DOWNLOAD}/{PYODIDE_VERSION}/pyodide-build-{PYODIDE_VERSION}.tar.bz2" + PYODIDE_ARCHIVE = f"pyodide-build-{PYODIDE_VERSION}.tar.bz2" + PYODIDE_URL = os.environ.get( + "JUPYTERLITE_PYODIDE_URL", + f"{PYODIDE_DOWNLOAD}/{PYODIDE_VERSION}/{PYODIDE_ARCHIVE}", ) PYODIDE_CDN_URL = ( f"https://cdn.jsdelivr.net/pyodide/v{PYODIDE_VERSION}/full/{PYODIDE_JS}" @@ -747,8 +761,9 @@ class C: DOCS_IN_CI = json.loads(os.environ.get("DOCS_IN_CI", "0")) LINTING_IN_CI = json.loads(os.environ.get("LINTING_IN_CI", "0")) TESTING_IN_CI = json.loads(os.environ.get("TESTING_IN_CI", "0")) - FORCE_PYODIDE = DOCS_IN_CI or bool(json.loads(os.environ.get("FORCE_PYODIDE", "0"))) - + FORCE_PYODIDE = "JUPYTERLITE_PYODIDE_URL" in os.environ or bool( + json.loads(os.environ.get("FORCE_PYODIDE", "0")) + ) PYM = [sys.executable, "-m"] FLIT = [*PYM, "flit"] SOURCE_DATE_EPOCH = ( @@ -785,6 +800,7 @@ class P: for p in EXAMPLES.rglob("*") if not p.is_dir() and ".cache" not in str(p) and ".doit" not in str(p) ] + PYODIDE_ARCHIVE_CACHE = EXAMPLES / ".cache/pyodide" / C.PYODIDE_ARCHIVE # set later PYOLITE_PACKAGES = {} diff --git a/py/jupyterlite/src/jupyterlite/addons/pyodide.py b/py/jupyterlite/src/jupyterlite/addons/pyodide.py index bb111e240..ec8df76db 100644 --- a/py/jupyterlite/src/jupyterlite/addons/pyodide.py +++ b/py/jupyterlite/src/jupyterlite/addons/pyodide.py @@ -39,39 +39,45 @@ def output_pyodide(self): """where labextensions will go in the output folder""" return self.manager.output_dir / "static" / PYODIDE + @property + def well_known_pyodide(self): + """a well-known path where pyodide might be stored""" + return self.manager.lite_dir / "static" / PYODIDE + def status(self, manager): + """report on the status of pyodide""" yield dict( name="pyodide", actions=[ lambda: print( - f" pydodide URL: {manager.pyodide_url}", + f" URL: {manager.pyodide_url}", ), + lambda: print(f" archive: {[*self.pyodide_cache.glob('*.bz2')]}"), lambda: print( - f" pyodide archives: {[*self.pyodide_cache.glob('*.bz2')]}" + f" cache: {len([*self.pyodide_cache.rglob('*')])} files", ), lambda: print( - f" pyodide cache: {len([*self.pyodide_cache.rglob('*')])} files", + f" local: {len([*self.well_known_pyodide.rglob('*')])} files" ), ], ) def post_init(self, manager): - """handle downloading of wheels""" - if not manager.pyodide_url: + """handle downloading of pyodide""" + if manager.pyodide_url is None: return yield from self.cache_pyodide(manager.pyodide_url) def build(self, manager): - """yield a doit task to copy a local pyodide into the output_dir""" - user_pyodide = manager.lite_dir / "static" / PYODIDE + """copy a local (cached or well-known) pyodide into the output_dir""" cached_pyodide = self.pyodide_cache / PYODIDE / PYODIDE the_pyodide = None - if user_pyodide.exists(): - the_pyodide = user_pyodide - elif manager.pyodide_url: + if self.well_known_pyodide.exists(): + the_pyodide = self.well_known_pyodide + elif manager.pyodide_url is not None: the_pyodide = cached_pyodide if not the_pyodide: @@ -89,7 +95,8 @@ def build(self, manager): ) def post_build(self, manager): - if not manager.pyodide_url: + """configure jupyter-lite.json for pyodide""" + if not self.well_known_pyodide.exists() and manager.pyodide_url is None: return jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON @@ -109,6 +116,7 @@ def post_build(self, manager): ) def check(self, manager): + """ensure the pyodide configuration is sound""" jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON yield dict( @@ -118,7 +126,7 @@ def check(self, manager): ) def check_config_paths(self, jupyterlite_json): - config = json.loads(jupyterlite_json.read_text(**UTF8))[JUPYTER_CONFIG_DATA] + config = json.loads(jupyterlite_json.read_text(**UTF8)) pyodide_url = ( config.setdefault(JUPYTER_CONFIG_DATA, {}) .setdefault(LITE_PLUGIN_SETTINGS, {}) @@ -137,6 +145,7 @@ def check_config_paths(self, jupyterlite_json): assert pyodide_packages.exists(), f"{pyodide_packages} not found" def patch_jupyterlite_json(self, jupyterlite_json, output_js): + """update jupyter-lite.json to use the local pyodide""" config = json.loads(jupyterlite_json.read_text(**UTF8)) pyolite_config = ( config.setdefault(JUPYTER_CONFIG_DATA, {}) @@ -150,6 +159,7 @@ def patch_jupyterlite_json(self, jupyterlite_json, output_js): self.maybe_timestamp(jupyterlite_json) def cache_pyodide(self, path_or_url): + """copy pyodide to the cache""" if re.findall(r"^https?://", path_or_url): url = urllib.parse.urlparse(path_or_url) name = url.path.split("/")[-1] @@ -165,9 +175,10 @@ def cache_pyodide(self, path_or_url): will_fetch = True else: local_path = (self.manager.lite_dir / path_or_url).resolve() + dest = self.pyodide_cache / local_path.name if local_path.is_dir(): - all_paths = local_path.rglob("*") + all_paths = sorted([p for p in local_path.rglob("*") if not p.is_dir()]) yield dict( name=f"copy:pyodide:{local_path.name}", file_dep=[*all_paths], diff --git a/py/jupyterlite/src/jupyterlite/config.py b/py/jupyterlite/src/jupyterlite/config.py index bbc7f7179..1989c24a5 100644 --- a/py/jupyterlite/src/jupyterlite/config.py +++ b/py/jupyterlite/src/jupyterlite/config.py @@ -86,7 +86,7 @@ class LiteBuildConfig(LoggingConfigurable): ).tag(config=True) pyodide_url: str = Unicode( - help="Local path or URL of a pyodide distribution tarball" + allow_none=True, help="Local path or URL of a pyodide distribution tarball" ).tag(config=True) settings_overrides: _Tuple[_Text] = TypedTuple( @@ -210,3 +210,7 @@ def _default_port(self): @default("base_url") def _default_base_url(self): return os.environ.get("JUPYTERLITE_BASE_URL", "/") + + @default("pyodide_url") + def _default_pyodide_url(self): + return os.environ.get("JUPYTERLITE_PYODIDE_URL") diff --git a/py/jupyterlite/src/jupyterlite/tests/test_pyodide.py b/py/jupyterlite/src/jupyterlite/tests/test_pyodide.py new file mode 100644 index 000000000..be3d4c57e --- /dev/null +++ b/py/jupyterlite/src/jupyterlite/tests/test_pyodide.py @@ -0,0 +1,83 @@ +"""tests of various mechanisms of using pyodide""" +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +ENV_VAR = "JUPYTERLITE_PYODIDE_URL" + +TEST_PYODIDE_URL = os.environ.pop(f"TEST_{ENV_VAR}") + +if TEST_PYODIDE_URL is None: + pytest.skip("skipping pyodide tests", allow_module_level=True) + + +@pytest.fixture +def a_pyodide_server(an_unused_port): + """serve up the pyodide archive""" + root = Path(TEST_PYODIDE_URL).parent + assert root.exists() + + p = subprocess.Popen( + ["python", "-m", "http.server", "-b", "127.0.0.1", f"{an_unused_port}"], + cwd=str(root), + ) + url = f"http://localhost:{an_unused_port}" + yield url + p.terminate() + + +@pytest.mark.parametrize( + "approach,path,kind", + [ + ["cli", "local", "archive"], + ["cli", "local", "folder"], + ["cli", "remote", "archive"], + ["env", "local", "archive"], + ["env", "local", "folder"], + ["env", "remote", "archive"], + ["wellknown", None, None], + ], +) +def test_pyodide( + an_empty_lite_dir, script_runner, a_pyodide_server, approach, path, kind +): + """can we fetch a pyodide archive, or use a local copy?""" + env = dict(os.environ) + pargs = [] + + if approach == "wellknown": + static = an_empty_lite_dir / "static" + static.mkdir(parents=True, exist_ok=True) + shutil.copytree( + Path(TEST_PYODIDE_URL).parent / "pyodide/pyodide", + static / "pyodide", + ) + else: + url = TEST_PYODIDE_URL + + if path == "remote": + url = f"{a_pyodide_server}/{Path(url).name}" + elif kind == "folder": + url = str(Path(url).parent / "pyodide") + + if approach == "env": + env[ENV_VAR] = url + else: + pargs += ["--pyodide", url] + + kwargs = dict(cwd=str(an_empty_lite_dir), env=env) + + status = script_runner.run("jupyter", "lite", "status", *pargs, **kwargs) + assert status.success, "status did NOT succeed" + + build = script_runner.run("jupyter", "lite", "build", *pargs, **kwargs) + assert build.success, "the build did NOT succeed" + + pyodide_path = an_empty_lite_dir / "_output/static/pyodide/pyodide.js" + assert pyodide_path.exists(), "pyodide.js does not exist" + + check = script_runner.run("jupyter", "lite", "check", *pargs, **kwargs) + assert check.success, "the check did NOT succeed"