start tests

.github/workflows/build.yml

@@ -157,6 +157,7 @@ jobs:
       - name: Test (py)
         run: doit test:py:*
       - name: Upload (reports)
+        if: always()
         uses: actions/upload-artifact@v2
         with:
           name: |
@@ -170,6 +171,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DOCS_IN_CI: 1
+      FORCE_PYODIDE: 1
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -223,6 +225,17 @@ jobs:
         run: doit docs:sphinx
       - name: Check Built Artifacts
         run: doit check
+      - name: Test (py)
+        run: doit test:py:jupyterlite
+      - name: Upload (reports)
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: |
+            jupyterlite reports ${{ github.run_number }} ${{ matrix.os }} ${{ matrix.python-version }}
+          path: |
+            build/htmlcov
+            build/pytest
       - name: Upload (sphinx logs)
         if: always()
         uses: actions/upload-artifact@v2

dodo.py

@@ -512,14 +512,18 @@ def task_docs():
 
     docs_app_targets = [B.DOCS_APP_WHEEL_INDEX, B.DOCS_APP_JS_BUNDLE]
 
+    uptodate = []
+
     if C.FORCE_PYODIDE:
         docs_app_targets += [B.DOCS_APP_PYODIDE_JS]
+        uptodate = [doit.tools.config_changed(dict(pyodide_url=C.PYODIDE_URL))]
 
     yield U.ok(
         B.OK_DOCS_APP,
         name="app:build",
         doc="use the jupyterlite CLI to (pre-)build the docs app",
         task_dep=[f"dev:py:{C.NAME}"],
+        uptodate=uptodate,
         actions=[(U.docs_app, [])],
         file_dep=app_build_deps,
         targets=docs_app_targets,
@@ -528,6 +532,7 @@ def task_docs():
     yield dict(
         name="app:pack",
         doc="build the as-deployed app archive",
+        uptodate=uptodate,
         file_dep=[B.OK_DOCS_APP],
         actions=[(U.docs_app, ["archive"])],
         targets=[B.DOCS_APP_ARCHIVE],
@@ -628,6 +633,12 @@ def task_test():
     if C.LINTING_IN_CI:
         return
 
+    env = dict(os.environ)
+
+    if P.PYODIDE_ARCHIVE_CACHE.exists():
+        # this makes some tests e.g. archive _very_ slow
+        env["TEST_JUPYTERLITE_PYODIDE_URL"] = str(P.PYODIDE_ARCHIVE_CACHE)
+
     pytest_args = [
         *C.PYM,
         "pytest",
@@ -679,6 +690,7 @@ def task_test():
                     f"--html={html_index}",
                     "--self-contained-html",
                     *pkg_args,
+                    env=env,
                     cwd=cwd,
                 )
             ],
@@ -722,8 +734,10 @@ class C:
     PYODIDE_DOWNLOAD = f"{PYODIDE_GH}/releases/download"
     PYODIDE_VERSION = "0.18.1"
     PYODIDE_JS = "pyodide.js"
-    PYODIDE_URL = (
-        f"{PYODIDE_DOWNLOAD}/{PYODIDE_VERSION}/pyodide-build-{PYODIDE_VERSION}.tar.bz2"
+    PYODIDE_ARCHIVE = f"pyodide-build-{PYODIDE_VERSION}.tar.bz2"
+    PYODIDE_URL = os.environ.get(
+        "JUPYTERLITE_PYODIDE_URL",
+        f"{PYODIDE_DOWNLOAD}/{PYODIDE_VERSION}/{PYODIDE_ARCHIVE}",
     )
     PYODIDE_CDN_URL = (
         f"https://cdn.jsdelivr.net/pyodide/v{PYODIDE_VERSION}/full/{PYODIDE_JS}"
@@ -747,8 +761,9 @@ class C:
     DOCS_IN_CI = json.loads(os.environ.get("DOCS_IN_CI", "0"))
     LINTING_IN_CI = json.loads(os.environ.get("LINTING_IN_CI", "0"))
     TESTING_IN_CI = json.loads(os.environ.get("TESTING_IN_CI", "0"))
-    FORCE_PYODIDE = DOCS_IN_CI or bool(json.loads(os.environ.get("FORCE_PYODIDE", "0")))
-
+    FORCE_PYODIDE = "JUPYTERLITE_PYODIDE_URL" in os.environ or bool(
+        json.loads(os.environ.get("FORCE_PYODIDE", "0"))
+    )
     PYM = [sys.executable, "-m"]
     FLIT = [*PYM, "flit"]
     SOURCE_DATE_EPOCH = (
@@ -785,6 +800,7 @@ class P:
         for p in EXAMPLES.rglob("*")
         if not p.is_dir() and ".cache" not in str(p) and ".doit" not in str(p)
     ]
+    PYODIDE_ARCHIVE_CACHE = EXAMPLES / ".cache/pyodide" / C.PYODIDE_ARCHIVE
 
     # set later
     PYOLITE_PACKAGES = {}

py/jupyterlite/src/jupyterlite/addons/pyodide.py

@@ -39,39 +39,45 @@ def output_pyodide(self):
         """where labextensions will go in the output folder"""
         return self.manager.output_dir / "static" / PYODIDE
 
+    @property
+    def well_known_pyodide(self):
+        """a well-known path where pyodide might be stored"""
+        return self.manager.lite_dir / "static" / PYODIDE
+
     def status(self, manager):
+        """report on the status of pyodide"""
         yield dict(
             name="pyodide",
             actions=[
                 lambda: print(
-                    f"        pydodide URL: {manager.pyodide_url}",
+                    f"     URL: {manager.pyodide_url}",
                 ),
+                lambda: print(f" archive: {[*self.pyodide_cache.glob('*.bz2')]}"),
                 lambda: print(
-                    f"    pyodide archives: {[*self.pyodide_cache.glob('*.bz2')]}"
+                    f"   cache: {len([*self.pyodide_cache.rglob('*')])} files",
                 ),
                 lambda: print(
-                    f"       pyodide cache: {len([*self.pyodide_cache.rglob('*')])} files",
+                    f"   local: {len([*self.well_known_pyodide.rglob('*')])} files"
                 ),
             ],
         )
 
     def post_init(self, manager):
-        """handle downloading of wheels"""
-        if not manager.pyodide_url:
+        """handle downloading of pyodide"""
+        if manager.pyodide_url is None:
             return
 
         yield from self.cache_pyodide(manager.pyodide_url)
 
     def build(self, manager):
-        """yield a doit task to copy a local pyodide into the output_dir"""
-        user_pyodide = manager.lite_dir / "static" / PYODIDE
+        """copy a local (cached or well-known) pyodide into the output_dir"""
         cached_pyodide = self.pyodide_cache / PYODIDE / PYODIDE
 
         the_pyodide = None
 
-        if user_pyodide.exists():
-            the_pyodide = user_pyodide
-        elif manager.pyodide_url:
+        if self.well_known_pyodide.exists():
+            the_pyodide = self.well_known_pyodide
+        elif manager.pyodide_url is not None:
             the_pyodide = cached_pyodide
 
         if not the_pyodide:
@@ -89,7 +95,8 @@ def build(self, manager):
         )
 
     def post_build(self, manager):
-        if not manager.pyodide_url:
+        """configure jupyter-lite.json for pyodide"""
+        if not self.well_known_pyodide.exists() and manager.pyodide_url is None:
             return
 
         jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON
@@ -109,6 +116,7 @@ def post_build(self, manager):
         )
 
     def check(self, manager):
+        """ensure the pyodide configuration is sound"""
         jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON
 
         yield dict(
@@ -118,7 +126,7 @@ def check(self, manager):
         )
 
     def check_config_paths(self, jupyterlite_json):
-        config = json.loads(jupyterlite_json.read_text(**UTF8))[JUPYTER_CONFIG_DATA]
+        config = json.loads(jupyterlite_json.read_text(**UTF8))
         pyodide_url = (
             config.setdefault(JUPYTER_CONFIG_DATA, {})
             .setdefault(LITE_PLUGIN_SETTINGS, {})
@@ -137,6 +145,7 @@ def check_config_paths(self, jupyterlite_json):
         assert pyodide_packages.exists(), f"{pyodide_packages} not found"
 
     def patch_jupyterlite_json(self, jupyterlite_json, output_js):
+        """update jupyter-lite.json to use the local pyodide"""
         config = json.loads(jupyterlite_json.read_text(**UTF8))
         pyolite_config = (
             config.setdefault(JUPYTER_CONFIG_DATA, {})
@@ -150,6 +159,7 @@ def patch_jupyterlite_json(self, jupyterlite_json, output_js):
             self.maybe_timestamp(jupyterlite_json)
 
     def cache_pyodide(self, path_or_url):
+        """copy pyodide to the cache"""
         if re.findall(r"^https?://", path_or_url):
             url = urllib.parse.urlparse(path_or_url)
             name = url.path.split("/")[-1]
@@ -165,9 +175,10 @@ def cache_pyodide(self, path_or_url):
                 will_fetch = True
         else:
             local_path = (self.manager.lite_dir / path_or_url).resolve()
+            dest = self.pyodide_cache / local_path.name
 
         if local_path.is_dir():
-            all_paths = local_path.rglob("*")
+            all_paths = sorted([p for p in local_path.rglob("*") if not p.is_dir()])
             yield dict(
                 name=f"copy:pyodide:{local_path.name}",
                 file_dep=[*all_paths],

py/jupyterlite/src/jupyterlite/config.py

@@ -86,7 +86,7 @@ class LiteBuildConfig(LoggingConfigurable):
     ).tag(config=True)
 
     pyodide_url: str = Unicode(
-        help="Local path or URL of a pyodide distribution tarball"
+        allow_none=True, help="Local path or URL of a pyodide distribution tarball"
     ).tag(config=True)
 
     settings_overrides: _Tuple[_Text] = TypedTuple(
@@ -210,3 +210,7 @@ def _default_port(self):
     @default("base_url")
     def _default_base_url(self):
         return os.environ.get("JUPYTERLITE_BASE_URL", "/")
+
+    @default("pyodide_url")
+    def _default_pyodide_url(self):
+        return os.environ.get("JUPYTERLITE_PYODIDE_URL")

py/jupyterlite/src/jupyterlite/tests/test_pyodide.py

@@ -0,0 +1,83 @@
+"""tests of various mechanisms of using pyodide"""
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ENV_VAR = "JUPYTERLITE_PYODIDE_URL"
+
+TEST_PYODIDE_URL = os.environ.pop(f"TEST_{ENV_VAR}")
+
+if TEST_PYODIDE_URL is None:
+    pytest.skip("skipping pyodide tests", allow_module_level=True)
+
+
+@pytest.fixture
+def a_pyodide_server(an_unused_port):
+    """serve up the pyodide archive"""
+    root = Path(TEST_PYODIDE_URL).parent
+    assert root.exists()
+
+    p = subprocess.Popen(
+        ["python", "-m", "http.server", "-b", "127.0.0.1", f"{an_unused_port}"],
+        cwd=str(root),
+    )
+    url = f"http://localhost:{an_unused_port}"
+    yield url
+    p.terminate()
+
+
+@pytest.mark.parametrize(
+    "approach,path,kind",
+    [
+        ["cli", "local", "archive"],
+        ["cli", "local", "folder"],
+        ["cli", "remote", "archive"],
+        ["env", "local", "archive"],
+        ["env", "local", "folder"],
+        ["env", "remote", "archive"],
+        ["wellknown", None, None],
+    ],
+)
+def test_pyodide(
+    an_empty_lite_dir, script_runner, a_pyodide_server, approach, path, kind
+):
+    """can we fetch a pyodide archive, or use a local copy?"""
+    env = dict(os.environ)
+    pargs = []
+
+    if approach == "wellknown":
+        static = an_empty_lite_dir / "static"
+        static.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(
+            Path(TEST_PYODIDE_URL).parent / "pyodide/pyodide",
+            static / "pyodide",
+        )
+    else:
+        url = TEST_PYODIDE_URL
+
+        if path == "remote":
+            url = f"{a_pyodide_server}/{Path(url).name}"
+        elif kind == "folder":
+            url = str(Path(url).parent / "pyodide")
+
+        if approach == "env":
+            env[ENV_VAR] = url
+        else:
+            pargs += ["--pyodide", url]
+
+    kwargs = dict(cwd=str(an_empty_lite_dir), env=env)
+
+    status = script_runner.run("jupyter", "lite", "status", *pargs, **kwargs)
+    assert status.success, "status did NOT succeed"
+
+    build = script_runner.run("jupyter", "lite", "build", *pargs, **kwargs)
+    assert build.success, "the build did NOT succeed"
+
+    pyodide_path = an_empty_lite_dir / "_output/static/pyodide/pyodide.js"
+    assert pyodide_path.exists(), "pyodide.js does not exist"
+
+    check = script_runner.run("jupyter", "lite", "check", *pargs, **kwargs)
+    assert check.success, "the check did NOT succeed"