Moving evaluated notebooks to doc dir (#96)

* Moving evalutated notebooks to doc dir - never clean

* Clean up

* fixing typo

Co-Authored-By: jsignell <[email protected]>

* Flipping cleaning arg back:

.gitignore

@@ -10,3 +10,12 @@
 *.out
 *.lock
 *.doit*
+
+# nbsite
+# these files normally shouldn't be checked in as they should be
+# dynamically built from notebooks
+doc/**/*.rst
+doc/**/*.ipynb
+doc/**/*.json
+# this dir contains the whole website and should not be checked in on master
+builtdocs/

nbsite/__main__.py

@@ -41,7 +41,7 @@ def main(args=None):
     build_parser.add_argument('--output',type=str,help='where to init doc',default="builtdocs")
     _add_common_args(build_parser,'--project-root','--doc','--examples')
     build_parser.add_argument('--examples-assets',type=str,help='where to init doc',default="assets")
-    build_parser.add_argument('--clean-force',action='store_true',help='whether to actually delete files when cleaning (useful for uploading)')
+    build_parser.add_argument('--clean-dry-run',action='store_true',help='whether to not actually delete files from output (useful for uploading)')
     _set_defaults(build_parser,build)
 
     # add commands from pyct, for examples

nbsite/cmd.py

@@ -29,7 +29,7 @@ def fix_links(output):
     subprocess.check_call(["nbsite_fix_links.py",output])
 
 
-def build(what,output,project_root='',doc='doc',examples='examples',examples_assets="assets", clean_force=False):
+def build(what,output,project_root='',doc='doc',examples='examples',examples_assets="assets", clean_dry_run=False):
     # TODO: also have an overwrite flag
     paths = _prepare_paths(project_root,examples=examples,doc=doc,examples_assets=examples_assets)
     subprocess.check_call(["sphinx-build","-b",what,paths['doc'],output])
@@ -40,11 +40,11 @@ def build(what,output,project_root='',doc='doc',examples='examples',examples_ass
     fix_links(output)
     # create a .nojekyll file in output for github compatibility
     subprocess.check_call(["touch", os.path.join(output, '.nojekyll')])
-    if not clean_force:
-        print("Call `nbsite build` with `--clean-force` to actually delete files.")
-    clean(output, not clean_force)
+    if not clean_dry_run:
+        print("Call `nbsite build` with `--clean-dry-run` to not actually delete files.")
+    clean(output, clean_dry_run)
 
-def clean(output, dry_run=True):
+def clean(output, dry_run=False):
     if dry_run:
         subprocess.check_call(["nbsite_cleandisthtml.py",output])
     else:

nbsite/nbbuild.py

@@ -48,7 +48,7 @@ def kc(self):
     @kc.deleter
     def kc(self):
         del self._kc
-    
+
     @kc.setter
     def kc(self,v):
         self._kc=v
@@ -159,8 +159,7 @@ def run(self):
         nb_abs_path = os.path.abspath(os.path.join(rst_dir, self.arguments[1]))
         nb_filepath, nb_basename = os.path.split(nb_abs_path)
 
-        rel_dir = os.path.relpath(rst_dir, setup.confdir)
-        dest_dir = os.path.join(setup.app.builder.outdir, rel_dir)
+        dest_dir = rst_dir
         dest_path = os.path.join(dest_dir, nb_basename)
 
         if not os.path.exists(dest_dir):
@@ -169,7 +168,6 @@ def run(self):
         # Process file inclusion options
         include_opts = self.arguments[2:]
         include_nb = True if 'ipynb' in include_opts else False
-        include_eval = True if 'eval' in include_opts else False
         include_script = True if 'py' in include_opts else False
 
         link_rst = ''
@@ -179,13 +177,6 @@ def run(self):
         if include_nb:
             link_rst += formatted_link(nb_basename) + '; '
 
-        if include_eval:
-            dest_path_eval = string.replace(dest_path, '.ipynb', '_evaluated.ipynb')
-            rel_path_eval = string.replace(nb_basename, '.ipynb', '_evaluated.ipynb')
-            link_rst += formatted_link(rel_path_eval) + ('; ' if include_script else '')
-        else:
-            dest_path_eval = os.path.join(dest_dir, 'temp_evaluated.ipynb')
-
         if include_script:
             dest_path_script = string.replace(dest_path, '.ipynb', '.py')
             rel_path_script = string.replace(nb_basename, '.ipynb', '.py')
@@ -202,13 +193,8 @@ def run(self):
         # Evaluate Notebook and insert into Sphinx doc
         skip_exceptions = 'skip_exceptions' in self.options
 
-        # Make temp_evaluated.ipynb include the notebook name
-        nb_name = os.path.split(nb_abs_path)[1].replace('.ipynb','')
-        dest_path_eval = dest_path_eval.replace('temp_evaluated.ipynb',
-                               '{nb_name}_temp_evaluated.ipynb'.format(nb_name=nb_name))
-
         # Parse slice
-        evaluated_text = evaluate_notebook(nb_abs_path, dest_path_eval,
+        evaluated_text = evaluate_notebook(nb_abs_path, dest_path,
                                            skip_exceptions=skip_exceptions,
                                            substring=self.options.get('substring'),
                                            end=self.options.get('end'),
@@ -283,7 +269,7 @@ def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False,substring=N
 
     if not os.path.isfile(dest_path):
         # TODO but this isn't true, is it? it's running the originl nb
-        print('INFO: Running temp notebook {dest_path!s}'.format(
+        print('INFO: Writing evaluated notebook to {dest_path!s}'.format(
             dest_path=os.path.abspath(dest_path)))
         try:
             if not skip_execute:
@@ -308,9 +294,8 @@ def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False,substring=N
                 for f in glob.glob(os.path.join(os.path.dirname(nb_path),pattern)):
                     print("mv %s %s"%(f, os.path.dirname(dest_path)))
                     shutil.move(f,os.path.dirname(dest_path))
-
     else:
-        print('INFO: Skipping existing temp notebook {dest_path!s}'.format(
+        print('INFO: Skipping existing evaluated notebook {dest_path!s}'.format(
             dest_path=os.path.abspath(dest_path)))
 
     preprocessors = [] if substring is None and not offset else [NotebookSlice(substring, end, offset)]
@@ -339,7 +324,7 @@ def setup(app):
     app.add_config_value('nbbuild_cell_timeout',300,'html')
     app.add_config_value('nbbuild_ipython_startup',"from nbsite.ipystartup import *",'html')
     app.add_config_value('nbbuild_patterns_to_take_along',["*.json"],'html')
-    
+
     app.add_node(notebook_node,
                  html=(visit_notebook_node, depart_notebook_node))
 

nbsite/tests/test_cmd.py

@@ -284,6 +284,8 @@ def test_build(tmp_project_with_docs_skeleton):
     (project / "doc" / "Example_Notebook_0.rst").write_text(EXAMPLE_0_RST)
     (project / "doc" / "Example_Notebook_1.rst").write_text(EXAMPLE_1_RST)
     build('html', str(project / "builtdocs"), project_root=str(project), examples_assets='')
+    assert (project / "doc" / "Example_Notebook_0.ipynb").is_file()
+    assert (project / "doc" / "Example_Notebook_1.ipynb").is_file()
     assert (project / "builtdocs" / "Example_Notebook_0.html").is_file()
     assert (project / "builtdocs" / "Example_Notebook_1.html").is_file()
 
@@ -310,21 +312,21 @@ def test_build_with_just_one_rst(tmp_project_with_docs_skeleton):
     assert (project / "builtdocs" / ".nojekyll").is_file()
 
 @pytest.mark.slow
-def test_build_does_not_deletes_by_default(tmp_project_with_docs_skeleton):
+def test_build_deletes_by_default(tmp_project_with_docs_skeleton):
     project = tmp_project_with_docs_skeleton
     (project / "doc" / "Example_Notebook_0.rst").write_text(EXAMPLE_0_RST)
     (project / "doc" / "Example_Notebook_1.rst").write_text(EXAMPLE_1_RST)
     build('html', str(project / "builtdocs"), project_root=str(project), examples_assets='')
-    assert (project / "builtdocs" / "Example_Notebook_1_temp_evaluated.ipynb").is_file()
+    assert not (project / "builtdocs" / ".doctrees").is_dir()
     assert (project / "builtdocs" / "Example_Notebook_1.html").is_file()
-    assert len(list((project / "builtdocs").iterdir())) == 14
+    assert len(list((project / "builtdocs").iterdir())) == 9
 
 @pytest.mark.slow
-def test_build_with_clean_force_deletes(tmp_project_with_docs_skeleton):
+def test_build_with_clean_dry_run_does_not_delere(tmp_project_with_docs_skeleton):
     project = tmp_project_with_docs_skeleton
     (project / "doc" / "Example_Notebook_0.rst").write_text(EXAMPLE_0_RST)
     (project / "doc" / "Example_Notebook_1.rst").write_text(EXAMPLE_1_RST)
-    build('html', str(project / "builtdocs"), project_root=str(project), examples_assets='', clean_force=True)
-    assert not (project / "builtdocs" / "Example_Notebook_1_temp_evaluated.ipynb").is_file()
+    build('html', str(project / "builtdocs"), project_root=str(project), examples_assets='', clean_dry_run=True)
+    assert (project / "builtdocs" / ".doctrees").is_dir()
     assert (project / "builtdocs" / "Example_Notebook_1.html").is_file()
-    assert len(list((project / "builtdocs").iterdir())) == 9
+    assert len(list((project / "builtdocs").iterdir())) == 12