From 62f4ba6426cc7d557e31f9ca7467f77ec9abe6d1 Mon Sep 17 00:00:00 2001 From: jasonvranek Date: Mon, 8 Mar 2021 18:06:53 -0800 Subject: [PATCH] update docs --- README.md | 6 - chisel_nbdev/__init__.py | 2 +- docs/_data/sidebars/home_sidebar.yml | 24 + docs/clean_scala.html | 463 ++++++ docs/cli.html | 609 +++++++ docs/export_scala.html | 1362 +++++++++++++++ docs/export_scala2html.html | 2298 ++++++++++++++++++++++++++ docs/index.html | 8 +- docs/merge.html | 515 ++++++ docs/show_scaladoc.html | 0 docs/sidebar.json | 8 + docs/sync_scala.html | 484 ++++++ docs/test_scala.html | 514 ++++++ nbs/00_export_scala.ipynb | 14 +- nbs/index.ipynb | 8 +- 15 files changed, 6281 insertions(+), 34 deletions(-) create mode 100644 docs/clean_scala.html create mode 100644 docs/cli.html create mode 100644 docs/export_scala.html create mode 100644 docs/export_scala2html.html create mode 100644 docs/merge.html create mode 100644 docs/show_scaladoc.html create mode 100644 docs/sync_scala.html create mode 100644 docs/test_scala.html diff --git a/README.md b/README.md index d6738d9..8df712c 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,6 @@ - create virtualenv: ```python3 -m virtualenv --python=/usr/local/bin/python3 chisel_nb_env``` - activate it: ```source chisel_nb_env/bin/activate``` - install jupyterlab (which includes jupyter notebook): ```pip install jupyterlab```## Requirements (mac) -- setup some virtualenv (im using python3.8.2) -- ```pip install virtualenv``` -- find your python3 binary: ```which python3``` which for me is ```/usr/local/bin/python3``` -- create virtualenv: ```python3 -m virtualenv --python=/usr/local/bin/python3 chisel_nb_env``` -- activate it: ```source chisel_nb_env/bin/activate``` -- install jupyterlab (which includes jupyter notebook): ```pip install jupyterlab``` ### Setup the Jupyter Scala kernel Almond (https://almond.sh) - Borrowed from the Chisel-Bootcamp install guide: https://github.com/freechipsproject/chisel-bootcamp/blob/master/Install.md diff --git a/chisel_nbdev/__init__.py b/chisel_nbdev/__init__.py index b3f4756..1276d02 100644 --- a/chisel_nbdev/__init__.py +++ b/chisel_nbdev/__init__.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.5" diff --git a/docs/_data/sidebars/home_sidebar.yml b/docs/_data/sidebars/home_sidebar.yml index c05981b..e2440d4 100644 --- a/docs/_data/sidebars/home_sidebar.yml +++ b/docs/_data/sidebars/home_sidebar.yml @@ -9,6 +9,30 @@ entries: - output: web,pdf title: Overview url: / + - output: web,pdf + title: 'Export to scala sbt project ' + url: export_scala.html + - output: web,pdf + title: Synchronize and diff + url: sync_scala.html + - output: web,pdf + title: show_scaladoc + url: show_scaladoc.html + - output: web,pdf + title: Convert to html + url: export_scala2html.html + - output: web,pdf + title: Extract tests + url: test_scala.html + - output: web,pdf + title: Fix merge conflicts + url: merge.html + - output: web,pdf + title: Command line functions + url: cli.html + - output: web,pdf + title: Clean notebooks + url: clean_scala.html - output: web,pdf title: Creating a basic Chisel module to be imported by other files. url: ToImport.html diff --git a/docs/clean_scala.html b/docs/clean_scala.html new file mode 100644 index 0000000..877eae7 --- /dev/null +++ b/docs/clean_scala.html @@ -0,0 +1,463 @@ +--- + +title: Clean notebooks + + +keywords: fastai +sidebar: home_sidebar + +summary: "Strip notebooks from superfluous metadata" +description: "Strip notebooks from superfluous metadata" +nb_path: "nbs/07_clean_scala.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

To avoid pointless conflicts while working with jupyter notebooks (with different execution counts or cell metadata), it is recommended to clean the notebooks before committing anything (done automatically if you install the git hooks with nbdev_install_git_hooks). The following functions are used to do that.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

rm_execution_count[source]

rm_execution_count(o)

+
+

Remove execution count in o

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_output_data_vnd[source]

clean_output_data_vnd(o)

+
+

Remove application/vnd.google.colaboratory.intrinsic+json in data entries

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_cell_output[source]

clean_cell_output(cell)

+
+

Remove execution count in cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_cell[source]

clean_cell(cell, clear_all=False)

+
+

Clean cell by removing superfluous metadata or everything except the input if clear_all

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
tst = {'cell_type': 'code',
+       'execution_count': 26,
+       'metadata': {'hide_input': True, 'meta': 23},
+       'outputs': [{'execution_count': 2, 
+                    'data': {
+                        'application/vnd.google.colaboratory.intrinsic+json': {
+                            'type': 'string'},
+                        'plain/text': ['sample output',]
+                    },
+                    'output': 'super'}],
+       
+       'source': 'awesome_code'}
+tst1 = tst.copy()
+
+clean_cell(tst)
+test_eq(tst, {'cell_type': 'code',
+              'execution_count': None,
+              'metadata': {'hide_input': True},
+              'outputs': [{'execution_count': None, 
+                           'data': {'plain/text': ['sample output',]},
+                           'output': 'super'}],
+              'source': 'awesome_code'})
+
+clean_cell(tst1, clear_all=True)
+test_eq(tst1, {'cell_type': 'code',
+               'execution_count': None,
+               'metadata': {},
+               'outputs': [],
+               'source': 'awesome_code'})
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_cell[source]

clean_cell(cell, clear_all=False)

+
+

Clean cell by removing superfluous metadata or everything except the input if clear_all

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
tst2 = {
+       'metadata': {'tags':[]},
+       'outputs': [{
+                    'metadata': {
+                        'tags':[]
+                    }}],
+       
+          "source": [
+    ""
+   ]}
+clean_cell(tst2, clear_all=False)
+test_eq(tst2, {
+               'metadata': {},
+               'outputs': [{
+                    'metadata':{}}],
+               'source': []})
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_nb[source]

clean_nb(nb, clear_all=False)

+
+

Clean nb from superfluous metadata, passing clear_all to clean_cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
tst = {'cell_type': 'code',
+       'execution_count': 26,
+       'metadata': {'hide_input': True, 'meta': 23},
+       'outputs': [{'execution_count': 2,
+                    'data': {
+                        'application/vnd.google.colaboratory.intrinsic+json': {
+                            'type': 'string'},
+                        'plain/text': ['sample output',]
+                    },
+                    'output': 'super'}],
+       'source': 'awesome_code'}
+nb = {'metadata': {'kernelspec': 'some_spec', 'jekyll': 'some_meta', 'meta': 37},
+      'cells': [tst]}
+
+clean_nb(nb)
+test_eq(nb['cells'][0], {'cell_type': 'code',
+              'execution_count': None,
+              'metadata': {'hide_input': True},
+              'outputs': [{'execution_count': None, 
+                           'data': { 'plain/text': ['sample output',]},
+                           'output': 'super'}],
+              'source': 'awesome_code'})
+test_eq(nb['metadata'], {'kernelspec': 'some_spec', 'jekyll': 'some_meta'})
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_cr[source]

clean_cr(s)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
assert clean_cr(fr'a{BSLASH}r\nb{BSLASH}rc\n') == fr'a\nb\nc\n'
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Main function

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_clean_nbs[source]

nbdev_clean_nbs(fname:"A notebook name or glob to convert"=None, clear_all:"Clean all metadata and outputs"=False, disp:"Print the cleaned outputs"=False, read_input_stream:"Read input stram and not nb folder"=False)

+
+

Clean all notebooks in fname to avoid merge conflicts

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the all the notebooks in lib_folder are cleaned. You can opt in to fully clean the notebook by removing every bit of metadata and the cell outputs by passing clear_all=True. disp is only used for internal use with git hooks and will print the clean notebook instead of saving it. Same for read_input_stream that will read the notebook from the input stream instead of the file names.

+ +
+
+
+
+ + diff --git a/docs/cli.html b/docs/cli.html new file mode 100644 index 0000000..95fe542 --- /dev/null +++ b/docs/cli.html @@ -0,0 +1,609 @@ +--- + +title: Command line functions + + +keywords: fastai +sidebar: home_sidebar + +summary: "Console commands added by the nbdev library" +description: "Console commands added by the nbdev library" +nb_path: "nbs/06_cli.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

nbdev comes with the following commands. To use any of them, you must be in one of the subfolders of your project: they will search for the settings.ini recursively in the parent directory but need to access it to be able to work. Their names all begin with nbdev so you can easily get a list with tab completion.

+
    +
  • chisel_nbdev_build_docs builds the documentation from the notebooks
    • +
  • chisel_nbdev_build_lib builds the library from the notebooks
    • +
  • chisel_nbdev_bump_version increments version in settings.py by one
    • +
  • chisel_nbdev_clean_nbs removes all superfluous metadata form the notebooks, to avoid merge conflicts
    • +
  • chisel_nbdev_detach exports cell attachments to dest and updates references
    • +
  • chisel_nbdev_diff_nbs gives you the diff between the notebooks and the exported library
    • +
  • chisel_nbdev_fix_merge will fix merge conflicts in a notebook file
    • +
  • chisel_nbdev_install_git_hooks installs the git hooks that use the last two command automatically on each commit/merge
    • +
  • chisel_nbdev_nb2md converts a notebook to a markdown file
    • +
  • chisel_nbdev_new creates a new nbdev project
    • +
  • chisel_nbdev_read_nbs reads all notebooks to make sure none are broken
    • +
  • chisel_nbdev_test_nbs runs tests in notebooks
    • +
  • chisel_nbdev_trust_nbs trusts all notebooks (so that the HTML content is shown)
    • +
  • chisel_nbdev_update_lib propagates any change in the library back to the notebooks
    • +
+ +
+
+
+
+
+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_build_lib[source]

nbdev_build_lib(fname:"A notebook name or glob to convert"=None, bare:"Omit nbdev annotation comments (may break some functionality)."=False, recursive:"Search directories for notebooks recursively."=False)

+
+

Export notebooks matching fname to python modules

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the whole library is built from the notebooks in the lib_folder set in your settings.ini.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_update_lib[source]

nbdev_update_lib(fname:"A python filename or glob to convert"=None, silent:"Don't print results"=False)

+
+

Propagates any change in the modules matching fname to the notebooks that created them

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the whole library is treated. Note that this tool is only designed for small changes such as typo or small bug fixes. You can't add new cells in notebook from the library.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_diff_nbs[source]

nbdev_diff_nbs()

+
+

Prints the diff between an export of the library in notebooks and the actual modules

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

Running tests

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_test_nbs[source]

nbdev_test_nbs(fname:"A notebook name or glob to convert"=None, flags:"Space separated list of flags"=None, n_workers:"Number of workers to use"=None, verbose:"Print errors along the way"=True, timing:"Timing each notebook to see the ones are slow"=False, pause:"Pause time (in secs) between notebooks to avoid race conditions"=0.5)

+
+

Test in parallel the notebooks matching fname, passing along flags

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the whole library is tested from the notebooks in the lib_folder set in your settings.ini.

+ +
+
+
+
+
+

Building documentation

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_build_docs[source]

nbdev_build_docs(fname:"A notebook name or glob to convert"=None, force_all:"Rebuild even notebooks that haven't changed"=False, mk_readme:"Also convert the index notebook to README"=True, n_workers:"Number of workers to use"=None, pause:"Pause time (in secs) between notebooks to avoid race conditions"=0.5)

+
+

Build the documentation by converting notebooks matching fname to html

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the whole documentation is build from the notebooks in the lib_folder set in your settings.ini, only converting the ones that have been modified since the their corresponding html was last touched unless you pass force_all=True. The index is also converted to make the README file, unless you pass along mk_readme=False.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_nb2md[source]

nbdev_nb2md(fname:"A notebook file name to convert", dest:"The destination folder"='.', img_path:"Folder to export images to"='', jekyll:"To use jekyll metadata for your markdown file or not"=False)

+
+

Convert the notebook in fname to a markdown file

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_detach[source]

nbdev_detach(path_nb:"Path to notebook", dest:"Destination folder"='', use_img:"Convert markdown images to img tags"=False, replace:"Write replacement notebook back to path_bn"=True)

+
+

Export cell attachments to dest and update references

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

Other utils

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_read_nbs[source]

nbdev_read_nbs(fname:"A notebook name or glob to convert"=None)

+
+

Check all notebooks matching fname can be opened

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the all the notebooks in lib_folder are checked.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_trust_nbs[source]

nbdev_trust_nbs(fname:"A notebook name or glob to convert"=None, force_all:"Trust even notebooks that haven't changed"=False)

+
+

Trust notebooks matching fname

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

By default (fname left to None), the all the notebooks in lib_folder are trusted. To speed things up, only the ones touched since the last time this command was run are trusted unless you pass along force_all=True.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_fix_merge[source]

nbdev_fix_merge(fname:"A notebook filename to fix", fast:"Fast fix: automatically fix the merge conflicts in outputs or metadata"=True, trust_us:"Use local outputs/metadata when fast merging"=True)

+
+

Fix merge conflicts in notebook fname

+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

When you have merge conflicts after a git pull, the notebook file will be broken and won't open in jupyter notebook anymore. This command fixes this by changing the notebook to a proper json file again and add markdown cells to signal the conflict, you just have to open that notebook again and look for >>>>>>> to see those conflicts and manually fix them. The old broken file is copied with a .ipynb.bak extension, so is still accessible in case the merge wasn't successful.

+

Moreover, if fast=True, conflicts in outputs and metadata will automatically be fixed by using the local version if trust_us=True, the remote one if trust_us=False. With this option, it's very likely you won't have anything to do, unless there is a real conflict.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

bump_version[source]

bump_version(version, part=2)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_eq(bump_version('0.1.1'   ), '0.1.2')
+test_eq(bump_version('0.1.1', 1), '0.2.0')
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_bump_version[source]

nbdev_bump_version(part:"Part of version to bump"=2)

+
+

Increment version in settings.py by one

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Git hooks

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_install_git_hooks[source]

nbdev_install_git_hooks()

+
+

Install git hooks to clean/trust notebooks automatically

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This command installs git hooks to make sure notebooks are cleaned before you commit them to GitHub and automatically trusted at each merge. To be more specific, this creates:

+
    +
  • an executable '.git/hooks/post-merge' file that contains the command nbdev_trust_nbs
    • +
  • a .gitconfig file that uses nbev_clean_nbs has a filter/diff on all notebook files inside nbs_folder and a .gitattributes file generated in this folder (copy this file in other folders where you might have notebooks you want cleaned as well)
    • +
+ +
+
+
+
+
+

Starting a new project

+
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

extract_tgz[source]

extract_tgz(url, dest='.')

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_new[source]

nbdev_new()

+
+

Create a new nbdev project from the current git repo

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

nbdev_new is a command line tool that creates a new nbdev project from the current directory, which must be a cloned git repo.

+

After you run nbdev_new, please check the contents of settings.ini look good, and then run nbdev_build_lib.

+ +
+
+
+
+ + diff --git a/docs/export_scala.html b/docs/export_scala.html new file mode 100644 index 0000000..a91ee5f --- /dev/null +++ b/docs/export_scala.html @@ -0,0 +1,1362 @@ +--- + +title: Export to scala sbt project + + +keywords: fastai +sidebar: home_sidebar + +summary: "The functions that transform notebooks in a library (nearly 1:1 translation from nbdev 00_export.ipynb, with the goal of transforming notebooks using Jupyter Scala kernel (https://almond.sh) into importable, testable, and well documented Scala files. " +description: "The functions that transform notebooks in a library (nearly 1:1 translation from nbdev 00_export.ipynb, with the goal of transforming notebooks using Jupyter Scala kernel (https://almond.sh) into importable, testable, and well documented Scala files. " +nb_path: "nbs/00_export_scala.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
import nbdev.showdoc
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

The most important function defined in this module is notebooks2script, so you may want to jump to it before scrolling though the rest, which explain the details behind the scenes of the conversion from notebooks to library. The main things to remember are:

+
    +
  • put # export on each cell you want exported
    • +
  • put # exports on each cell you want exported with the source code shown in the docs
    • +
  • put # exporti on each cell you want exported without it being added to __all__, and without it showing up in the docs.
    • +
  • one cell should contain # default_exp followed by the name of the module (with points for submodules and without the py extension) everything should be exported in (if one specific cell needs to be exported in a different module, just indicate it after #export: #export special.module)
    • +
  • all left members of an equality, functions and classes will be exported and variables that are not private will be put in the __all__ automatically
    • +
  • to add something to __all__ if it's not picked automatically, write an exported cell with something like #add2all "my_name"
    • +
+ +
+
+
+
+
+

Basic foundations

+
+
+
+
+
+

For bootstrapping nbdev we have a few basic foundations defined in imports, which we test a show here. First, a simple config file class, Config that read the content of your settings.ini file and make it accessible:

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

Config[source]

Config(cfg_name='settings.ini')

+
+

Reading and writing settings.ini

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cfg = Config(cfg_name='settings.ini')
+test_eq(cfg.lib_name, 'chisel_nbdev')
+test_eq(cfg.git_url, "https://github.com/ucsc-vama/chisel_nbdev")
+test_eq(cfg.path("lib_path"), Path.cwd().parent/'chisel_nbdev')
+test_eq(cfg.path("nbs_path"), Path.cwd())
+# test_eq(cfg.path("doc_path"), Path.cwd().parent/'docs')
+# test_eq(cfg.custom_sidebar, 'False')
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Reading a notebook

+
+
+
+
+
+

What's a notebook?

+
+
+
+
+
+

A jupyter notebook is a json file behind the scenes. We can just read it with the json module, which will return a nested dictionary of dictionaries/lists of dictionaries, but there are some small differences between reading the json and using the tools from nbformat so we'll use this one.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

read_nb[source]

read_nb(fname)

+
+

Read the notebook in fname.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

fname can be a string or a pathlib object.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_nb = read_nb('test.ipynb')
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

The root has four keys: cells contains the cells of the notebook, metadata some stuff around the version of python used to execute the notebook, nbformat and nbformat_minor the version of nbformat.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_nb.keys()
+
+ +
+
+
+ +
+
+ +
+ + + +
+
dict_keys(['cells', 'metadata', 'nbformat', 'nbformat_minor'])
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_nb['metadata']
+
+ +
+
+
+ +
+
+ +
+ + + +
+
{'kernelspec': {'display_name': 'Scala', 'language': 'scala', 'name': 'scala'},
+ 'language_info': {'codemirror_mode': 'text/x-scala',
+  'file_extension': '.scala',
+  'mimetype': 'text/x-scala',
+  'name': 'scala',
+  'nbconvert_exporter': 'script',
+  'version': '2.12.10'}}
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
f"{test_nb['nbformat']}.{test_nb['nbformat_minor']}"
+
+ +
+
+
+ +
+
+ +
+ + + +
+
'4.4'
+
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

The cells key then contains a list of cells. Each one is a new dictionary that contains entries like the type (code or markdown), the source (what is written in the cell) and the output (for code cells).

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_nb['cells'][0]
+
+ +
+
+
+ +
+
+ +
+ + + +
+
{'cell_type': 'markdown',
+ 'metadata': {},
+ 'source': '# Test Chisel notebook\n> An example that defines a module over multiple notebook cells.'}
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

is_scala_nb[source]

is_scala_nb(fname)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
assert(is_scala_nb('test.ipynb'))
+assert(not is_scala_nb('00_export_scala.ipynb'))
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Finding patterns

+
+
+
+
+
+

The following functions are used to catch the flags used in the code cells.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

check_re[source]

check_re(cell, pat, code_only=True)

+
+

Check if cell contains a line with regex pat

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

pat can be a string or a compiled regex. If code_only=True, this function ignores non-code cells, such as markdown.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
cell = test_nb['cells'][2].copy()
+# print(cell)
+assert check_re(cell, '//export') is not None
+assert check_re(cell, re.compile('//export')) is not None
+assert check_re(cell, '# bla') is None
+cell['cell_type'] = 'markdown'
+assert check_re(cell, '//export') is None # don't export markdown
+assert check_re(cell, '//export', code_only=False) is not None # unless specified
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

check_re_multi[source]

check_re_multi(cell, pats, code_only=True)

+
+

Check if cell contains a line matching any regex in pats, returning the first match found

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cell = test_nb['cells'][2].copy()
+cell['source'] = "a b c"
+# print(cell)
+assert check_re(cell, 'a') is not None
+assert check_re(cell, 'd') is None
+# show that searching with patterns ['d','b','a'] will match 'b'
+# i.e. 'd' is not found and we don't search for 'a'
+assert check_re_multi(cell, ['d','b','a']).span() == (2,3)
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This function returns a regex object that can be used to find nbdev flags in multiline text

+
    +
  • body regex fragment to match one or more flags,
    • +
  • n_params number of flag parameters to match and catch (-1 for any number of params; (0,1) for 0 for 1 params),
    • +
  • comment explains what the compiled regex should do.
    • +
+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

is_export[source]

is_export(cell, default)

+
+

Check if cell is to be exported and returns the name of the module to export it if provided

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

is_export returns;

+
    +
  • a tuple of ("module name", "external boolean" (False for an internal export)) if cell is to be exported or
    • +
  • None if cell will not be exported.
    • +
+

The cells to export are marked with //export///exporti///exports, potentially with a module name where we want it exported. The default module is given in a cell of the form //default_exp bla inside the notebook (usually at the top), though in this function, it needs the be passed (the final script will read the whole notebook to find it).

+
    +
  • a cell marked with //export///exporti///exports will be exported to the default module
    • +
  • an exported cell marked with special.module appended will be exported in special.module (located in lib_name/special/module.py)
    • +
  • a cell marked with //export will have its signature added to the documentation
    • +
  • a cell marked with //exports will additionally have its source code added to the documentation
    • +
  • a cell marked with //exporti will not show up in the documentation, and will also not be added to __all__.
    • +
+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

is_non_def_export[source]

is_non_def_export(cell)

+
+

Check if cell is to be exported to special (non-defualt) module returns the name of the module to export

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cell = test_nb['cells'][2].copy()
+cell['source'] = "// export ModA"
+test_eq(is_non_def_export(cell), ('ModA', True))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cell = test_nb['cells'][2].copy()
+test_eq(is_export(cell, 'export'), ('export', True))
+cell['source'] = "// exports"
+test_eq(is_export(cell, 'export'), ('export', True))
+cell['source'] = "// exporti"
+test_eq(is_export(cell, 'export'), ('export', False))
+cell['source'] = "// export mod"
+test_eq(is_export(cell, 'export'), ('mod', True))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

find_default_export[source]

find_default_export(cells)

+
+

Find in cells the default export module.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

find_non_default_exports[source]

find_non_default_exports(cells)

+
+

Find in cells all non default export modules: //export my_mod_name

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_eq(find_non_default_exports(test_nb['cells']), ['NewScript', 'NewScript2'])
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Stops at the first cell containing // default_exp (if there are several) and returns the value behind. Returns None if there are no cell with that code.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_eq(find_default_export(test_nb['cells']), 'test')
+assert find_default_export(test_nb['cells'][3:]) is None
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Listing all exported objects

+
+
+
+
+
+

Until now the above code has been verbatim to the 00_export.ipynb from nbdev minus changes to regexes swapping '#' to '//'. +Now Scala syntax parsing is required as well as a target build infrastructure (sbt, mill, or Ammonite scripts).

+ +
+
+
+
+
+

For documentation we need to extract out the names of our classes, objects, methods, etc including:

+
    +
  • def
    • +
  • object
    • +
  • class | abstract class |
    • +
  • case class
    • +
  • trait
    • +
  • sealed trait
    • +
  • abstract
    • +
  • package
    • +
  • override
    • +
  • private?
    • +
  • implicit
    • +
  • protected
    • +
  • import
    • +
  • extends
    • +
  • final
    • +
+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

export_names[source]

export_names(code)

+
+

Find the names of objects, functions or classes defined in code that are exported.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This function finds all of the function/class/object names.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_eq(export_names("def my_func(x: Int): Unit = {\n\tprint(x)\n}\n class MyClass(){}"), ["my_func", "MyClass"])
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Create the library

+
+
+
+
+
+

Saving an index

+
+
+
+
+
+

To be able to build back a correspondence between functions and the notebooks they are defined in, we need to store an index. It's done in the private module _nbdev inside your library, and the following function are used to define it.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

reset_nbdev_module[source]

reset_nbdev_module()

+
+

Create a skeleton for _nbdev

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

get_nbdev_module[source]

get_nbdev_module()

+
+

Reads _nbdev

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

save_nbdev_module[source]

save_nbdev_module(mod)

+
+

Save mod inside _nbdev

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Create the modules

+
+
+
+
+
+

return_type tells us if the tuple returned will contain lists of lines or strings with line breaks.

+

We treat the first comment line as a flag +

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

split_flags_and_code[source]

split_flags_and_code(cell, return_type=list)

+
+

Splits the source of a cell into 2 parts and returns (flags, code)

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
def _test_split_flags_and_code(expected_flags, expected_code):
+    cell = nbformat.v4.new_code_cell('\n'.join(expected_flags + expected_code))
+    test_eq((expected_flags, expected_code), split_flags_and_code(cell))
+    expected=('\n'.join(expected_flags), '\n'.join(expected_code))
+    test_eq(expected, split_flags_and_code(cell, str))
+    
+_test_split_flags_and_code([
+    '//export'],
+    ['// TODO: write this function',
+    'def func(x) = ???'])
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

create_mod_file[source]

create_mod_file(fname, nb_path, bare=False)

+
+

Create a module file for fname.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

A new module filename is created each time a notebook has a cell marked with #default_exp. In your collection of notebooks, you should only have one notebook that creates a given module since they are re-created each time you do a library build (to ensure the library is clean). Note that any file you create manually will never be overwritten (unless it has the same name as one of the modules defined in a #default_exp cell) so you are responsible to clean up those yourself.

+

fname is the notebook that contained the #default_exp cell.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

create_mod_files[source]

create_mod_files(files, to_dict=False, bare=False)

+
+

Create mod files for default exports found in files

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Create module files for all #default_export flags found in files and return a list containing the names of modules created.

+

Note: The number if modules returned will be less that the number of files passed in if files do not #default_export.

+

By creating all module files before calling _notebook2script, the order of execution no longer matters - so you can now export to a notebook that is run "later".

+

You might still have problems when

+
    +
  • converting a subset of notebooks or
    • +
  • exporting to a module that does not have a #default_export yet
    • +
+

in which case _notebook2script will print warnings like;

+ +
Warning: Exporting to "core.py" but this module is not part of this build
+

If you see a warning like this

+
    +
  • and the module file (e.g. "core.py") does not exist, you'll see a FileNotFoundError
    • +
  • if the module file exists, the exported cell will be written - even if the exported cell is already in the module file
    • +
+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

update_baseurl[source]

update_baseurl()

+
+

Add or update baseurl in _config.yml for the docs

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbglob[source]

nbglob(fname=None, recursive=False)

+
+

Find all notebooks in a directory given a glob. Ignores hidden directories and filenames starting with _

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
assert not nbglob(recursive=True).filter(lambda x: '.ipynb_checkpoints' in str(x))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

scala_notebook2script[source]

scala_notebook2script(fname=None, silent=False, to_dict=False, bare=False, recursive=False)

+
+

Convert notebooks matching fname to modules

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Finds cells starting with #export and puts them into the appropriate module. If fname is not specified, this will convert all notebook not beginning with an underscore in the nb_folder defined in setting.ini. Otherwise fname can be a single filename or a glob expression.

+

silent makes the command not print any statement and to_dict is used internally to convert the library to a dictionary.

+ +
+
+
+
+ + diff --git a/docs/export_scala2html.html b/docs/export_scala2html.html new file mode 100644 index 0000000..af276c5 --- /dev/null +++ b/docs/export_scala2html.html @@ -0,0 +1,2298 @@ +--- +toc: true +title: Convert to html + + +keywords: fastai +sidebar: home_sidebar + +summary: "The functions that transform the dev notebooks in the documentation of the library" +description: "The functions that transform the dev notebooks in the documentation of the library" +nb_path: "nbs/03_export_scala2html.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

The most important function defined in this module is notebook2html, so you may want to jump to it before scrolling though the rest, which explain the details behind the scenes of the conversion from notebooks to the html documentation. The main things to remember are:

+
    +
  • put #hide at the top of any cell you want to completely hide in the docs
    • +
  • use the hide input jupyter extension to hide the input of some cells (by default all show_doc cells have that marker added)
    • +
  • you can define some jekyll metadata in the markdown cell with the title, see get_metadata
    • +
  • use backticks for terms you want automatic links to be found, but use <code> and </code> when you have homonyms and don't want those links
    • +
  • you can define the default toc level of classes with #default_cls_lvl followed by a number (default is 2)
    • +
  • you can add jekyll warnings, important or note banners with appropriate block quotes (see add_jekyll_notes)
    • +
  • put any images you want to use in the images folder of your notebook folder, they will be automatically copied over to the docs folder
    • +
  • put #hide_input at the top of a cell if you don't want code to be shown in the docs
      +
    • cells containing #export or show_doc have their code hidden automatically
      • +
    +
    • +
  • put #hide_output at the top of a cell if you don't want output to be shown in the docs
    • +
  • use #collapse_input or #collapse_output to include code or output in the docs under a collapsable element
    • +
+ +
+
+
+
+
+

Preprocessing notebook

+
+
+
+
+
+

Cell processors

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

class HTMLParseAttrs[source]

HTMLParseAttrs(convert_charrefs=True) :: HTMLParser

+
+

Simple HTML parser which stores any attributes in attrs dict

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
h = HTMLParseAttrs()
+t = h('<img src="src" alt="alt" width="700" caption="cap" />')
+test_eq(t['width'], '700')
+test_eq(t['src'  ], 'src')
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
t['width'] = '600'
+test_eq(h.show(), '<img src="src" alt="alt" width="600" caption="cap" />')
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

The following functions are applied on individual cells as a preprocessing step before the conversion to html.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

remove_widget_state[source]

remove_widget_state(cell)

+
+

Remove widgets in the output of cells

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Those outputs usually can't be rendered properly in html.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

upd_metadata[source]

upd_metadata(cell, key, value=True)

+
+

Sets key to value on the metadata of cell without replacing metadata

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

hide_cells[source]

hide_cells(cell)

+
+

Hide inputs of cell that need to be hidden

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This concerns all the cells with #export or #hide_input flags and all the cells containing a show_doc for a function or class.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
for source in ['show_doc(read_nb)', '// export\nfrom local.core import *', '// hide_input\n2+2',
+              'line1\n show_doc (read_nb) \nline3', '// export with.mod\nfrom local.core import *']:
+    cell = {'cell_type': 'code', 'source': source}
+    cell1 = hide_cells(cell.copy())
+    assert 'metadata' in cell1
+    assert 'hide_input' in cell1['metadata']
+    assert cell1['metadata']['hide_input']
+
+flag = '// exports'
+cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core2 import *'}
+test_eq(hide_cells(cell.copy()), cell)
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

This concerns all the cells with #hide_output.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
for source in ['// hide-output\nfrom local.core import *', '// hide_output\n2+2']:
+    cell = {'cell_type': 'code', 'source': source}
+    cell1 = hide_cells(cell.copy())
+    assert 'metadata' in cell1
+    assert 'hide_output' in cell1['metadata']
+    assert cell1['metadata']['hide_output']
+
+cell = {'cell_type': 'code', 'source': '// hide-outputs\nfrom local.core import *'}
+test_eq(hide_cells(cell.copy()), cell)
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

clean_exports[source]

clean_exports(cell)

+
+

Remove all flags from code cells

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

The rest of the cell is displayed without any modification.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
flag = '// exports'
+cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core import *'}
+test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
+cell['cell_type'] = 'markdown'
+test_eq(clean_exports(cell.copy()), cell)
+cell = {'cell_type': 'code', 'source': f'{flag} core\nfrom local.core import *'}
+test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
+    
+cell = {'cell_type': 'code', 'source': f'// comment \n// exports\nprint("something")'}
+test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': '// exports\nprint("something")'})
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

treat_backticks[source]

treat_backticks(cell)

+
+

Add links to backticks words in cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cell = {'cell_type': 'markdown', 'source': 'This is a `DocsTestClass`'}
+# test_eq(treat_backticks(cell), {'cell_type': 'markdown',
+#     'source': 'This is a [`DocsTestClass`](/export.html#DocsTestClass)'})
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

add_jekyll_notes[source]

add_jekyll_notes(cell)

+
+

Convert block quotes to jekyll notes in cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Supported styles are Warning, Note Tip and Important:

+

Typing > Warning: There will be no second warning! will render in the docs: +{% include warning.html content='There will be no second warning!' %} +Typing > Important: Pay attention! It's important. will render in the docs: +{% include important.html content='Pay attention! It’s important.' %} +Typing > Tip: This is my tip. will render in the docs: +{% include tip.html content='This is my tip.' %} +Typing > Note: Take note of this. will render in the docs: +{% include note.html content='Take note of this.' %} +Typing > Note: A doc link to [`add_jekyll_notes`](/export_scala2html.html#add_jekyll_notes) should also work fine. will render in the docs: +{% include note.html content='A doc link to add_jekyll_notes should also work fine.' %}

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
m=_re_image.search('![Alt](images/logo.png)')
+test_eq(m.groups(), ('![Alt](', 'images/logo.png', ')', None))
+# using ) or whitespace to close the group means we don't need a special case for captions
+m=_re_image.search('![Alt](images/logo.png "caption (something)")')
+test_eq(m.groups(), ('![Alt](', 'images/logo.png', '', None))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

copy_images[source]

copy_images(cell, fname, dest, jekyll=True)

+
+

Copy images referenced in cell from fname parent folder to dest folder

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This is to ensure that all images defined in nbs_folder/images and used in notebooks are copied over to doc_folder/images.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
dest_img = Config().path("doc_path")/'images'/'logo.jpg'
+cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.jpg)'}
+try:
+    copy_images(cell, Path('ToImport.ipynb'), Config().path("doc_path"))
+    test_eq(cell["source"], 'Text\n![Alt](/images/logo.jpg)')
+    #Image has been copied
+    assert dest_img.exists()
+    cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.jpg "caption (something)")'}
+    copy_images(cell, Path('ToImport.ipynb'), Config().path("doc_path"))
+    test_eq(cell["source"], 'Text\n![Alt](/images/logo.jpg "caption (something)")')
+finally: dest_img.unlink()
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

adapt_img_path[source]

adapt_img_path(cell, fname, dest, jekyll=True)

+
+

Adapt path of images referenced in cell from fname to work in folder dest

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This function is slightly different as it ensures that a notebook convert to a file that will be placed in dest will have the images location updated. It is used for the README.md file (generated automatically from the index) since the images are copied inside the github repo, but in general, you should make sure your images are going to be accessible from the location your file ends up being.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
cell = {'cell_type': 'markdown', 'source': 'Text\n![Alt](images/logo.png)'}
+cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
+test_eq(cell1['source'], 'Text\n![Alt](nbs/images/logo.png)')
+
+cell = {'cell_type': 'markdown', 'source': 'Text\n![Alt](http://site.logo.png)'}
+cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
+test_eq(cell1['source'], 'Text\n![Alt](http://site.logo.png)')
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Escape Latex in liquid

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

escape_latex[source]

escape_latex(cell)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cell = {'cell_type': 'markdown', 
+        'source': 'lala\n$$equation$$\nlala'}
+cell = escape_latex(cell)
+test_eq(cell['source'], 'lala\n{% raw %}\n$$equation$$\n{% endraw %}\nlala')
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Collapsable Code Cells

+
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

collapse_cells[source]

collapse_cells(cell)

+
+

Add a collapse button to inputs or outputs of cell in either the open or closed position

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+
    +
  • Placing #collapse_input open in a code cell will include your code under a collapsable element that is open by default.
    • +
+ +
+
+
+ {% raw %} + +
+
+ +
+
+
print('This code cell is not collapsed by default but you can collapse it to hide it from view!')
+print("Note that the output always shows with `%collapse_input`.")
+
+ +
+
+
+ +
+
+ +
+ +
+
This code cell is not collapsed by default but you can collapse it to hide it from view!
+Note that the output always shows with `%collapse_input`.
+
+
+
+ +
+
+ +
+ {% endraw %} + +
+
+
    +
  • Placing #collapse_input in a code cell will include your code in a collapsable element that is closed by default. For example:
    • +
+ +
+
+
+ {% raw %} + +
+
+ +
+
+
print('The code cell that produced this output is collapsed by default but you can expand it!')
+
+ +
+
+
+ +
+
+ +
+ +
+
The code cell that produced this output is collapsed by default but you can expand it!
+
+
+
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
print('The input of this cell is visible as usual.\nHowever, the OUTPUT of this cell is collapsed by default but you can expand it!')
+
+ +
+
+
+
+ + + +
+
+ +
+ +
+
The input of this cell is visible as usual.
+However, the OUTPUT of this cell is collapsed by default but you can expand it!
+
+
+
+ +
+
+ +
+
+ {% endraw %} + +
+
+

Preprocessing the list of cells

+
+
+
+
+
+

The following functions are applied to the entire list of cells of the notebook as a preprocessing step before the conversion to html.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

remove_hidden[source]

remove_hidden(cells)

+
+

Remove in cells the ones with a flag //hide, //default_exp, //default_cls_lvl or //exporti

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
cells = [{'cell_type': 'code', 'source': source, 'hide': hide} for hide, source in [
+    (False, '// export\nfrom local.core import *'), 
+    (False, '// exporti mod file'), # Note: this used to get removed but we're more strict now
+    (True, '// hide\nfrom local.core import *'),
+    (False, '// hide_input\nfrom local.core import *'),
+    (False, '//exports\nsuper code'),
+    (True, '//default_exp notebook.export'),
+    (False, 'show_doc(read_nb)'),
+    (False, '//hide (last test of to_concat)'),
+    (True, '// exporti\n1 + 1')]] + [
+    {'cell_type': 'markdown', 'source': source, 'hide': hide} for hide, source in [
+    (False, '//hide_input\nnice'), 
+    (True, '//hide\n\nto hide')]]
+
+for a,b in zip([cell for cell in cells if not cell['hide']], remove_hidden(cells)):
+    test_eq(a,b)
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

find_default_level[source]

find_default_level(cells)

+
+

Find in cells the default class level.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
tst_nb = read_nb('ToImport.ipynb')
+test_eq(find_default_level(tst_nb['cells']), 3)
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nb_code_cell[source]

nb_code_cell(source)

+
+

A code cell (as a dict) containing source

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

A scala notebook will not have any python imports in scope. This _import_str must also be included when a new show_docs cell is added. This way when we execute just the doc cells in a scala notebook, we will be able to generate the markdown from show_doc.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

add_show_docs[source]

add_show_docs(cells, cls_lvl=None)

+
+

Add show_doc for each exported function or class

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This only adds cells with a show_doc for non-documented functions, so if you add yourself a show_doc cell (because you want to change one of the default argument), there won't be any duplicates.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
for i,cell in enumerate(tst_nb['cells']):
+    if cell['source'].startswith('//export\nclass Add'): break
+tst_cells = [c.copy() for c in tst_nb['cells'][i-1:i+1]]
+added_cells = add_show_docs(tst_cells, cls_lvl=3)
+test_eq(len(added_cells), 3)
+test_eq(added_cells[0], tst_nb['cells'][i-1])
+test_eq(added_cells[2], tst_nb['cells'][i])
+test_eq(added_cells[1], _show_doc_cell('Add', cls_lvl=3))
+test_eq(added_cells[1]['source'], f"{_import_str}show_doc('Add', default_cls_level=3)")
+
+for flag in ['//export', '//exports']:
+    for show_doc_source in [
+            ('show_doc(my_func)', 'show_doc(my_func, title_level=3)')]:
+        #Check show_doc isn't added if it was already there.
+        tst_cells1 = [{'cell_type':'code', 'source': f'{flag}\ndef my_func(x):\n    return x'},
+                      {'cell_type':'code', 'source': show_doc_source[0]}]
+        test_eq(add_show_docs(tst_cells1), tst_cells1)
+        #Check show_doc is added
+        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))
+        tst_cells1 = [{'cell_type':'code', 'source': f'{flag} with.mod\ndef my_func(x):\n    return x'},
+                      {'cell_type':'markdown', 'source': 'Some text'},
+                      {'cell_type':'code', 'source': show_doc_source[1]}]
+        test_eq(add_show_docs(tst_cells1), tst_cells1)
+        #Check show_doc is added when using mod export
+        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

remove_fake_headers[source]

remove_fake_headers(cells)

+
+

Remove in cells the fake header

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

You can fake headers in your notebook to navigate them more easily with collapsible headers, just make them finish with a dash and they will be removed. One typical use case is to have a header of level 2 with the name of a class, since the show_doc cell of that class will create the same anchor, you need to have the one you created manually disappear to avoid any duplicate.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
cells = [{'cell_type': 'markdown',
+          'metadata': {},
+          'source': '### Fake-'}] + tst_nb['cells'][:10]
+cells1 = remove_fake_headers(cells)
+test_eq(len(cells1), len(cells)-1)
+test_eq(cells1[0], cells[1])
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

remove_empty[source]

remove_empty(cells)

+
+

Remove in cells the empty cells

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Grabbing metadata

+
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

get_metadata[source]

get_metadata(cells)

+
+

Find the cell with title and summary in cells.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

In the markdown cell with the title, you can add the summary as a block quote (just put an empty block quote for an empty summary) and a list with any additional metadata you would like to add, for instance:

+ +
# Title
+> Awesome summary
+- toc:False
+

The toc: False metadata will prevent the table of contents from showing on the page.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
tst_nb = read_nb('ToImport.ipynb')
+test_eq(get_metadata(tst_nb['cells']), {
+    'keywords': 'chisel',
+    'summary': 'This file imports the necessary Chisel dependencies and defines an Add module that future notebooks will import.',
+    'title': 'Creating a basic Chisel module to be imported by other files.'})
+
+#The cell with the metada is popped out, so if we do it a second time we get the default.
+test_eq(get_metadata(tst_nb['cells']), {'keywords': 'chisel', 'title'   : 'Title'})
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Executing show_doc cells

+
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

class ExecuteShowDocPreprocessor[source]

ExecuteShowDocPreprocessor(*args, **kwargs) :: ExecutePreprocessor

+
+

An ExecutePreprocessor that only executes show_doc and import cells

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Cells containing:

+
    +
  • a zero indented call to notebook2script
    • +
+

are not run while building docs. This avoids failures caused by importing empty or partially built modules.

+

Cells containing:

+
    +
  • show_doc (which could be indented) or
    • +
  • a "library import" (zero indent import from current library) e.g. from LIB_NAME.core import *
    • +
+

are executed and must run without error. If running these cells raises an exception, the build will stop.

+

Cells containing zero indented imports. e.g.

+
    +
  • from module import * or
    • +
  • import module
    • +
+

are executed but errors will not stop the build.

+

If you need to show_doc something, please make sure it's imported via a cell that does not depend on previous cells being run. The easiest way to do this is to use a cell that contains nothing but imports.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

execute_nb[source]

execute_nb(nb, mod=None, metadata=None, show_doc_only=True)

+
+

Execute nb (or only the show_doc cells) with metadata

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Converting bibtex citations

+
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

cite2link(cell)

+
+

Creates links from \cite{} to Reference section generated by jupyter_latex_envs

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

jupyter_latex_envs is a jupyter extension https://github.com/jfbercher/jupyter_latex_envs.

+

You can find relevant section here +Note, that nbdev now only supports [<a class="latex_cit" id="call-" href="#cit-"></a>] conversion and not the rest, e.g., \figure{} and so on.

+ +
+
+
+
+
+

It's important to execute all show_doc cells before exporting the notebook to html because some of them have just been added automatically or others could have outdated links.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
fake_nb = {k:v for k,v in tst_nb.items() if k != 'cells'}
+fake_nb['cells'] = [tst_nb['cells'][0].copy()] + added_cells
+fake_nb = execute_nb(fake_nb, mod='export')
+assert len(fake_nb['cells'][-2]['outputs']) > 0
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Filling templates

+
+
+
+
+
+

The following functions automatically adds jekyll templates if they are missing.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

write_tmpl[source]

write_tmpl(tmpl, nms, cfg, dest)

+
+

Write tmpl to dest (if missing) filling in nms in template using dict cfg

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

write_tmpls[source]

write_tmpls()

+
+

Write out _config.yml and _data/topnav.yml using templates

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
store_true()
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_build_lib[source]

nbdev_build_lib(fname:"A notebook name or glob to convert"=None, bare:"Omit nbdev annotation comments (may break some functionality)."=False, recursive:"Search directories for notebooks recursively."=False)

+
+

Export notebooks matching fname to python modules

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Conversion

+
+
+
+ {% raw %} + +
+
+ +
+
+
__file__ = str(Config().path("lib_path")/'export_scala2html.py')
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_exporter[source]

nbdev_exporter(template_file=None)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

convert_nb[source]

convert_nb(fname, template_file=None, exporter=None, dest=None, execute=True)

+
+

Convert a notebook fname to html file in dest_path.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

notebook2html[source]

notebook2html(fname=None, force_all=False, n_workers=None, template_file=None, exporter=None, dest=None, pause=0, execute=True)

+
+

Convert all notebooks matching fname to html files

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Hide cells starting with #export and only leaves the prose and the tests. If fname is not specified, this will convert all notebooks not beginning with an underscore in the nb_folder defined in setting.ini. Otherwise fname can be a single filename or a glob expression.

+

By default, only the notebooks that are more recent than their html counterparts are modified, pass force_all=True to change that behavior.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

convert_md[source]

convert_md(fname, dest_path, img_path='docs/images/', jekyll=True)

+
+

Convert a notebook fname to a markdown file in dest_path.

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This is used to convert the index into the README.md.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
t = '![screenshot](attachment:image.png)'
+test_eq(_re_att_ref.match(t).groups(), ('screenshot', None))
+
+t = '![screenshot](attachment:image.png "Deploying to Binder")'
+test_eq(_re_att_ref.match(t).groups(), ('screenshot', "Deploying to Binder"))
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_detach[source]

nbdev_detach(path_nb:"Path to notebook", dest:"Destination folder"='', use_img:"Convert markdown images to img tags"=False, replace:"Write replacement notebook back to path_bn"=True)

+
+

Export cell attachments to dest and update references

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

make_readme[source]

make_readme()

+
+

Convert the index notebook to README.md

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_build_docs[source]

nbdev_build_docs(fname:"A notebook name or glob to convert"=None, force_all:"Rebuild even notebooks that haven't changed"=False, mk_readme:"Also convert the index notebook to README"=True, n_workers:"Number of workers to use"=None, pause:"Pause time (in secs) between notebooks to avoid race conditions"=0.5)

+
+

Build the documentation by converting notebooks matching fname to html

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_nb2md[source]

nbdev_nb2md(fname:"A notebook file name to convert", dest:"The destination folder"='.', img_path:"Folder to export images to"='', jekyll:"To use jekyll metadata for your markdown file or not"=False)

+
+

Convert the notebook in fname to a markdown file

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

create_default_sidebar[source]

create_default_sidebar()

+
+

Create the default sidebar for the docs website

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

The default sidebar lists all html pages with their respective title, except the index that is named "Overview". To build a custom sidebar, set the flag custom_sidebar in your settings.ini to True then change the sidebar.json file in the doc_folder to your liking. Otherwise, the sidebar is updated at each doc build.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

make_sidebar[source]

make_sidebar()

+
+

Making sidebar for the doc website form the content of doc_folder/sidebar.json

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
nbdev_build_docs(force_all=True)
+
+ +
+
+
+ +
+
+ +
+ +
+
+---------------------------------------------------------------------------
+EOFError                                  Traceback (most recent call last)
+~/code/notebooks/chisel_nbdev/chisel_nbdev/export_scala2html.py in <module>
+----> 1 nbdev_build_docs(force_all=True)
+
+~/code/notebooks/chisel_nbdev/chisel_nbdev/export_scala2html.py in nbdev_build_docs(fname, force_all, mk_readme, n_workers, pause)
+      7                      pause:Param("Pause time (in secs) between notebooks to avoid race conditions", float)=0.5):
+      8     "Build the documentation by converting notebooks matching `fname` to html"
+----> 9     notebook2html(fname=fname, force_all=force_all, n_workers=n_workers, pause=pause)
+     10     if fname is None: make_sidebar()
+     11     if mk_readme: make_readme()
+
+~/code/notebooks/chisel_nbdev/chisel_nbdev/export_scala2html.py in notebook2html(fname, force_all, n_workers, cls, template_file, exporter, dest, pause, execute)
+     21     if len(files)==0: print("No notebooks were modified")
+     22     else:
+---> 23         passed = parallel(_notebook2html, files, n_workers=n_workers, cls=cls,
+     24                           template_file=template_file, exporter=exporter, dest=dest, pause=pause, execute=execute)
+     25 #         passed = [_notebook2html(f, cls=cls, template_file=template_file, exporter=exporter,
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/fastcore/parallel.py in parallel(f, items, n_workers, total, progress, pause, threadpool, timeout, chunksize, *args, **kwargs)
+    100     pool = ThreadPoolExecutor if threadpool else ProcessPoolExecutor
+    101     with pool(n_workers, pause=pause) as ex:
+--> 102         r = ex.map(f,items, *args, timeout=timeout, chunksize=chunksize, **kwargs)
+    103         if progress and progress_bar:
+    104             if total is None: total = len(items)
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/fastcore/parallel.py in map(self, f, items, timeout, chunksize, *args, **kwargs)
+     83 
+     84     def map(self, f, items, *args, timeout=None, chunksize=1, **kwargs):
+---> 85         self.lock = Manager().Lock()
+     86         g = partial(f, *args, **kwargs)
+     87         if self.not_parallel: return map(g, items)
+
+/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/multiprocessing/context.py in Manager(self)
+     55         from .managers import SyncManager
+     56         m = SyncManager(ctx=self.get_context())
+---> 57         m.start()
+     58         return m
+     59 
+
+/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/multiprocessing/managers.py in start(self, initializer, initargs)
+    581         # get address of server
+    582         writer.close()
+--> 583         self._address = reader.recv()
+    584         reader.close()
+    585 
+
+/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/multiprocessing/connection.py in recv(self)
+    248         self._check_closed()
+    249         self._check_readable()
+--> 250         buf = self._recv_bytes()
+    251         return _ForkingPickler.loads(buf.getbuffer())
+    252 
+
+/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/multiprocessing/connection.py in _recv_bytes(self, maxsize)
+    412 
+    413     def _recv_bytes(self, maxsize=None):
+--> 414         buf = self._recv(4)
+    415         size, = struct.unpack("!i", buf.getvalue())
+    416         if size == -1:
+
+/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/multiprocessing/connection.py in _recv(self, size, read)
+    381             if n == 0:
+    382                 if remaining == size:
+--> 383                     raise EOFError
+    384                 else:
+    385                     raise OSError("got end of file during message")
+
+EOFError:
+
+
+ +
+
+ +
+ {% endraw %} + +
+ + diff --git a/docs/index.html b/docs/index.html index 829ba31..ce48575 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,7 +3,7 @@ title: chisel-nbdev -keywords: chisel +keywords: fastai sidebar: home_sidebar summary: "The goal of this repo is to provide a seamless way to develop Chisel code in a Jupyter Notebook environment." @@ -38,12 +38,6 @@

Requirements (mac) + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

When working with jupyter notebooks (which are json files behind the scenes) and GitHub, it is very common that a merge conflict (that will add new lines in the notebook source file) will break some notebooks you are working on. This module defines the function fix_conflicts to fix those notebooks for you, and attempt to automatically merge standard conflicts. The remaining ones will be delimited by markdown cells like this:

+ +
+
+
+
+
+

{% include image.html alt="Fixed notebook" width="700" caption="A notebook fixed after a merged conflict. The file couldn't be opened before the command was run, but after it the conflict is highlighted by markdown cells." max-width="700" file="/images/merge.PNG" %}

+ +
+
+
+
+
+

Walk cells

+
+
+
+
+
+

This is an example of broken notebook we defined in tst_nb. The json format is broken by the lines automatically added by git. Such a file can't be opened again in jupyter notebook, leaving the user with no other choice than to fix the text file manually.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
print(tst_nb)
+
+ +
+
+
+ +
+
+ +
+ +
+
{
+ "cells": [
+  {
+   "cell_type": "code",
+<<<<<<< HEAD
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "z=3
+",
+    "z"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+=======
+   "execution_count": 5,
+>>>>>>> a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "6"
+      ]
+     },
+<<<<<<< HEAD
+     "execution_count": 7,
+=======
+     "execution_count": 5,
+>>>>>>> a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x=3
+",
+    "y=3
+",
+    "x+y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+
+
+
+ +
+
+ +
+ {% endraw %} + +
+
+

Note that in this example, the second conflict is easily solved: it just concerns the execution count of the second cell and can be solved by choosing either option without really impacting your notebook. This is the kind of conflicts fix_conflicts will (by default) fix automatically. The first conflict is more complicated as it spans across two cells and there is a cell present in one version, not the other. Such a conflict (and generally the ones where the inputs of the cells change form one version to the other) aren't automatically fixed, but fix_conflicts will return a proper json file where the annotations introduced by git will be placed in markdown cells.

+

The first step to do this is to walk the raw text file to extract the cells. We can't read it as a JSON since it's broken, so we have to parse the text.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

extract_cells[source]

extract_cells(raw_txt)

+
+

Manually extract cells in potential broken json raw_txt

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This function returns the beginning of the text (before the cells are defined), the list of cells and the end of the text (after the cells are defined).

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
start,cells,end = extract_cells(tst_nb)
+test_eq(len(cells), 3)
+test_eq(cells[0], """  {
+   "cell_type": "code",
+<<<<<<< HEAD
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "z=3\n",
+    "z"
+   ]
+  },""")
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

When walking the broken cells, we will add conflicts marker before and after the cells with conflicts as markdown cells. To do that we use this function.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

get_md_cell[source]

get_md_cell(txt)

+
+

A markdown cell with txt

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
tst = '''  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A bit of markdown"
+   ]
+  },'''
+assert get_md_cell("A bit of markdown") == tst
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

same_inputs[source]

same_inputs(t1, t2)

+
+

Test if the cells described in t1 and t2 have the same inputs

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
ts = ['''  {
+   "cell_type": "code",
+   "source": [
+    "'''+code+'''"
+   ]
+  },''' for code in ["a=1", "b=1",  "a=1"]]
+assert same_inputs(ts[0],ts[2])
+assert not same_inputs(ts[0], ts[1])
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

analyze_cell[source]

analyze_cell(cell, cf, names, prev=None, added=False, fast=True, trust_us=True)

+
+

Analyze and solve conflicts in cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This is the main function used to walk through the cells of a notebook. cell is the cell we're at, cf the conflict state: 0 if we're not in any conflict, 1 if we are inside the first part of a conflict (between <<<<<<< and =======) and 2 for the second part of a conflict. names contains the names of the branches (they start at [None,None] and get updated as we pass along conflicts). prev contains a copy of what should be included at the start of the second version (if cf=1 or cf=2). added starts at False and keeps track of whether we added any markdown cells (this flag allows us to know if a fast merge didn't leave any conflicts at the end). fast and trust_us are passed along by fix_conflicts: if fast is True, we don't point out conflict between cells if the inputs in the two versions are the same. Instead we merge using the local or remote branch, depending on trust_us.

+

The function then returns the updated text (with one or several cells, depending on the conflicts to solve), the updated cf, names, prev and added.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
tst = '\n'.join(['a', f'{conflicts[0]} HEAD', 'b', conflicts[1], 'c'])
+c,cf,names,prev,added = analyze_cell(tst, 0, [None,None], None, False,fast=False)
+test_eq(c, get_md_cell('`<<<<<<< HEAD`')+'\na\nb')
+test_eq(cf, 2)
+test_eq(names, ['HEAD', None])
+test_eq(prev, ['a\nc'])
+test_eq(added, True)
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Here in this example, we were entering cell tst with no conflict state. At the end of the cells, we are still in the second part of the conflict, hence cf=2. The result returns a marker for the branch head, then the whole cell in version 1 (a + b). We save a (prior to the conflict hence common to the two versions) and c (only in version 2) for the next cell in prev (that should contain the resolution of this conflict).

+ +
+
+
+
+
+

Main function

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_fix_merge[source]

nbdev_fix_merge(fname:"A notebook filename to fix", fast:"Fast fix: automatically fix the merge conflicts in outputs or metadata"=True, trust_us:"Use local outputs/metadata when fast merging"=True)

+
+

Fix merge conflicts in notebook fname

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

This begins by backing the notebook fname to fname.bak in case something goes wrong. Then it parses the broken json, solving conflicts in cells. If fast=True, every conflict that only involves metadata or outputs of cells will be solved automatically by using the local (trust_us=True) or the remote (trust_us=False) branch. Otherwise, or for conflicts involving the inputs of cells, the json will be repaired by including the two version of the conflicted cell(s) with markdown cells indicating the conflicts. You will be able to open the notebook again and search for the conflicts (look for <<<<<<<) then fix them as you wish.

+

If fast=True, the function will print a message indicating whether the notebook was fully merged or if conflicts remain.

+ +
+
+
+
+ + diff --git a/docs/show_scaladoc.html b/docs/show_scaladoc.html new file mode 100644 index 0000000..e69de29 diff --git a/docs/sidebar.json b/docs/sidebar.json index 77d535f..ecdf504 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -1,6 +1,14 @@ { "chisel_nbdev": { "Overview": "/", + "Export to scala sbt project ": "export_scala.html", + "Synchronize and diff": "sync_scala.html", + "show_scaladoc": "show_scaladoc.html", + "Convert to html": "export_scala2html.html", + "Extract tests": "test_scala.html", + "Fix merge conflicts": "merge.html", + "Command line functions": "cli.html", + "Clean notebooks": "clean_scala.html", "Creating a basic Chisel module to be imported by other files.": "ToImport.html", "Import the module we defined in ToImport.ipynb renamed to -> ModA.sc": "import_chisel_mod.html", "Import the module we defined in import_chisel_mod.ipynb renamed to -> ModB.sc": "import_composed_mod.html", diff --git a/docs/sync_scala.html b/docs/sync_scala.html new file mode 100644 index 0000000..e12916d --- /dev/null +++ b/docs/sync_scala.html @@ -0,0 +1,484 @@ +--- + +title: Synchronize and diff + + +keywords: fastai +sidebar: home_sidebar + +summary: "The functions that propagates small changes in the library back to notebooks" +description: "The functions that propagates small changes in the library back to notebooks" +nb_path: "nbs/01_sync_scala.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

The library is primarily developed in notebooks so any big changes should be made there. But sometimes, it's easier to fix small bugs or typos in the modules directly. nbdev_update_lib is the function that will propagate those changes back to the corresponding notebooks. Note that you can't create new cells with that functionality, so your corrections should remain limited.

+ +
+
+
+
+
+

Finding the way back to notebooks

+
+
+
+
+
+

We need to get the name of the object we are looking for, and then we'll try to find it in our index file.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

get_name[source]

get_name(obj)

+
+

Get the name of obj

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
from nbdev.export import DocsTestClass
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_eq(get_name(in_ipython), 'in_ipython')
+test_eq(get_name(DocsTestClass.test), 'test')
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

qual_name[source]

qual_name(obj)

+
+

Get the qualified name of obj

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_eq(qual_name(DocsTestClass.test), 'DocsTestClass.test')
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

source_nb[source]

source_nb(func, is_name=None, return_all=False, mod=None)

+
+

Return the name of the notebook where func was defined

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

You can either pass an object or its name (by default is_name will look if func is a string or not to determine if it should be True or False, but you can override if there is some inconsistent behavior).

+

If passed a method of a class, the function will return the notebook in which the largest part of the function name was defined in case there is a monkey-matching that defines class.method in a different notebook than class. If return_all=True, the function will return a tuple with the name by which the function was found and the notebook.

+

For properties defined using property or our own add_props helper, we approximate the name by looking at their getter functions, since we don't seem to have access to the property name itself. If everything fails (a getter cannot be found), we return the name of the object that contains the property. This suffices for source_nb to work.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
test_eq(source_nb("test.Add"), 'test.ipynb')
+test_eq(source_nb("Add"), 'ToImport.ipynb')
+test_eq(source_nb("Operator"), 'test.ipynb')
+test_eq(source_nb("MulDiv"), 'test.ipynb')
+assert source_nb(int) is None
+
+ +
+
+
+ +
+
+ +
+ +
+
+---------------------------------------------------------------------------
+AssertionError                            Traceback (most recent call last)
+<ipython-input-10-e5991e20c770> in <module>
+----> 1 test_eq(source_nb("test.Add"), 'test.ipynb')
+      2 test_eq(source_nb("Add"), 'ToImport.ipynb')
+      3 test_eq(source_nb("Operator"), 'test.ipynb')
+      4 test_eq(source_nb("MulDiv"), 'test.ipynb')
+      5 assert source_nb(int) is None
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/fastcore/test.py in test_eq(a, b)
+     33 def test_eq(a,b):
+     34     "`test` that `a==b`"
+---> 35     test(a,b,equals, '==')
+     36 
+     37 # Cell
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/fastcore/test.py in test(a, b, cmp, cname)
+     23     "`assert` that `cmp(a,b)`; display inputs and `cname or cmp.__name__` if it fails"
+     24     if cname is None: cname=cmp.__name__
+---> 25     assert cmp(a,b),f"{cname}:\n{a}\n{b}"
+     26 
+     27 # Cell
+
+AssertionError: ==:
+None
+test.ipynb
+
+
+ +
+
+ +
+ {% endraw %} + +
+
+

Reading the library

+
+
+
+
+
+

If someone decides to change a module instead of the notebooks, the following functions help update the notebooks accordingly.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_update_lib[source]

nbdev_update_lib(fname:"A python filename or glob to convert"=None, silent:"Don't print results"=False)

+
+

Propagates any change in the modules matching fname to the notebooks that created them

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

If fname is not specified, this will convert all modules and submodules in the lib_folder defined in setting.ini. Otherwise fname can be a single filename or a glob expression.

+

silent makes the command not print any statement.

+ +
+
+
+
+
+

Diff & trust notebooks

+
+
+
+
+
+

Before making a commit, you may want to check there is no diff between the exported library and the notebooks. You may also want to make this part of your CI, so that you don't accidentally merge a PR that introduces some changes between the two. This function is there to print this diff.

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_diff_nbs[source]

nbdev_diff_nbs()

+
+

Prints the diff between an export of the library in notebooks and the actual modules

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

If you receive an output, you'll need to either run notebook2script() or nbdev_update_lib() to fix the difference.

+ +
+
+
+ {% raw %} + +
+
+ +
+
+
nbdev_diff_nbs()
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_trust_nbs[source]

nbdev_trust_nbs(fname:"A notebook name or glob to convert"=None, force_all:"Trust even notebooks that haven't changed"=False)

+
+

Trust notebooks matching fname

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
nbdev_trust_nbs()
+
+ +
+
+
+ +
+ {% endraw %} + +
+ + diff --git a/docs/test_scala.html b/docs/test_scala.html new file mode 100644 index 0000000..49d2ebe --- /dev/null +++ b/docs/test_scala.html @@ -0,0 +1,514 @@ +--- + +title: Extract tests + + +keywords: fastai +sidebar: home_sidebar + +summary: "The functions that grab the cells containing tests (filtering with potential flags) and execute them" +description: "The functions that grab the cells containing tests (filtering with potential flags) and execute them" +nb_path: "nbs/04_test_scala.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Until can build an actual package with chisel-nbdev python code, import my 00_export_scala.ipynb -> nbdev/export.py using this hacky method:

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + +
+
+

Everything that is not an exported cell is considered a test, so you should make sure your notebooks can all run smoothly (and fast) if you want to use this functionality as the CLI. You can mark some cells with special flags (like slow) to make sure they are only executed when you authorize it. Those flags should be configured in your settings.ini (separated by a | if you have several of them). You can also apply flags to one entire notebook by using the all option, e.g. #all_slow, in code cells.

+

If tst_flags=slow|fastai in settings.ini, you can:

+
    +
  • mark slow tests with #slow flag
    • +
  • mark tests that depend on fastai with the #fastai flag.
    • +
+ +
+
+
+
+
+

Detect flags

The following functions detect the cells that should be excluded from the tests (unless their special flag is passed).

+ +
+
+
+ {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

get_all_flags[source]

get_all_flags(cells)

+
+

Check for all test flags in cells

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
nb = read_nb("04_test_scala.ipynb")
+assert get_all_flags(nb['cells']) == set()
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

get_cell_flags[source]

get_cell_flags(cell)

+
+

Check for any special test flag in cell

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
test_eq(get_cell_flags({'cell_type': 'code', 'source': "//hide\n"}), [])
+
+ +
+
+
+ +
+ {% endraw %} + +
+
+

Testing a notebook

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

class NoExportPreprocessor[source]

NoExportPreprocessor(*args, **kwargs) :: ExecutePreprocessor

+
+

An ExecutePreprocessor that executes cells that don't have a flag in flags

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

If there is a mix of different notebook kernels (i.e Scala and Python notebooks), we only want to run the Scala tests.

+ +
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

test_nb[source]

test_nb(fn, flags=None)

+
+

Execute tests in notebook in fn with flags

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_test_nbs[source]

nbdev_test_nbs(fname:"A notebook name or glob to convert"=None, flags:"Space separated list of flags"=None, n_workers:"Number of workers to use"=None, verbose:"Print errors along the way"=True, timing:"Timing each notebook to see the ones are slow"=False, pause:"Pause time (in secs) between notebooks to avoid race conditions"=0.5)

+
+

Test in parallel the notebooks matching fname, passing along flags

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
# files
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

nbdev_read_nbs[source]

nbdev_read_nbs(fname:"A notebook name or glob to convert"=None)

+
+

Check all notebooks matching fname can be opened

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
nbdev_test_nbs()
+
+ +
+
+
+ +
+
+ +
+ +
+
[Path('/Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/ToImport.ipynb'), Path('/Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_chisel_mod.ipynb'), Path('/Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_composed_mod.ipynb'), Path('/Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/index.ipynb'), Path('/Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/test.ipynb')]
+testing /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/ToImport.ipynb
+testing /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_chisel_mod.ipynb
+Error in /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_chisel_mod.ipynb:
+An error occurred while executing the following cell:
+------------------
+//export
+import $file.^.nbdev.ModA, ModA._
+------------------
+
+Cannot resolve $file import: /Users/jasonvranek/code/notebooks/chisel_nbdev/nbdev/ModA.sc
+: 
+
+testing /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_composed_mod.ipynb
+Error in /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/import_composed_mod.ipynb:
+An error occurred while executing the following cell:
+------------------
+//export
+import $file.^.nbdev.ModB, ModB._
+------------------
+
+Cannot resolve $file import: /Users/jasonvranek/code/notebooks/chisel_nbdev/nbdev/ModB.sc
+: 
+
+testing /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/index.ipynb
+testing /Users/jasonvranek/code/notebooks/chisel_nbdev/nbs/test.ipynb
+
+
+
+ +
+ +
+
+---------------------------------------------------------------------------
+KeyboardInterrupt                         Traceback (most recent call last)
+<ipython-input-19-b1c77381ffda> in <module>
+      1 # nbdev_test_nbs("test.ipynb")
+----> 2 nbdev_test_nbs()
+
+<ipython-input-16-3e7a1b620eb4> in nbdev_test_nbs(fname, flags, n_workers, verbose, timing, pause)
+     18     os.chdir(Config().path("nbs_path"))
+     19 #     results = parallel(_test_one, files, flags=flags, verbose=verbose, n_workers=n_workers, pause=pause)
+---> 20     results = [_test_one(f, flags=flags, verbose=verbose) for f in files]#todo fix parallel tester
+     21     passed,times = [r[0] for r in results],[r[1] for r in results]
+     22     if all(passed): print("All tests are passing!")
+
+<ipython-input-16-3e7a1b620eb4> in <listcomp>(.0)
+     18     os.chdir(Config().path("nbs_path"))
+     19 #     results = parallel(_test_one, files, flags=flags, verbose=verbose, n_workers=n_workers, pause=pause)
+---> 20     results = [_test_one(f, flags=flags, verbose=verbose) for f in files]#todo fix parallel tester
+     21     passed,times = [r[0] for r in results],[r[1] for r in results]
+     22     if all(passed): print("All tests are passing!")
+
+<ipython-input-15-f3b99b2c5eef> in _test_one(fname, flags, verbose)
+      4     start = time.time()
+      5     try:
+----> 6         test_nb(fname, flags=flags)
+      7         return True,time.time()-start
+      8     except Exception as e:
+
+<ipython-input-14-28122f25158c> in test_nb(fn, flags)
+     10         ep = NoExportPreprocessor(flags, timeout=600, kernel_name='scala')
+     11         pnb = nbformat.from_dict(nb)
+---> 12         ep.preprocess(pnb)
+     13     finally: os.environ.pop("IN_TEST")
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/nbconvert/preprocessors/execute.py in preprocess(self, nb, resources, km)
+    403         with self.setup_preprocessor(nb, resources, km=km):
+    404             self.log.info("Executing notebook with kernel: %s" % self.kernel_name)
+--> 405             nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
+    406             info_msg = self._wait_for_reply(self.kc.kernel_info())
+    407             nb.metadata['language_info'] = info_msg['content']['language_info']
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/nbconvert/preprocessors/base.py in preprocess(self, nb, resources)
+     67         """
+     68         for index, cell in enumerate(nb.cells):
+---> 69             nb.cells[index], resources = self.preprocess_cell(cell, resources, index)
+     70         return nb, resources
+     71 
+
+<ipython-input-13-306abf9ec327> in preprocess_cell(self, cell, resources, index)
+     11             if f not in self.flags: return cell, resources
+     12         if check_re(cell, _re_notebook2script): return cell, resources
+---> 13         return super().preprocess_cell(cell, resources, index)
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/nbconvert/preprocessors/execute.py in preprocess_cell(self, cell, resources, cell_index, store_history)
+    436             return cell, resources
+    437 
+--> 438         reply, outputs = self.run_cell(cell, cell_index, store_history)
+    439         # Backwards compatibility for processes that wrap run_cell
+    440         cell.outputs = outputs
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/nbconvert/preprocessors/execute.py in run_cell(self, cell, cell_index, store_history)
+    587                         # polling for output.
+    588                         timeout = self._timeout_with_deadline(timeout, deadline)
+--> 589                     msg = self.kc.iopub_channel.get_msg(timeout=timeout)
+    590                 except Empty:
+    591                     if polling_exec_reply:
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/jupyter_client/blocking/channels.py in get_msg(self, block, timeout)
+     45             if timeout is not None:
+     46                 timeout *= 1000  # seconds to ms
+---> 47             ready = self.socket.poll(timeout)
+     48         else:
+     49             ready = self.socket.poll(timeout=0)
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/zmq/sugar/socket.py in poll(self, timeout, flags)
+    791         p = self._poller_class()
+    792         p.register(self, flags)
+--> 793         evts = dict(p.poll(timeout))
+    794         # return 0 if no events, otherwise return event bitfield
+    795         return evts.get(self, 0)
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/zmq/sugar/poll.py in poll(self, timeout)
+    101         elif isinstance(timeout, float):
+    102             timeout = int(timeout)
+--> 103         return zmq_poll(self.sockets, timeout=timeout)
+    104 
+    105 
+
+zmq/backend/cython/_poll.pyx in zmq.backend.cython._poll.zmq_poll()
+
+~/Code/classes/jlab_3.8/lib/python3.8/site-packages/zmq/backend/cython/checkrc.pxd in zmq.backend.cython.checkrc._check_rc()
+
+KeyboardInterrupt:
+
+
+ +
+
+ +
+ {% endraw %} + +
+ + diff --git a/nbs/00_export_scala.ipynb b/nbs/00_export_scala.ipynb index b5e03c4..b71090c 100644 --- a/nbs/00_export_scala.ipynb +++ b/nbs/00_export_scala.ipynb @@ -777,18 +777,6 @@ "test_eq(export_names(\"try:\\n this_might_work\\nexcept:\\n b=2\"), [])" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "todo need to handle imports in the form:\n", - "```scala\n", - " import $file.ToImport, ToImport._\n", - "```\n", - "\n", - "where ToImport.sc is the name of the module we extracted code to." - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -953,7 +941,7 @@ "`return_type` tells us if the tuple returned will contain `list`s of lines or `str`ings with line breaks. \n", "\n", "We treat the first comment line as a flag\n", - "![image.png](attachment:760534ae-50d7-47a4-9244-e742e1bc8dd6.png)" + "" ] }, { diff --git a/nbs/index.ipynb b/nbs/index.ipynb index a46ca45..edf4f08 100644 --- a/nbs/index.ipynb +++ b/nbs/index.ipynb @@ -18,13 +18,7 @@ "- find your python3 binary: ```which python3``` which for me is ```/usr/local/bin/python3```\n", "- create virtualenv: ```python3 -m virtualenv --python=/usr/local/bin/python3 chisel_nb_env```\n", "- activate it: ```source chisel_nb_env/bin/activate```\n", - "- install jupyterlab (which includes jupyter notebook): ```pip install jupyterlab```## Requirements (mac)\n", - "- setup some virtualenv (im using python3.8.2)\n", - "- ```pip install virtualenv```\n", - "- find your python3 binary: ```which python3``` which for me is ```/usr/local/bin/python3```\n", - "- create virtualenv: ```python3 -m virtualenv --python=/usr/local/bin/python3 chisel_nb_env```\n", - "- activate it: ```source chisel_nb_env/bin/activate```\n", - "- install jupyterlab (which includes jupyter notebook): ```pip install jupyterlab```" + "- install jupyterlab (which includes jupyter notebook): ```pip install jupyterlab```## Requirements (mac)" ] }, {