Notebook

In [ ]:

#hide
#default_exp export2html
#default_cls_lvl 3
from nbdev.showdoc import show_doc

In [ ]:

#export
from nbdev.imports import *
from nbdev.sync import *
from nbdev.export import *
from nbdev.export import _mk_flag_re
from nbdev.showdoc import *
from nbdev.template import *
from fastcore.foundation import *
from fastcore.script import *

from html.parser import HTMLParser
from nbconvert.preprocessors import ExecutePreprocessor, Preprocessor
from nbconvert import HTMLExporter,MarkdownExporter
import traitlets
import nbformat

Convert to html¶

The functions that transform the dev notebooks in the documentation of the library

toc: true

The most important function defined in this module is notebook2html, so you may want to jump to it before scrolling though the rest, which explain the details behind the scenes of the conversion from notebooks to the html documentation. The main things to remember are:

put #hide at the top of any cell you want to completely hide in the docs
use the hide input jupyter extension to hide the input of some cells (by default all show_doc cells have that marker added)
you can define some jekyll metadata in the markdown cell with the title, see get_metadata
use backticks for terms you want automatic links to be found, but use <code> and </code> when you have homonyms and don't want those links
you can define the default toc level of classes with #default_cls_lvl followed by a number (default is 2)
you can add jekyll warnings, important or note banners with appropriate block quotes (see add_jekyll_notes)
put any images you want to use in the images folder of your notebook folder, they will be automatically copied over to the docs folder
put #hide_input at the top of a cell if you don't want code to be shown in the docs
- cells containing #export or show_doc have their code hidden automatically
put #hide_output at the top of a cell if you don't want output to be shown in the docs
use #collapse_input or #collapse_output to include code or output in the docs under a collapsable element

Preprocessing notebook¶

Cell processors¶

In [ ]:

#export
class HTMLParseAttrs(HTMLParser):
    "Simple HTML parser which stores any attributes in `attrs` dict"
    def handle_starttag(self, tag, attrs): self.tag,self.attrs = tag,dict(attrs)

    def attrs2str(self):
        "Attrs as string"
        return ' '.join([f'{k}="{v}"' for k,v in self.attrs.items()])

    def show(self):
        "Tag with updated attrs"
        return f'<{self.tag} {self.attrs2str()} />'

    def __call__(self, s):
        "Parse `s` and store attrs"
        self.feed(s)
        return self.attrs

In [ ]:

h = HTMLParseAttrs()
t = h('<img src="src" alt="alt" width="700" caption="cap" />')
test_eq(t['width'], '700')
test_eq(t['src'  ], 'src')

In [ ]:

t['width'] = '600'
test_eq(h.show(), '<img src="src" alt="alt" width="600" caption="cap" />')

The following functions are applied on individual cells as a preprocessing step before the conversion to html.

In [ ]:

#export
def remove_widget_state(cell):
    "Remove widgets in the output of `cells`"
    if cell['cell_type'] == 'code' and 'outputs' in cell:
        cell['outputs'] = [l for l in cell['outputs']
                           if not ('data' in l and 'application/vnd.jupyter.widget-view+json' in l.data)]
    return cell

Those outputs usually can't be rendered properly in html.

In [ ]:

#export
# Note: `_re_show_doc` will catch show_doc even if it's commented out etc
_re_show_doc = re.compile(r"""
# Catches any show_doc and get the first argument in group 1
^\s*show_doc # line can start with any amount of whitespace followed by show_doc
\s*\(\s*     # Any number of whitespace, opening (, any number of whitespace
([^,\)\s]*)  # Catching group for any character but a comma, a closing ) or a whitespace
[,\)\s]      # A comma, a closing ) or a whitespace
""", re.MULTILINE | re.VERBOSE)

_re_hide_input = [
    _mk_flag_re('export', (0,1), "Cell that has `#export"),
    _mk_flag_re('(hide_input|hide-input)', 0, "Cell that has `#hide_input` or `#hide-input`")]

_re_hide_output = _mk_flag_re('(hide_output|hide-output)', 0, "Cell that has `#hide_output` or `#hide-output`")

In [ ]:

#export
def upd_metadata(cell, key, value=True):
    "Sets `key` to `value` on the `metadata` of `cell` without replacing metadata"
    cell.setdefault('metadata',{})[key] = value

In [ ]:

#export
def hide_cells(cell):
    "Hide inputs of `cell` that need to be hidden"
    if check_re_multi(cell, [_re_show_doc, *_re_hide_input]): upd_metadata(cell, 'hide_input')
    elif check_re(cell, _re_hide_output): upd_metadata(cell, 'hide_output')
    return cell

This concerns all the cells with #export or #hide_input flags and all the cells containing a show_doc for a function or class.

In [ ]:

for source in ['show_doc(read_nb)', '# export\nfrom local.core import *', '# hide_input\n2+2',
              'line1\n show_doc (read_nb) \nline3', '# export with.mod\nfrom local.core import *']:
    cell = {'cell_type': 'code', 'source': source}
    cell1 = hide_cells(cell.copy())
    assert 'metadata' in cell1
    assert 'hide_input' in cell1['metadata']
    assert cell1['metadata']['hide_input']

flag = '# exports'
cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core2 import *'}
test_eq(hide_cells(cell.copy()), cell)

This concerns all the cells with #hide_output.

In [ ]:

for source in ['# hide-output\nfrom local.core import *', '# hide_output\n2+2']:
    cell = {'cell_type': 'code', 'source': source}
    cell1 = hide_cells(cell.copy())
    assert 'metadata' in cell1
    assert 'hide_output' in cell1['metadata']
    assert cell1['metadata']['hide_output']

cell = {'cell_type': 'code', 'source': '# hide-outputs\nfrom local.core import *'}
test_eq(hide_cells(cell.copy()), cell)

In [ ]:

#export
def clean_exports(cell):
    "Remove all flags from code `cell`s"
    if cell['cell_type'] == 'code': cell['source'] = split_flags_and_code(cell, str)[1]
    return cell

The rest of the cell is displayed without any modification.

In [ ]:

flag = '# exports'
cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core import *'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
cell['cell_type'] = 'markdown'
test_eq(clean_exports(cell.copy()), cell)
cell = {'cell_type': 'code', 'source': f'{flag} core\nfrom local.core import *'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
    
cell = {'cell_type': 'code', 'source': f'# comment \n# exports\nprint("something")'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': '# exports\nprint("something")'})

In [ ]:

#export
def treat_backticks(cell):
    "Add links to backticks words in `cell`"
    if cell['cell_type'] == 'markdown': cell['source'] = add_doc_links(cell['source'])
    return cell

In [ ]:

cell = {'cell_type': 'markdown', 'source': 'This is a `DocsTestClass`'}
# test_eq(treat_backticks(cell), {'cell_type': 'markdown',
#     'source': 'This is a [`DocsTestClass`](/export.html#DocsTestClass)'})

In [ ]:

#export
_re_nb_link = re.compile(r"""
# Catches any link to a local notebook and keeps the title in group 1, the link without .ipynb in group 2
\[          # Opening [
([^\]]*)    # Catching group for any character except ]
\]\(        # Closing ], opening (
([^http]    # Catching group that must not begin by html (local notebook)
[^\)]*)     # and containing anything but )
.ipynb\)    # .ipynb and closing )
""", re.VERBOSE)

In [ ]:

#export
_re_block_notes = re.compile(r"""
# Catches any pattern > Title: content with title in group 1 and content in group 2
^\s*>\s*     # > followed by any number of whitespace
([^:]*)      # Catching group for any character but :
:\s*         # : then any number of whitespace
([^\n]*)     # Catching group for anything but a new line character
(?:\n|$)     # Non-catching group for either a new line or the end of the text
""", re.VERBOSE | re.MULTILINE)

In [ ]:

#export
def _to_html(text):
    return text.replace("'", "&#8217;")

In [ ]:

#export
def add_jekyll_notes(cell):
    "Convert block quotes to jekyll notes in `cell`"
    styles = get_config().get('jekyll_styles', 'note,warning,tip,important').split(',')
    def _inner(m):
        title,text = m.groups()
        if title.lower() not in styles: return f"> {title}:{text}"
        return '{% include '+title.lower()+".html content=\'"+_to_html(text)+"\' %}"
    if cell['cell_type'] == 'markdown':
        cell['source'] = _re_block_notes.sub(_inner, cell['source'])
    return cell

Supported styles are Warning, Note Tip and Important:

Typing > Warning: There will be no second warning! will render in the docs:

Warning: There will be no second warning!

Typing > Important: Pay attention! It's important. will render in the docs:

Important: Pay attention! It's important.

Typing > Tip: This is my tip. will render in the docs:

Tip: This is my tip.

Typing > Note: Take note of this. will render in the docs:

Note: Take note of this.

Typing > Note: A doc link to `add_jekyll_notes` should also work fine. will render in the docs:

Note: A doc link to add_jekyll_notes should also work fine.

In [ ]:

#hide
for w in ['Warning', 'Note', 'Important', 'Tip', 'Bla']:
    cell = {'cell_type': 'markdown', 'source': f"> {w}: This is my final {w.lower()}!"}
    res = '{% include '+w.lower()+'.html content=\'This is my final '+w.lower()+'!\' %}'
    if w != 'Bla': test_eq(add_jekyll_notes(cell), {'cell_type': 'markdown', 'source': res})
    else: test_eq(add_jekyll_notes(cell), cell)

In [ ]:

#hide
cell = {'cell_type': 'markdown', 'source': f"> This is a link, don't break me! https://my.link.com"}
test_eq(add_jekyll_notes(cell.copy()), cell)

In [ ]:

#export
_re_image = re.compile(r"""
# Catches any image file used, either with `![alt](image_file)` or `<img src="image_file">`
^(!\[           #   Beginning of line (since re.MULTILINE is passed) followed by ![ in a catching group
[^\]]*          #   Anything but ]
\]\()           #   Closing ] and opening (, end of the first catching group
[ \t]*          #   Whitespace before the image path
([^\) \t]*)     #   Catching block with any character that is not ) or whitespace
(\)| |\t)       #   Catching group with closing ) or whitespace
|               # OR
^(<img\ [^>]*>) #   Catching group with <img some_html_code>
""", re.MULTILINE | re.VERBOSE)

In [ ]:

m=_re_image.search('![Alt](images/logo.png)')
test_eq(m.groups(), ('![Alt](', 'images/logo.png', ')', None))
# using ) or whitespace to close the group means we don't need a special case for captions
m=_re_image.search('![Alt](images/logo.png "caption (something)")')
test_eq(m.groups(), ('![Alt](', 'images/logo.png', '', None))

In [ ]:

#export
def _img2jkl(d, h, jekyll=True):
    if d.get("src","").startswith("http"): jekyll=False
    if jekyll:
        if 'width' in d: d['max-width'] = d.get('width')
    else:
        if 'width' in d: d['style'] = f'max-width: {d.get("width")}px'
        d.pop('align','')
        return '<img ' + h.attrs2str() + '>'
    if 'src' in d:   d['file'] = d.pop('src')
    return '{% include image.html ' + h.attrs2str() + ' %}'

In [ ]:

#export
def _is_real_image(src):
    return not (src.startswith('http://') or src.startswith('https://') or src.startswith('data:image/') or src.startswith('attachment:'))

In [ ]:

#export
def copy_images(cell, fname, dest, jekyll=True):
    "Copy images referenced in `cell` from `fname` parent folder to `dest` folder"
    def _rep_src(m):
        grps = m.groups()
        if grps[3] is not None:
            h = HTMLParseAttrs()
            dic = h(grps[3])
            src = dic['src']
        else: src = grps[1]
        if _is_real_image(src):
            os.makedirs((Path(dest)/src).parent, exist_ok=True)
            shutil.copy(Path(fname).parent/src, Path(dest)/src)
            src = get_config().doc_baseurl + src
        if grps[3] is not None:
            dic['src'] = src
            return _img2jkl(dic, h, jekyll=jekyll)
        else: return f"{grps[0]}{src}{grps[2]}"
    if cell['cell_type'] == 'markdown': cell['source'] = _re_image.sub(_rep_src, cell['source'])
    return cell

This is to ensure that all images defined in nbs_folder/images and used in notebooks are copied over to doc_folder/images.

In [ ]:

dest_img = get_config().path("doc_path")/'images'/'logo.png'
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.png)'}
try:
    copy_images(cell, Path('01_export.ipynb'), get_config().path("doc_path"))
    test_eq(cell["source"], 'Text\n![Alt](/images/logo.png)')
    #Image has been copied
    assert dest_img.exists()
    cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.png "caption (something)")'}
    copy_images(cell, Path('01_export.ipynb'), get_config().path("doc_path"))
    test_eq(cell["source"], 'Text\n![Alt](/images/logo.png "caption (something)")')
finally: dest_img.unlink()

In [ ]:

#hide
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](https://site.logo.png)'}
copy_images(cell, Path('01_export.ipynb'), get_config().path("doc_path"))
test_eq(cell["source"], 'Text\n![Alt](https://site.logo.png)')
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](https://site.logo.png "caption")'}
copy_images(cell, Path('01_export.ipynb'), get_config().path("doc_path"))
test_eq(cell["source"], 'Text\n![Alt](https://site.logo.png "caption")')

In [ ]:

#export
def _relative_to(path1, path2):
    p1,p2 = Path(path1).absolute().parts,Path(path2).absolute().parts
    i=0
    while i <len(p1) and i<len(p2) and p1[i] == p2[i]: i+=1
    p1,p2 = p1[i:],p2[i:]
    return os.path.sep.join(['..' for _ in p2] + list(p1))

In [ ]:

#hide
test_eq(_relative_to(Path('images/logo.png'), get_config().path("doc_path")), str(Path('../nbs/images/logo.png')))
test_eq(_relative_to(Path('images/logo.png'), get_config().path("doc_path").parent), str(Path('nbs/images/logo.png')))

In [ ]:

#export
def adapt_img_path(cell, fname, dest, jekyll=True):
    "Adapt path of images referenced in `cell` from `fname` to work in folder `dest`"
    def _rep(m):
        gps = m.groups()
        if gps[0] is not None:
            start,img,end = gps[:3]
            if not (img.startswith('http:/') or img.startswith('https:/')):
                img = _relative_to(fname.parent/img, dest)
            return f'{start}{img}{end}'
        else:
            h = HTMLParseAttrs()
            dic = h(gps[3])
            if not (dic['src'].startswith('http:/') or dic['src'].startswith('https:/')):
                dic['src'] = _relative_to(fname.parent/dic['src'], dest)
            return _img2jkl(dic, h, jekyll=jekyll)
    if cell['cell_type'] == 'markdown': cell['source'] = _re_image.sub(_rep, cell['source'])
    return cell

This function is slightly different as it ensures that a notebook convert to a file that will be placed in dest will have the images location updated. It is used for the README.md file (generated automatically from the index) since the images are copied inside the github repo, but in general, you should make sure your images are going to be accessible from the location your file ends up being.

In [ ]:

cell = {'cell_type': 'markdown', 'source': str(Path('Text\n![Alt](images/logo.png)'))}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], str(Path('Text\n![Alt](nbs/images/logo.png)')))

cell = {'cell_type': 'markdown', 'source': 'Text\n![Alt](http://site.logo.png)'}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], 'Text\n![Alt](http://site.logo.png)')

Escape Latex in liquid

In [ ]:

#export
_re_latex = re.compile(r'^(\$\$.*\$\$)$', re.MULTILINE)

In [ ]:

#export
def escape_latex(cell):
    if cell['cell_type'] != 'markdown': return cell
    cell['source'] = _re_latex.sub(r'{% raw %}\n\1\n{% endraw %}', cell['source'])
    return cell

In [ ]:

cell = {'cell_type': 'markdown', 
        'source': 'lala\n$$equation$$\nlala'}
cell = escape_latex(cell)
test_eq(cell['source'], 'lala\n{% raw %}\n$$equation$$\n{% endraw %}\nlala')

Collapsable Code Cells¶

In [ ]:

#export
_re_cell_to_collapse_closed = _mk_flag_re('(collapse|collapse_hide|collapse-hide)', 0, "Cell with #collapse or #collapse_hide")
_re_cell_to_collapse_open = _mk_flag_re('(collapse_show|collapse-show)', 0, "Cell with #collapse_show")
_re_cell_to_collapse_output = _mk_flag_re('(collapse_output|collapse-output)', 0, "Cell with #collapse_output")

In [ ]:

#export
def collapse_cells(cell):
    "Add a collapse button to inputs or outputs of `cell` in either the open or closed position"
    if check_re(cell, _re_cell_to_collapse_closed): upd_metadata(cell,'collapse_hide')
    elif check_re(cell, _re_cell_to_collapse_open): upd_metadata(cell,'collapse_show')
    elif check_re(cell, _re_cell_to_collapse_output): upd_metadata(cell,'collapse_output')
    return cell

In [ ]:

#hide
for flag in [
    ('collapse_hide', '#collapse'),
    ('collapse_hide', '# collapse_hide'),
    ('collapse_hide', ' # collapse-hide'),
    ('collapse_show', '#collapse_show'),
    ('collapse_show', '#collapse-show'),
    ('collapse_output', ' #collapse_output'),
    ('collapse_output', '#collapse-output')]:
    cell = nbformat.v4.new_code_cell(f'#comment\n{flag[1]} \ndef some_code')
    test_eq(True, collapse_cells(cell)['metadata'][flag[0]])

In [ ]:

#hide
# check that we can't collapse both input and output
cell = nbformat.v4.new_code_cell(f'#hide-input\n#collapse_output \ndef some_code')
test_eq({'hide_input': True, 'collapse_output': True}, hide_cells(collapse_cells(cell))['metadata'])

Placing #collapse_input open in a code cell will include your code under a collapsable element that is open by default.

In [ ]:

#collapse_input open
print('This code cell is not collapsed by default but you can collapse it to hide it from view!')
print("Note that the output always shows with `%collapse_input`.")

Placing #collapse_input in a code cell will include your code in a collapsable element that is closed by default. For example:

In [ ]:

#collapse_input
print('The code cell that produced this output is collapsed by default but you can expand it!')

Placing #collapse_output in a code cell will hide the output under a collapsable element that is closed by default.

In [ ]:

#collapse_output
print('The input of this cell is visible as usual.\nHowever, the OUTPUT of this cell is collapsed by default but you can expand it!')

Preprocessing the list of cells¶

The following functions are applied to the entire list of cells of the notebook as a preprocessing step before the conversion to html.

In [ ]:

#export
_re_hide = _mk_flag_re('hide', 0, 'Cell with #hide')
_re_cell_to_remove = _mk_flag_re('(default_exp|exporti)', (0,1), 'Cell with #default_exp or #exporti')
_re_default_cls_lvl = _mk_flag_re('default_cls_lvl', 1, "Cell with #default_cls_lvl")

In [ ]:

#export
def remove_hidden(cells):
    "Remove in `cells` the ones with a flag `#hide`, `#default_exp`, `#default_cls_lvl` or `#exporti`"
    _hidden = lambda c: check_re(c, _re_hide, code_only=False) or check_re(c, _re_cell_to_remove)
    return L(cells).filter(_hidden, negate=True)

In [ ]:

cells = [{'cell_type': 'code', 'source': source, 'hide': hide} for hide, source in [
    (False, '# export\nfrom local.core import *'), 
    (False, '# exporti mod file'), # Note: this used to get removed but we're more strict now
    (True, '# hide\nfrom local.core import *'),
    (False, '# hide_input\nfrom local.core import *'),
    (False, '#exports\nsuper code'),
    (True, '#default_exp notebook.export'),
    (False, 'show_doc(read_nb)'),
    (False, '#hide (last test of to_concat)'),
    (True, '# exporti\n1 + 1')]] + [
    {'cell_type': 'markdown', 'source': source, 'hide': hide} for hide, source in [
    (False, '#hide_input\nnice'), 
    (True, '#hide\n\nto hide')]]

for a,b in zip([cell for cell in cells if not cell['hide']], remove_hidden(cells)):
    test_eq(a,b)

In [ ]:

#export
def find_default_level(cells):
    "Find in `cells` the default class level."
    res = L(cells).map_first(check_re_multi, pats=_re_default_cls_lvl)
    return int(res.groups()[0]) if res else 2

In [ ]:

tst_nb = read_nb('00_export.ipynb')
test_eq(find_default_level(tst_nb['cells']), 3)

In [ ]:

#export
_re_export = _mk_flag_re("exports?", (0,1), "Line with #export or #exports with or without module name")

In [ ]:

#export
def nb_code_cell(source):
    "A code cell (as a dict) containing `source`"
    return {'cell_type': 'code', 'execution_count': None, 'metadata': {}, 'outputs': [], 'source': source}

In [ ]:

#export
def _show_doc_cell(name, cls_lvl=None):
    show_all = get_config().d.getboolean('show_all_docments', False)
    return nb_code_cell(f"show_doc({name}{'' if cls_lvl is None else f', default_cls_level={cls_lvl}'}, show_all_docments={show_all})")

def add_show_docs(cells, cls_lvl=None):
    "Add `show_doc` for each exported function or class"
    documented = L(cells).map(check_re, pat=_re_show_doc).filter().map(Self.group(1))
    res = []
    for cell in cells:
        res.append(cell)
        if check_re(cell, _re_export):
            for n in export_names(cell['source'], func_only=True):
                if not n in documented: res.insert(len(res)-1, _show_doc_cell(n, cls_lvl=cls_lvl))
    return res

This only adds cells with a show_doc for non-documented functions, so if you add yourself a show_doc cell (because you want to change one of the default argument), there won't be any duplicates.

In [ ]:

for i,cell in enumerate(tst_nb['cells']):
    if cell['source'].startswith('#export\ndef read_nb'): break
tst_cells = [c.copy() for c in tst_nb['cells'][i-1:i+1]]
added_cells = add_show_docs(tst_cells, cls_lvl=3)
test_eq(len(added_cells), 3)
test_eq(added_cells[0], tst_nb['cells'][i-1])
test_eq(added_cells[2], tst_nb['cells'][i])
test_eq(added_cells[1], _show_doc_cell('read_nb', cls_lvl=3))
test_eq(added_cells[1]['source'], 'show_doc(read_nb, default_cls_level=3, show_all_docments=False)')

for flag in ['#export', '#exports']:
    for show_doc_source in [
            ('show_doc(my_func)', 'show_doc(my_func, title_level=3)')]:
        #Check show_doc isn't added if it was already there.
        tst_cells1 = [{'cell_type':'code', 'source': f'{flag}\ndef my_func(x):\n    return x'},
                      {'cell_type':'code', 'source': show_doc_source[0]}]
        test_eq(add_show_docs(tst_cells1), tst_cells1)
        #Check show_doc is added
        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))
        tst_cells1 = [{'cell_type':'code', 'source': f'{flag} with.mod\ndef my_func(x):\n    return x'},
                      {'cell_type':'markdown', 'source': 'Some text'},
                      {'cell_type':'code', 'source': show_doc_source[1]}]
        test_eq(add_show_docs(tst_cells1), tst_cells1)
        #Check show_doc is added when using mod export
        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))

In [ ]:

#export
_re_fake_header = re.compile(r"""
# Matches any fake header (one that ends with -)
\#+    # One or more #
\s+    # One or more of whitespace
.*     # Any char
-\s*   # A dash followed by any number of white space
$      # End of text
""", re.VERBOSE)

In [ ]:

#export
def remove_fake_headers(cells):
    "Remove in `cells` the fake header"
    return [c for c in cells if c['cell_type']=='code' or _re_fake_header.search(c['source']) is None]

You can fake headers in your notebook to navigate them more easily with collapsible headers, just make them finish with a dash and they will be removed. One typical use case is to have a header of level 2 with the name of a class, since the show_doc cell of that class will create the same anchor, you need to have the one you created manually disappear to avoid any duplicate.

In [ ]:

cells = [{'cell_type': 'markdown',
          'metadata': {},
          'source': '### Fake-'}] + tst_nb['cells'][:10]
cells1 = remove_fake_headers(cells)
test_eq(len(cells1), len(cells)-1)
test_eq(cells1[0], cells[1])

In [ ]:

#export
def remove_empty(cells):
    "Remove in `cells` the empty cells"
    return [c for c in cells if len(c['source']) >0]

Grabbing metadata¶

In [ ]:

#export
_re_title_summary = re.compile(r"""
# Catches the title and summary of the notebook, presented as # Title > summary, with title in group 1 and summary in group 2
^\s*       # Beginning of text followed by any number of whitespace
\#\s+      # # followed by one or more of whitespace
([^\n]*)   # Catching group for any character except a new line
\n+        # One or more new lines
>[ ]*       # > followed by any number of whitespace
([^\n]*)   # Catching group for any character except a new line
""", re.VERBOSE)

_re_title_only = re.compile(r"""
# Catches the title presented as # Title without a summary
^\s*       # Beginning of text followed by any number of whitespace
\#\s+      # # followed by one or more of whitespace
([^\n]*)   # Catching group for any character except a new line
(?:\n|$)    # New line or end of text
""", re.VERBOSE)

_re_properties = re.compile(r"""
^-\s+      # Beginning of a line followed by - and at least one space
(.*?)      # Any pattern (shortest possible)
\s*:\s*    # Any number of whitespace, :, any number of whitespace
(.*?)$     # Any pattern (shortest possible) then end of line
""", re.MULTILINE | re.VERBOSE)

_re_mdlinks = re.compile(r"\[(.+)]\((.+)\)", re.MULTILINE)

In [ ]:

#export
def _md2html_links(s):
    'Converts markdown links to html links'
    return _re_mdlinks.sub(r"<a href='\2'>\1</a>", s)

In [ ]:

#export
def get_metadata(cells):
    "Find the cell with title and summary in `cells`."
    for i,cell in enumerate(cells):
        if cell['cell_type'] == 'markdown':
            match = _re_title_summary.match(cell['source'])
            if match:
                cells.pop(i)
                attrs = {k:v for k,v in _re_properties.findall(cell['source'])}
                return {'keywords': 'fastai',
                        'summary' : _md2html_links(match.groups()[1]),
                        'title'   : match.groups()[0],
                        **attrs}
            elif _re_title_only.search(cell['source']) is not None:
                title = _re_title_only.search(cell['source']).groups()[0]
                cells.pop(i)
                attrs = {k:v for k,v in _re_properties.findall(cell['source'])}
                return {'keywords': 'fastai',
                        'title'   : title,
                        **attrs}

    return {'keywords': 'fastai',
            'title'   : 'Title'}

In the markdown cell with the title, you can add the summary as a block quote (just put an empty block quote for an empty summary) and a list with any additional metadata you would like to add, for instance:

# Title

> Awesome summary
- toc: False

The toc: False metadata will prevent the table of contents from showing on the page.

In [ ]:

tst_nb = read_nb('00_export.ipynb')
test_eq(get_metadata(tst_nb['cells']), {
    'keywords': 'fastai',
    'summary': 'The functions that transform notebooks in a library',
    'title': 'Export to modules'})

#The cell with the metada is popped out, so if we do it a second time we get the default.
test_eq(get_metadata(tst_nb['cells']), {'keywords': 'fastai', 'title'   : 'Title'})

In [ ]:

#hide
#test with title only
test_eq(get_metadata([{'cell_type': 'markdown', 'source': '# Awesome title'}]), 
        {'keywords': 'fastai', 'title': 'Awesome title'})

In [ ]:

#hide
text = r"""
[This](https://nbdev.fast.ai) goes to docs.
This [one:here](00_export.ipynb) goes to a local nb. 
\n[And-this](http://dev.fast.ai/) goes to fastai docs
"""
res = """
<a href='https://nbdev.fast.ai'>This</a> goes to docs.\nThis <a href='00_export.ipynb'>one:here</a> goes to a local nb. \n\\n<a href='http://dev.fast.ai/'>And-this</a> goes to fastai docs
"""
test_eq(_md2html_links(text), res)

In [ ]:

#hide
cells = [{'cell_type': 'markdown', 'source': "# Title\n\n> s\n\n- toc: false"}]
test_eq(get_metadata(cells), {'keywords': 'fastai', 'summary': 's', 'title': 'Title', 'toc': 'false'})

Executing show_doc cells¶

In [ ]:

#export
_re_mod_export = _mk_flag_re("export[s]?", 1,
    "Matches any line with #export or #exports with a module name and catches it in group 1")

def _gather_export_mods(cells):
    res = []
    for cell in cells:
        tst = check_re(cell, _re_mod_export)
        if tst is not None: res.append(tst.groups()[0])
    return res

In [ ]:

#hide
cells = [
    {'cell_type': 'markdown', 'source': '#export ignored'},
    {'cell_type': 'code', 'source': '#export'},
    {'cell_type': 'code', 'source': '#export normal'},
    {'cell_type': 'code', 'source': '# exports show'},
    {'cell_type': 'code', 'source': '# exporti hidden'},
    {'cell_type': 'code', 'source': '#export\n@call_parse'},
    {'cell_type': 'code', 'source': '#export \n@delegates(concurrent.futures.ProcessPoolExecutor)'}
]

test_eq(_gather_export_mods(cells), ['normal', 'show'])

In [ ]:

#export
# match any cell containing a zero indented import from the current lib
_re_lib_import = ReLibName(r"^from LIB_NAME\.", re.MULTILINE)
# match any cell containing a zero indented import
_re_import = re.compile(r"^from[ \t]+\S+[ \t]+import|^import[ \t]", re.MULTILINE)
# match any cell containing a zero indented call to notebook2script
_re_notebook2script = re.compile(r"^notebook2script\(", re.MULTILINE)

In [ ]:

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (True, 'show_doc(Tensor.p)'),
    (True, ' show_doc(Tensor.p)'),
    (True, 'if something:\n    show_doc(Tensor.p)'),
    (False, '# show_doc(Tensor.p)'),
    (True, '# comment \n show_doc(Tensor.p)'),
    (True, '"""\nshow_doc(Tensor.p)\n"""'),
    (True, 'import torch\nshow_doc(Tensor.p)'),
    (False,'class Ex(ExP):\n"An `ExP` that ..."\ndef preprocess_cell(self, cell, resources, index):\n'),
    (False, 'from somewhere import something'),
    (False, 'from '),
    (False, 'import re'),
    (False, 'import '),
    (False, 'try: from PIL import Image\except: pass'),
    (False, 'from PIL import Image\n@patch\ndef p(x:Image):\n    pass'),
    (False, '@patch\ndef p(x:Image):\n    pass\nfrom PIL import Image')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re_multi(cell, [_re_show_doc, _re_lib_import.re])), f'expected {exp} for {cell}'

In [ ]:

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (False, 'show_doc(Tensor.p)'),
    (True, 'import torch\nshow_doc(Tensor.p)'),
    (False,'class Ex(ExP):\n"An `ExP` that ..."\ndef preprocess_cell(self, cell, resources, index):\n'),
    (False, ' from somewhere import something'),
    (True, 'from somewhere import something'),
    (False, 'from '),
    (False, 'select * \nfrom database'),
    (False, ' import re'),
    (True, 'import re'),
    (True, 'import '),
    (False, 'try: from PIL import Image\except: pass'),
    (True, 'from PIL import Image\n@patch\ndef p(x:Image):\n    pass'),
    (True, '@patch\ndef p(x:Image):\n    pass\nfrom PIL import Image')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re(cell, _re_import)), f'expected {exp} for {cell}'

In [ ]:

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (False, 'show_doc(Tensor.p)'),
    (False, 'notebook2script'),
    (False, '#notebook2script()'),
    (True, 'notebook2script()'),
    (True, 'notebook2script(anything at all)')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re(cell, _re_notebook2script)), f'expected {exp} for {cell}'

In [ ]:

#export
def _non_comment_code(s):
    if re.match(r'\s*#', s): return False
    if _re_import.findall(s) or _re_lib_import.re.findall(s): return False
    return re.match(r'\s*\w', s)

In [ ]:

#export
class ExecuteShowDocPreprocessor(ExecutePreprocessor):
    "An `ExecutePreprocessor` that only executes `show_doc` and `import` cells"
    def preprocess_cell(self, cell, resources, index):
        if not check_re(cell, _re_notebook2script):
            if check_re(cell, _re_show_doc):
                return super().preprocess_cell(cell, resources, index)
            elif check_re_multi(cell, [_re_import, _re_lib_import.re]):
                if check_re_multi(cell, [_re_export, 'show_doc', '^\s*#\s*import']):
#                     r = list(filter(_non_comment_code, cell['source'].split('\n')))
#                     if r: print("You have import statements mixed with other code", r)
                    return super().preprocess_cell(cell, resources, index)
#                 try: return super().preprocess_cell(cell, resources, index)
#                 except: pass
        return cell, resources

Cells containing:

a zero indented call to notebook2script

are not run while building docs. This avoids failures caused by importing empty or partially built modules.

Cells containing:

show_doc (which could be indented) or
a "library import" (zero indent import from current library) e.g. from LIB_NAME.core import *

are executed and must run without error. If running these cells raises an exception, the build will stop.

Cells containing zero indented imports. e.g.

from module import * or
import module

are executed but errors will not stop the build.

If you need to show_doc something, please make sure it's imported via a cell that does not depend on previous cells being run. The easiest way to do this is to use a cell that contains nothing but imports.

In [ ]:

#export
def _import_show_doc_cell(mods=None):
    "Add an import show_doc cell."
    source = f"from nbdev.showdoc import show_doc"
    if mods is not None:
        for mod in mods: source += f"\nfrom {get_config().lib_name}.{mod} import *"
    return {'cell_type': 'code',
            'execution_count': None,
            'metadata': {'hide_input': True},
            'outputs': [],
            'source': source}

def execute_nb(nb, mod=None, metadata=None, show_doc_only=True):
    "Execute `nb` (or only the `show_doc` cells) with `metadata`"
    mods = ([] if mod is None else [mod]) + _gather_export_mods(nb['cells'])
    nb['cells'].insert(0, _import_show_doc_cell(mods))
    ep_cls = ExecuteShowDocPreprocessor if show_doc_only else ExecutePreprocessor
    ep = ep_cls(timeout=600, kernel_name='python3')
    metadata = metadata or {}
    pnb = nbformat.from_dict(nb)
    ep.preprocess(pnb, metadata)
    return pnb

Converting bibtex citations¶

In [ ]:

#export
_re_cite = re.compile(r"(\\cite{)([^}]*)(})", re.MULTILINE | re.VERBOSE) # Catches citations used with `\cite{}`

In [ ]:

#export
def _textcite2link(text):
    citations = _re_cite.finditer(text)
    out = []
    start_pos = 0
    for cit_group in citations:
        cit_pos_st =  cit_group.span()[0]
        cit_pos_fin =  cit_group.span()[1]
        out.append(text[start_pos:cit_pos_st])
        out.append('[')
        cit_group = cit_group[2].split(',')
        for i, cit in enumerate(cit_group):
            cit=cit.strip()
            out.append(f"""<a class="latex_cit" id="call-{cit}" href="#cit-{cit}">{cit}</a>""")
            if i != len(cit_group) - 1:
                out.append(',')
        out.append(']')
        start_pos = cit_pos_fin
    out.append(text[start_pos:])
    return ''.join(out)

In [ ]:

#export
def cite2link(cell):
    '''Creates links from \cite{} to Reference section generated by jupyter_latex_envs'''
    if cell['cell_type'] == 'markdown': cell['source'] = _textcite2link(cell['source'])
    return cell

jupyter_latex_envs is a jupyter extension https://github.com/jfbercher/jupyter_latex_envs.

You can find relevant section here Note, that nbdev now only supports \cite{} conversion and not the rest, e.g., \figure{} and so on.

In [ ]:

#hide
cell = {'cell_type': 'markdown', 'source': r"""This is cited multireference \cite{Frob1, Frob3}.
And single \cite{Frob2}."""}
expected=r"""This is cited multireference [<a class="latex_cit" id="call-Frob1" href="#cit-Frob1">Frob1</a>,<a class="latex_cit" id="call-Frob3" href="#cit-Frob3">Frob3</a>].
And single [<a class="latex_cit" id="call-Frob2" href="#cit-Frob2">Frob2</a>]."""
test_eq(cite2link(cell)["source"], expected)

It's important to execute all show_doc cells before exporting the notebook to html because some of them have just been added automatically or others could have outdated links.

In [ ]:

#slow
fake_nb = {k:v for k,v in tst_nb.items() if k != 'cells'}
fake_nb['cells'] = [tst_nb['cells'][0].copy()] + added_cells
fake_nb = execute_nb(fake_nb, mod='export')
assert len(fake_nb['cells'][-2]['outputs']) > 0

Filling templates¶

The following functions automatically adds jekyll templates if they are missing.

In [ ]:

#export
def write_tmpl(tmpl, nms, cfg, dest):
    "Write `tmpl` to `dest` (if missing) filling in `nms` in template using dict `cfg`"
    if dest.exists(): return
    vs = {o:cfg.d[o] for o in nms.split()}
    outp = tmpl.format(**vs)
    dest.mk_write(outp)

In [ ]:

#export
def write_tmpls():
    "Write out _config.yml and _data/topnav.yml using templates"
    cfg = get_config()
    path = Path(cfg.get('doc_src_path', cfg.path("doc_path")))
    write_tmpl(config_tmpl, 'user lib_name title copyright description recursive', cfg, path/'_config.yml')
    write_tmpl(topnav_tmpl, 'host git_url', cfg, path/'_data'/'topnav.yml')
    write_tmpl(makefile_tmpl, 'nbs_path lib_name', cfg, cfg.config_file.parent/'Makefile')

In [ ]:

#export
@call_parse
def nbdev_build_lib(
    fname:str=None,  # A notebook name or glob to convert
    bare:store_true=False  # Omit nbdev annotation comments (may break some functionality).
):
    "Export notebooks matching `fname` to python modules"
    if fname is None: write_tmpls()
    notebook2script(fname=fname, bare=bare)

Conversion¶

In [ ]:

__file__ = str(get_config().path("lib_path")/'export2html.py')

In [ ]:

#export
def nbdev_exporter(cls=HTMLExporter, template_file=None):
    cfg = traitlets.config.get_config()
    exporter = cls(cfg)
    exporter.exclude_input_prompt=True
    exporter.exclude_output_prompt=True
    exporter.anchor_link_text = ' '
    exporter.template_file = 'jekyll.tpl' if template_file is None else template_file
    exporter.template_paths.append(str(Path(__file__).parent/'templates'))
    return exporter

In [ ]:

#export
process_cells = [remove_fake_headers, remove_hidden, remove_empty]
process_cell  = [hide_cells, collapse_cells, remove_widget_state, add_jekyll_notes, escape_latex, cite2link]

In [ ]:

#export
def _nb2htmlfname(nb_path, dest=None):
    if dest is None: dest = get_config().path("doc_path")
    return Path(dest)/re_digits_first.sub('', nb_path.with_suffix('.html').name)

In [ ]:

#hide
test_eq(_nb2htmlfname(Path('00a_export.ipynb')), get_config().path("doc_path")/'export.html')
test_eq(_nb2htmlfname(Path('export.ipynb')), get_config().path("doc_path")/'export.html')
test_eq(_nb2htmlfname(Path('00ab_export_module_1.ipynb')), get_config().path("doc_path")/'export_module_1.html')
test_eq(_nb2htmlfname(Path('export.ipynb'), '.'), Path('export.html'))

In [ ]:

#export
def convert_nb(fname, cls=HTMLExporter, template_file=None, exporter=None, dest=None, execute=True):
    "Convert a notebook `fname` to html file in `dest_path`."
    fname = Path(fname).absolute()
    nb = read_nb(fname)
    meta_jekyll = get_metadata(nb['cells'])
    meta_jekyll['nb_path'] = str(fname.relative_to(get_config().path("lib_path").parent))
    cls_lvl = find_default_level(nb['cells'])
    mod = find_default_export(nb['cells'])
    nb['cells'] = compose(*process_cells,partial(add_show_docs, cls_lvl=cls_lvl))(nb['cells'])
    _func = compose(partial(copy_images, fname=fname, dest=get_config().path("doc_path")), *process_cell, treat_backticks)
    nb['cells'] = [_func(c) for c in nb['cells']]
    if execute: nb = execute_nb(nb, mod=mod)
    nb['cells'] = [clean_exports(c) for c in nb['cells']]
    if exporter is None: exporter = nbdev_exporter(cls=cls, template_file=template_file)
    with open(_nb2htmlfname(fname, dest=dest),'w') as f:
        f.write(exporter.from_notebook_node(nb, resources=meta_jekyll)[0])

In [ ]:

#export
def _notebook2html(fname, cls=HTMLExporter, template_file=None, exporter=None, dest=None, execute=True):
    time.sleep(random.random())
    print(f"converting: {fname}")
    try:
        convert_nb(fname, cls=cls, template_file=template_file, exporter=exporter, dest=dest, execute=execute)
        return True
    except Exception as e:
        print(e)
        return False

In [ ]:

#export
def notebook2html(fname=None, force_all=False, n_workers=None, cls=HTMLExporter, template_file=None,
                  exporter=None, dest=None, pause=0, execute=True):
    "Convert all notebooks matching `fname` to html files"
    files = nbglob(fname)
    if len(files)==1:
        force_all = True
        if n_workers is None: n_workers=0
    if not force_all:
        # only rebuild modified files
        files,_files = [],files.copy()
        for fname in _files:
            fname_out = _nb2htmlfname(Path(fname).absolute(), dest=dest)
            if not fname_out.exists() or os.path.getmtime(fname) >= os.path.getmtime(fname_out):
                files.append(fname)
    if len(files)==0: print("No notebooks were modified")
    else:
        if sys.platform == "win32": n_workers = 0
        passed = parallel(_notebook2html, files, n_workers=n_workers, cls=cls,
                          template_file=template_file, exporter=exporter, dest=dest, pause=pause, execute=execute)
        if not all(passed):
            msg = "Conversion failed on the following:\n"
            print(msg + '\n'.join([f.name for p,f in zip(passed,files) if not p]))

In [ ]:

with tempfile.TemporaryDirectory() as d:
    print(d)

In [ ]:

notebook2html('01_sync.ipynb', dest='.', n_workers=0);

In [ ]:

#hide
# Test when an argument is given to notebook2html
with tempfile.TemporaryDirectory() as d:
    p1 = Path(d).joinpath('sync.html') 
    notebook2html('01_sync.ipynb', dest=d, n_workers=0);
    assert p1.exists()

In [ ]:

#slow
#hide
# Test when no argument is given to notebook2html
with tempfile.TemporaryDirectory() as d:
    dest_files = [_nb2htmlfname(Path(f), dest=d) for f in nbglob()]
    [f.unlink() for f in dest_files if f.exists()]
    notebook2html(fname=None, dest=d);
    for f in dest_files: assert f.exists(), f

Hide cells starting with #export and only leaves the prose and the tests. If fname is not specified, this will convert all notebooks not beginning with an underscore in the nb_folder defined in setting.ini. Otherwise fname can be a single filename or a glob expression.

By default, only the notebooks that are more recent than their html counterparts are modified, pass force_all=True to change that behavior.

In [ ]:

#hide
#notebook2html(force_all=True)

In [ ]:

#export
def convert_md(fname, dest_path, img_path='docs/images/', jekyll=True):
    "Convert a notebook `fname` to a markdown file in `dest_path`."
    fname = Path(fname).absolute()
    if not img_path: img_path = fname.stem + '_files/'
    Path(img_path).mkdir(exist_ok=True, parents=True)
    nb = read_nb(fname)
    meta_jekyll = get_metadata(nb['cells'])
    try: meta_jekyll['nb_path'] = str(fname.relative_to(get_config().path("lib_path").parent))
    except: meta_jekyll['nb_path'] = str(fname)
    nb['cells'] = compose(*process_cells)(nb['cells'])
    nb['cells'] = [compose(partial(adapt_img_path, fname=fname, dest=dest_path, jekyll=jekyll), *process_cell)(c)
                   for c in nb['cells']]
    fname = Path(fname).absolute()
    dest_name = fname.with_suffix('.md').name
    exp = nbdev_exporter(cls=MarkdownExporter, template_file='jekyll-md.tpl' if jekyll else 'md.tpl')
    export = exp.from_notebook_node(nb, resources=meta_jekyll)
    md = export[0]
    for ext in ['png', 'svg']:
        md = re.sub(r'!\['+ext+'\]\((.+)\)', '!['+ext+'](' + img_path + '\\1)', md)
    with (Path(dest_path)/dest_name).open('w') as f: f.write(md)
    if hasattr(export[1]['outputs'], 'items'):
        for n,o in export[1]['outputs'].items():
            with open(Path(dest_path)/img_path/n, 'wb') as f: f.write(o)

This is used to convert the index into the README.md.

In [ ]:

#hide
def _test_md(fn):
    fn,dest = Path(fn),Path().absolute().parent
    try: convert_md(fn, dest, jekyll=False)
    finally: (dest/f'{fn.stem}.md').unlink()

In [ ]:

#hide
_test_md('index.ipynb')
# `export[1]['outputs']` will be a `str` if the notebook has no markdown cells to convert.
# e.g. the nb could have a single jekyll markdown cell or just code cells ...
_test_md(f'../tests/single-cell-index.ipynb')

In [ ]:

#export
_re_att_ref = re.compile(r' *!\[(.*)\]\(attachment:image.png(?: "(.*)")?\)')

In [ ]:

t = '![screenshot](attachment:image.png)'
test_eq(_re_att_ref.match(t).groups(), ('screenshot', None))

t = '![screenshot](attachment:image.png "Deploying to Binder")'
test_eq(_re_att_ref.match(t).groups(), ('screenshot', "Deploying to Binder"))

In [ ]:

#export
try: from PIL import Image
except: pass # Only required for _update_att_ref

In [ ]:

#export
_tmpl_img = '<img alt="{title}" width="{width}" caption="{title}" id="{id}" src="{name}">'

def _update_att_ref(line, path, img):
    m = _re_att_ref.match(line)
    if not m: return line
    alt,title = m.groups()
    w = img.size[0]
    if alt=='screenshot': w //= 2
    if not title: title = "TK: add title"
    return _tmpl_img.format(title=title, width=str(w), id='TK: add it', name=str(path))

In [ ]:

#export
def _nb_detach_cell(cell, dest, use_img):
    att,src = cell['attachments'],cell['source']
    mime,img = first(first(att.values()).items())
    ext = mime.split('/')[1]
    for i in range(99999):
        p = dest/(f'att_{i:05d}.{ext}')
        if not p.exists(): break
    img = b64decode(img)
    p.write_bytes(img)
    del(cell['attachments'])
    if use_img:  return [_update_att_ref(o,p,Image.open(p)) for o in src]
    else: return [o.replace('attachment:image.png', str(p)) for o in src]

In [ ]:

#export
def _nbdev_detach(path_nb, dest="", use_img=False, replace=True):
    path_nb = Path(path_nb)
    if not dest: dest = f'{path_nb.stem}_files'
    dest = Path(dest)
    dest.mkdir(exist_ok=True, parents=True)
    j = json.load(path_nb.open())
    atts = [o for o in j['cells'] if 'attachments' in o]
    for o in atts: o['source'] = _nb_detach_cell(o, dest, use_img)
    if atts and replace: json.dump(j, path_nb.open('w'))
    if not replace: return j

@call_parse
def nbdev_detach(path_nb:Param("Path to notebook"),
    dest:str="",  # Destination folder
    use_img:bool_arg=False,  # Convert markdown images to img tags
    replace:bool_arg=True  # Write replacement notebook back to `path_bn`
):
    "Export cell attachments to `dest` and update references"
    _nbdev_detach(path_nb, dest, use_img, replace)

In [ ]:

#export
_re_index = re.compile(r'^(?:\d*_|)index\.ipynb$')

In [ ]:

#export
def make_readme():
    "Convert the index notebook to README.md"
    index_fn = None
    for f in get_config().path("nbs_path").glob('*.ipynb'):
        if _re_index.match(f.name): index_fn = f
    assert index_fn is not None, "Could not locate index notebook"
    print(f"converting {index_fn} to README.md")
    convert_md(index_fn, get_config().config_file.parent, jekyll=False)
    n = get_config().config_file.parent/index_fn.with_suffix('.md').name
    shutil.move(n, get_config().config_file.parent/'README.md')
    if Path(get_config().config_file.parent/'PRE_README.md').is_file():
        with open(get_config().config_file.parent/'README.md', 'r') as f: readme = f.read()
        with open(get_config().config_file.parent/'PRE_README.md', 'r') as f: pre_readme = f.read()
        with open(get_config().config_file.parent/'README.md', 'w') as f: f.write(f'{pre_readme}\n{readme}')

In [ ]:

#export
@call_parse
def nbdev_build_docs(
    fname:str=None,  # A notebook name or glob to convert
    force_all:bool_arg=False, # Rebuild even notebooks that havent changed
    mk_readme:bool_arg=True,  # Also convert the index notebook to README
    n_workers:int=None,  # Number of workers to use
    pause:float=0.5  # Pause time (in secs) between notebooks to avoid race conditions
):
    "Build the documentation by converting notebooks matching `fname` to html"
    notebook2html(fname=fname, force_all=force_all, n_workers=n_workers, pause=pause)
    if fname is None: make_sidebar()
    if mk_readme: make_readme()

In [ ]:

#export
@call_parse
def nbdev_nb2md(
    fname:str,  # A notebook file name to convert
    dest:str='.',  # The destination folder
    img_path:None="",  # Folder to export images to
    jekyll:bool_arg=False  # To use jekyll metadata for your markdown file or not
):
    "Convert the notebook in `fname` to a markdown file"
    _nbdev_detach(fname, dest=img_path)
    convert_md(fname, dest, jekyll=jekyll, img_path=img_path)

In [ ]:

#export
import time,random,warnings

In [ ]:

#export
def _leaf(k,v):
    url = 'external_url' if "http" in v else 'url'
    #if url=='url': v=v+'.html'
    return {'title':k, url:v, 'output':'web,pdf'}

In [ ]:

#export
_k_names = ['folders', 'folderitems', 'subfolders', 'subfolderitems']
def _side_dict(title, data, level=0):
    k_name = _k_names[level]
    level += 1
    res = [(_side_dict(k, v, level) if isinstance(v,dict) else _leaf(k,v))
        for k,v in data.items()]
    return ({k_name:res} if not title
            else res if title.startswith('empty')
            else {'title': title, 'output':'web', k_name: res})

In [ ]:

#export
_re_catch_title = re.compile('^title\s*:\s*(\S+.*)$', re.MULTILINE)

In [ ]:

#export
def _get_title(fname):
    "Grabs the title of html file `fname`"
    with open(fname, 'r') as f: code = f.read()
    src =  _re_catch_title.search(code)
    return fname.stem if src is None else src.groups()[0]

In [ ]:

#hide
test_eq(_get_title(get_config().path("doc_path")/'export.html'), "Export to modules")

In [ ]:

#export
def _create_default_sidebar():
    "Create the default sidebar for the docs website"
    dic = {"Overview": "/"}
    files = nbglob()
    fnames = [_nb2htmlfname(f) for f in sorted(files)]
    names = [f for f in fnames if f.stem!='index']
    for t,f in groupby(names, _get_title).items():
        if len(f) > 1: print(f'WARNING: The title: "{t}" appears in {len(f)} pages:\n\t\t{L(f).map(str)}')
    dic.update({_get_title(f):f.name if get_config().host=='github' else f.with_suffix('').name for f in fnames if f.stem!='index'})
    return dic

In [ ]:

#export
def create_default_sidebar():
    "Create the default sidebar for the docs website"
    dic = {get_config().lib_name: _create_default_sidebar()}
    json.dump(dic, open(get_config().path("doc_path")/'sidebar.json', 'w'), indent=2)

The default sidebar lists all html pages with their respective title, except the index that is named "Overview". To build a custom sidebar, set the flag custom_sidebar in your settings.ini to True then change the sidebar.json file in the doc_folder to your liking. Otherwise, the sidebar is updated at each doc build.

In [ ]:

#export
def make_sidebar():
    "Making sidebar for the doc website form the content of `doc_folder/sidebar.json`"
    cfg = get_config()
    if not (cfg.path("doc_path")/'sidebar.json').exists() or cfg.get('custom_sidebar', 'False') == 'False':
        create_default_sidebar()
    sidebar_d = json.load(open(cfg.path("doc_path")/'sidebar.json', 'r'))
    res = _side_dict('Sidebar', sidebar_d)
    res = {'entries': [res]}
    res_s = yaml.dump(res, default_flow_style=False)
    res_s = res_s.replace('- subfolders:', '  subfolders:').replace(' - - ', '   - ')
    res_s = f"""
#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# Instead edit {'../../sidebar.json'}
"""+res_s
    pth = cfg.path("doc_path")/'_data/sidebars/home_sidebar.yml'
    pth.mk_write(res_s)

In [ ]:

make_sidebar()

Export-¶

In [ ]:

#hide
from nbdev.export import *
notebook2script()

Converted 00_export.ipynb.
Converted 01_sync.ipynb.
Converted 02_showdoc.ipynb.
Converted 03_export2html.ipynb.
Converted 04_test.ipynb.
Converted 05_merge.ipynb.
Converted 06_cli.ipynb.
Converted 07_clean.ipynb.
Converted 99_search.ipynb.
Converted example.ipynb.
Converted index.ipynb.
Converted nbdev_comments.ipynb.
Converted tutorial.ipynb.
Converted tutorial_colab.ipynb.

In [ ]:

Convert to html¶

Preprocessing notebook¶

Cell processors¶

Collapsable Code Cells¶

Preprocessing the list of cells¶

Grabbing metadata¶

Executing show_doc cells¶

Converting bibtex citations¶

Filling templates¶

Conversion¶

Sidebar¶

Export-¶