#hide
#default_exp showdoc
#default_cls_lvl 3
from nbdev.showdoc import show_doc
#export
from nbdev.imports import *
from nbdev.export import *
from nbdev.sync import *
from nbconvert import HTMLExporter
from fastcore.utils import IN_NOTEBOOK
if IN_NOTEBOOK:
from IPython.display import Markdown,display
from IPython.core import page
Functions to show the doc cells in notebooks
All the automatic documentation of functions and classes are generated with the show_doc
function. It displays the name, arguments, docstring along with a link to the source code on GitHub.
The inspect module lets us know quickly if an object is a function or a class but it doesn't distinguish classes and enums.
#export
def is_enum(cls):
"Check if `cls` is an enum or another type of class"
return type(cls) in (enum.Enum, enum.EnumMeta)
e = enum.Enum('e', 'a b')
assert is_enum(e)
assert not is_enum(e.__class__)
assert not is_enum(int)
#export
def is_lib_module(name):
"Test if `name` is a library module."
if name.startswith('_'): return False
try:
_ = importlib.import_module(f'{Config().lib_name}.{name}')
return True
except: return False
assert is_lib_module('export')
assert not is_lib_module('transform')
#export
re_digits_first = re.compile('^[0-9]+[a-z]*_')
#export
def try_external_doc_link(name, packages):
"Try to find a doc link for `name` in `packages`"
for p in packages:
try:
mod = importlib.import_module(f"{p}._nbdev")
try_pack = source_nb(name, is_name=True, mod=mod)
if try_pack:
page = re_digits_first.sub('', try_pack).replace('.ipynb', '')
return f'{mod.doc_url}{page}#{name}'
except ModuleNotFoundError: return None
This function will only work for other packages built with nbdev
.
test_eq(try_external_doc_link('get_name', ['nbdev']), 'https://nbdev.fast.ai/sync#get_name')
#export
def is_doc_name(name):
"Test if `name` corresponds to a notebook that could be converted to a doc page"
for f in Config().path("nbs_path").glob(f'*{name}.ipynb'):
if re_digits_first.sub('', f.name) == f'{name}.ipynb': return True
return False
test_eq(is_doc_name('flaags'),False)
test_eq(is_doc_name('export'),True)
test_eq(is_doc_name('index'),True)
#export
def doc_link(name, include_bt=True):
"Create link to documentation for `name`."
cname = f'`{name}`' if include_bt else name
try:
#Link to modules
if is_lib_module(name) and is_doc_name(name): return f"[{cname}]({Config().doc_baseurl}{name}.html)"
#Link to local functions
try_local = source_nb(name, is_name=True)
if try_local:
page = re_digits_first.sub('', try_local).replace('.ipynb', '')
return f'[{cname}]({Config().doc_baseurl}{page}.html#{name})'
##Custom links
mod = get_nbdev_module()
link = mod.custom_doc_links(name)
return f'[{cname}]({link})' if link is not None else cname
except: return cname
This function will generate links for modules (pointing to the html conversion of the corresponding notebook) and functions (pointing to the html conversion of the notebook where they were defined, with the first anchor found before). If the function/module is not part of the library you are writing, it will call the function custom_doc_links
generated in _nbdev
(you can customize it to your needs) and just return the name between backticks if that function returns None
.
For instance, fastai has the following custom_doc_links
that tries to find a doc link for name
in fastcore then nbdev (in this order):
def custom_doc_links(name):
from nbdev.showdoc import try_external_doc_link
return try_external_doc_link(name, ['fastcore', 'nbdev'])
Please note that module links only work if your notebook names "correspond" to your module names:
Notebook name | Doc name | Module name | Module file | Can doc link? |
---|---|---|---|---|
export.ipynb | export.html | export | export.py | Yes |
00_export.ipynb | export.html | export | export.py | Yes |
00a_export.ipynb | export.html | export | export.py | Yes |
export_1.ipynb | export_1.html | export | export.py | No |
03_data.core.ipynb | data.core.html | data.core | data/core.py | Yes |
03_data_core.ipynb | data_core.html | data.core | data/core.py | No |
test_eq(doc_link('export'), f'[`export`](/export.html)')
test_eq(doc_link('DocsTestClass'), f'[`DocsTestClass`](/export.html#DocsTestClass)')
test_eq(doc_link('DocsTestClass.test'), f'[`DocsTestClass.test`](/export.html#DocsTestClass.test)')
test_eq(doc_link('Tenso'),'`Tenso`')
test_eq(doc_link('_nbdev'), f'`_nbdev`')
test_eq(doc_link('__main__'), f'`__main__`')
test_eq(doc_link('flags'), '`flags`') # we won't have a flags doc page even though we do have a flags module
#export
_re_backticks = re.compile(r"""
# Catches any link of the form \[`obj`\](old_link) or just `obj`,
# to either update old links or add the link to the docs of obj
\[` # Opening [ and `
([^`]*) # Catching group with anything but a `
`\] # ` then closing ]
(?: # Beginning of non-catching group
\( # Opening (
[^)]* # Anything but a closing )
\) # Closing )
) # End of non-catching group
| # OR
` # Opening `
([^`]*) # Anything but a `
` # Closing `
""", re.VERBOSE)
#export
def add_doc_links(text, elt=None):
"Search for doc links for any item between backticks in `text` and insert them"
def _replace_link(m):
try:
if m.group(2) in inspect.signature(elt).parameters: return f'`{m.group(2)}`'
except: pass
return doc_link(m.group(1) or m.group(2))
return _re_backticks.sub(_replace_link, text)
This function not only add links to backtick keywords, it also update the links that are already in the text (in case they have changed).
tst = add_doc_links('This is an example of `DocsTestClass`')
test_eq(tst, "This is an example of [`DocsTestClass`](/export.html#DocsTestClass)")
tst = add_doc_links('This is an example of [`DocsTestClass`](old_link.html)')
test_eq(tst, "This is an example of [`DocsTestClass`](/export.html#DocsTestClass)")
Names in backticks will not be converted to links if elt
has a parameter of the same name
def t(a,export):
"Test func that uses 'export' as a parameter name and has `export` in its doc string"
assert '[`export`](/export.html)' not in add_doc_links(t.__doc__, t)
Names in backticks used in markdown links will be updated like normal
def t(a,export):
"Test func that uses 'export' as a parameter name and has [`export`]() in its doc string"
assert '[`export`](/export.html)' in add_doc_links(t.__doc__, t)
#hide
# if the name in backticks is not a param name, links will be added/updated like normal
def t(a,exp):
"Test func with `export` in its doc string"
assert '[`export`](/export.html)' in add_doc_links(t.__doc__, t)
def t(a,exp):
"Test func with [`export`]() in its doc string"
assert '[`export`](/export.html)' in add_doc_links(t.__doc__, t)
If elt
is a class, add_doc_links
looks at parameter names used in __init__
class T:
def __init__(self, add_doc_links): pass
test_eq(add_doc_links('Lets talk about `add_doc_links`'),
'Lets talk about [`add_doc_links`](/showdoc.html#add_doc_links)')
test_eq(add_doc_links('Lets talk about `add_doc_links`', T), 'Lets talk about `add_doc_links`')
test_eq(add_doc_links('Lets talk about `doc_link`'),
'Lets talk about [`doc_link`](/showdoc.html#doc_link)')
test_eq(add_doc_links('Lets talk about `doc_link`', T),
'Lets talk about [`doc_link`](/showdoc.html#doc_link)')
#export
def _is_type_dispatch(x): return type(x).__name__ == "TypeDispatch"
def _unwrapped_type_dispatch_func(x): return x.first() if _is_type_dispatch(x) else x
def _is_property(x): return type(x)==property
def _has_property_getter(x): return _is_property(x) and hasattr(x, 'fget') and hasattr(x.fget, 'func')
def _property_getter(x): return x.fget.func if _has_property_getter(x) else x
def _unwrapped_func(x):
x = _unwrapped_type_dispatch_func(x)
x = _property_getter(x)
return x
#export
def get_source_link(func):
"Return link to `func` in source code"
func = _unwrapped_func(func)
try: line = inspect.getsourcelines(func)[1]
except Exception: return ''
mod = inspect.getmodule(func)
module = mod.__name__.replace('.', '/') + '.py'
try:
nbdev_mod = importlib.import_module(mod.__package__.split('.')[0] + '._nbdev')
return f"{nbdev_mod.git_url}{module}#L{line}"
except: return f"{module}#L{line}"
Be sure to properly set the git_url
in setting.ini (derived from lib_name
and branch
on top of the prefix you will need to adapt) so that those links are correct.
#hide
assert get_source_link(DocsTestClass.test).startswith(Config().git_url + 'nbdev/export.py')
#hide
#test
from fastcore.foundation import L
assert get_source_link(L).startswith("https://github.com/fastai/fastcore/tree/master/fastcore/foundation.py")
As important as the source code, we want to quickly jump to where the function is defined when we are in a development notebook.
#export
_re_header = re.compile(r"""
# Catches any header in markdown with the title in group 1
^\s* # Beginning of text followed by any number of whitespace
\#+ # One # or more
\s* # Any number of whitespace
(.*) # Catching group with anything
$ # End of text
""", re.VERBOSE)
#export
def colab_link(path):
"Get a link to the notebook at `path` on Colab"
cfg = Config()
res = f'https://colab.research.google.com/github/{cfg.user}/{cfg.lib_name}/blob/{cfg.branch}/{cfg.path("nbs_path").name}/{path}.ipynb'
display(Markdown(f'[Open `{path}` in Colab]({res})'))
colab_link('02_showdoc')
#export
def get_nb_source_link(func, local=False, is_name=None):
"Return a link to the notebook where `func` is defined."
func = _unwrapped_type_dispatch_func(func)
pref = '' if local else Config().git_url.replace('github.com', 'nbviewer.jupyter.org/github')+ Config().path("nbs_path").name+'/'
is_name = is_name or isinstance(func, str)
src = source_nb(func, is_name=is_name, return_all=True)
if src is None: return '' if is_name else get_source_link(func)
find_name,nb_name = src
nb = read_nb(nb_name)
pat = re.compile(f'^{find_name}\s+=|^(def|class)\s+{find_name}\s*\(', re.MULTILINE)
if len(find_name.split('.')) == 2:
clas,func = find_name.split('.')
pat2 = re.compile(f'@patch\s*\ndef\s+{func}\s*\([^:]*:\s*{clas}\s*(?:,|\))')
else: pat2 = None
for i,cell in enumerate(nb['cells']):
if cell['cell_type'] == 'code':
if re.search(pat, cell['source']): break
if pat2 is not None and re.search(pat2, cell['source']): break
if re.search(pat, cell['source']) is None and (pat2 is not None and re.search(pat2, cell['source']) is None):
return '' if is_name else get_function_source(func)
header_pat = re.compile(r'^\s*#+\s*(.*)$')
while i >= 0:
cell = nb['cells'][i]
if cell['cell_type'] == 'markdown' and _re_header.search(cell['source']):
title = _re_header.search(cell['source']).groups()[0]
anchor = '-'.join([s for s in title.split(' ') if len(s) > 0])
return f'{pref}{nb_name}#{anchor}'
i-=1
return f'{pref}{nb_name}'
test_eq(get_nb_source_link(DocsTestClass.test), get_nb_source_link(DocsTestClass))
test_eq(get_nb_source_link('DocsTestClass'), get_nb_source_link(DocsTestClass))
test_eq(get_nb_source_link(check_re, local=True), f'00_export.ipynb#Finding-patterns')
You can either pass an object or its name (by default is_name
will look if func
is a string or not to decide if it's True
or False
, but you can override if there is some inconsistent behavior). local
will return a local link, otherwise it will point to a the notebook on Google Colab.
#export
def nb_source_link(func, is_name=None, disp=True, local=True):
"Show a relative link to the notebook where `func` is defined"
is_name = is_name or isinstance(func, str)
func_name = func if is_name else qual_name(func)
link = get_nb_source_link(func, local=local, is_name=is_name)
text = func_name if local else f'{func_name} (GitHub)'
if disp: display(Markdown(f'[{text}]({link})'))
else: return link
This function assumes you are in one notebook in the development folder, otherwise you can use disp=False
to get the relative link. You can either pass an object or its name (by default is_name
will look if func
is a string or not to decide if it's True
or False
, but you can override if there is some inconsistent behavior).
test_eq(nb_source_link(check_re, disp=False), f'00_export.ipynb#Finding-patterns')
test_eq(nb_source_link('check_re', disp=False), f'00_export.ipynb#Finding-patterns')
nb_source_link(check_re, local=False)
#export
from fastcore.script import Param
#export
def type_repr(t):
"Representation of type `t` (in a type annotation)"
if (isinstance(t, Param)): return f'"{t.help}"'
if getattr(t, '__args__', None):
args = t.__args__
if len(args)==2 and args[1] == type(None):
return f'`Optional`\[{type_repr(args[0])}\]'
reprs = ', '.join([type_repr(o) for o in args])
return f'{doc_link(get_name(t))}\[{reprs}\]'
else: return doc_link(get_name(t))
The representation tries to find doc links if possible.
tst = type_repr(Optional[DocsTestClass])
test_eq(tst, '`Optional`\\[[`DocsTestClass`](/export.html#DocsTestClass)\\]')
tst = type_repr(Union[int, float])
test_eq(tst, '`Union`\\[`int`, `float`\\]')
test_eq(type_repr(Param("description")), '"description"')
#export
_arg_prefixes = {inspect._VAR_POSITIONAL: '\*', inspect._VAR_KEYWORD:'\*\*'}
def format_param(p):
"Formats function param to `param:Type=val` with font weights: param=bold, val=italic"
arg_prefix = _arg_prefixes.get(p.kind, '') # asterisk prefix for *args and **kwargs
res = f"**{arg_prefix}`{p.name}`**"
if hasattr(p, 'annotation') and p.annotation != p.empty: res += f':{type_repr(p.annotation)}'
if p.default != p.empty:
default = getattr(p.default, 'func', p.default) #For partials
if hasattr(default,'__name__'): default = getattr(default, '__name__')
else: default = repr(default)
if is_enum(default.__class__): #Enum have a crappy repr
res += f'=*`{default.__class__.__name__}.{default.name}`*'
else: res += f'=*`{default}`*'
return res
sig = inspect.signature(notebook2script)
params = [format_param(p) for _,p in sig.parameters.items()]
test_eq(params, ['**`fname`**=*`None`*', '**`silent`**=*`False`*', '**`to_dict`**=*`False`*', '**`bare`**=*`False`*', '**`recursive`**=*`None`*'])
#export
def _format_enum_doc(enum, full_name):
"Formatted `enum` definition to show in documentation"
vals = ', '.join(enum.__members__.keys())
return f'<code>{full_name}</code>',f'<code>Enum</code> = [{vals}]'
#hide
tst = _format_enum_doc(e, 'e')
test_eq(tst, ('<code>e</code>', '<code>Enum</code> = [a, b]'))
#export
def _escape_chars(s):
return s.replace('_', '\_')
def _format_func_doc(func, full_name=None):
"Formatted `func` definition to show in documentation"
try:
sig = inspect.signature(func)
fmt_params = [format_param(param) for name,param
in sig.parameters.items() if name not in ('self','cls')]
except: fmt_params = []
name = f'<code>{full_name or func.__name__}</code>'
arg_str = f"({', '.join(fmt_params)})"
f_name = f"<code>class</code> {name}" if inspect.isclass(func) else name
return f'{f_name}',f'{name}{arg_str}'
#hide
test_eq(_format_func_doc(notebook2script), ('<code>notebook2script</code>',
'<code>notebook2script</code>(**`fname`**=*`None`*, **`silent`**=*`False`*, **`to_dict`**=*`False`*, **`bare`**=*`False`*, **`recursive`**=*`None`*)'))
#export
def _format_cls_doc(cls, full_name):
"Formatted `cls` definition to show in documentation"
parent_class = inspect.getclasstree([cls])[-1][0][1][0]
name,args = _format_func_doc(cls, full_name)
if parent_class != object: args += f' :: {doc_link(get_name(parent_class))}'
return name,args
#hide
test_eq(_format_cls_doc(DocsTestClass, 'DocsTestClass'), ('<code>class</code> <code>DocsTestClass</code>',
'<code>DocsTestClass</code>()'))
#export
def show_doc(elt, doc_string=True, name=None, title_level=None, disp=True, default_cls_level=2):
"Show documentation for element `elt`. Supported types: class, function, and enum."
elt = getattr(elt, '__func__', elt)
qname = name or qual_name(elt)
if inspect.isclass(elt):
if is_enum(elt): name,args = _format_enum_doc(elt, qname)
else: name,args = _format_cls_doc (elt, qname)
elif callable(elt): name,args = _format_func_doc(elt, qname)
else: name,args = f"<code>{qname}</code>", ''
link = get_source_link(elt)
source_link = f'<a href="{link}" class="source_link" style="float:right">[source]</a>'
title_level = title_level or (default_cls_level if inspect.isclass(elt) else 4)
doc = f'<h{title_level} id="{qname}" class="doc_header">{name}{source_link}</h{title_level}>'
doc += f'\n\n> {args}\n\n' if len(args) > 0 else '\n\n'
if doc_string and inspect.getdoc(elt):
s = inspect.getdoc(elt)
# show_doc is used by doc so should not rely on Config
try: monospace = (Config().get('monospace_docstrings') == 'True')
except: monospace = False
# doc links don't work inside markdown pre/code blocks
s = f'```\n{s}\n```' if monospace else add_doc_links(s, elt)
doc += s
if disp: display(Markdown(doc))
else: return doc
doc_string
determines if we show the docstring of the function or not. name
can be used to provide an alternative to the name automatically found. title_level
determines the level of the anchor (default 3 for classes and 4 for functions). If disp
is False
, the function returns the markdown code instead of displaying it. If doc_string
is True
and monospace_docstrings
is set to True
in settings.ini
, the docstring of the function is formatted in a code block to preserve whitespace.
For instance
show_doc(notebook2script)
will display
show_doc(notebook2script)
notebook2script
[source]
notebook2script
(fname
=None
,silent
=False
,to_dict
=False
,bare
=False
,recursive
=None
)
Convert notebooks matching fname
to modules
#hide
def t(a,exp):
"Test func with `export` in its doc string"
assert '[`export`](/export.html)' in show_doc(t, disp=False)
def t(a,export):
"Test func that uses 'export' as a parameter name and has `export` in its doc string"
assert '[`export`](/export.html)' not in show_doc(t, disp=False)
#hide
show_doc(DocsTestClass)
#hide
show_doc(DocsTestClass.test)
DocsTestClass.test
[source]
DocsTestClass.test
()
#hide
show_doc(check_re)
#hide
show_doc(e)
#hide
def test_func_with_args_and_links(foo, bar):
"""
Doc link: `show_doc`.
Args:
foo: foo
bar: bar
Returns:
None
"""
pass
show_doc(test_func_with_args_and_links)
Config()["monospace_docstrings"] = "True"
show_doc(test_func_with_args_and_links)
Config()["monospace_docstrings"] = "False"
#export
def md2html(md):
"Convert markdown `md` to HTML code"
import nbconvert
if nbconvert.__version__ < '5.5.0': return HTMLExporter().markdown2html(md)
else: return HTMLExporter().markdown2html(collections.defaultdict(lambda: collections.defaultdict(dict)), md)
#export
def get_doc_link(func):
mod = inspect.getmodule(func)
module = mod.__name__.replace('.', '/') + '.py'
try:
nbdev_mod = importlib.import_module(mod.__package__.split('.')[0] + '._nbdev')
try_pack = source_nb(func, mod=nbdev_mod)
if try_pack:
page = '.'.join(try_pack.partition('_')[-1:]).replace('.ipynb', '')
return f'{nbdev_mod.doc_url}{page}#{qual_name(func)}'
except: return None
test_eq(get_doc_link(notebook2script), 'https://nbdev.fast.ai/export#notebook2script')
#test
from nbdev.sync import get_name
test_eq(get_doc_link(get_name), 'https://nbdev.fast.ai/sync#get_name')
#export
def doc(elt):
"Show `show_doc` info in preview window when used in a notebook"
md = show_doc(elt, disp=False)
doc_link = get_doc_link(elt)
if doc_link is not None:
md += f'\n\n<a href="{doc_link}" target="_blank" rel="noreferrer noopener">Show in docs</a>'
output = md2html(md)
if IN_COLAB: get_ipython().run_cell_magic(u'html', u'', output)
else:
try: page.page({'text/html': output})
except: display(Markdown(md))
#hide
from nbdev.export import notebook2script
notebook2script()
Converted 00_export.ipynb. Converted 01_sync.ipynb. Converted 02_showdoc.ipynb. Converted 03_export2html.ipynb. Converted 04_test.ipynb. Converted 05_merge.ipynb. Converted 06_cli.ipynb. Converted 07_clean.ipynb. Converted 99_search.ipynb. Converted example.ipynb. Converted index.ipynb. Converted tutorial.ipynb.