#hide
#default_exp sync
#default_cls_lvl 3
from nbdev.showdoc import show_doc
#export
from nbdev.imports import *
from nbdev.export import *
from fastcore.script import *
import nbformat
from nbformat.sign import NotebookNotary
The functions that propagates small changes in the library back to notebooks
The library is primarily developed in notebooks so any big changes should be made there. But sometimes, it's easier to fix small bugs or typos in the modules directly. nbdev_update_lib is the function that will propagate those changes back to the corresponding notebooks. Note that you can't create new cells with that functionality, so your corrections should remain limited.
We need to get the name of the object we are looking for, and then we'll try to find it in our index file.
#export
def _get_property_name(p):
"Get the name of property `p`"
if hasattr(p, 'fget'):
return p.fget.func.__qualname__ if hasattr(p.fget, 'func') else p.fget.__qualname__
else: return next(iter(re.findall(r'\'(.*)\'', str(p)))).split('.')[-1]
def get_name(obj):
"Get the name of `obj`"
if hasattr(obj, '__name__'): return obj.__name__
elif getattr(obj, '_name', False): return obj._name
elif hasattr(obj,'__origin__'): return str(obj.__origin__).split('.')[-1] #for types
elif type(obj)==property: return _get_property_name(obj)
else: return str(obj).split('.')[-1]
from nbdev.export import DocsTestClass
test_eq(get_name(in_ipython), 'in_ipython')
test_eq(get_name(DocsTestClass.test), 'test')
#export
def qual_name(obj):
"Get the qualified name of `obj`"
if hasattr(obj,'__qualname__'): return obj.__qualname__
if inspect.ismethod(obj): return f"{get_name(obj.__self__)}.{get_name(fn)}"
return get_name(obj)
Qualified name is different from name in python for methods and properties:
test_eq(qual_name(DocsTestClass.test), 'DocsTestClass.test')
#hide
class _PropertyClass:
p_lambda = property(lambda x: x)
def some_getter(self): return 7
p_getter = property(some_getter)
test_eq(get_name(_PropertyClass.p_lambda), '_PropertyClass.<lambda>')
test_eq(get_name(_PropertyClass.p_getter), '_PropertyClass.some_getter')
test_eq(get_name(_PropertyClass), '_PropertyClass')
#export
def source_nb(func, is_name=None, return_all=False, mod=None):
"Return the name of the notebook where `func` was defined"
is_name = is_name or isinstance(func, str)
if mod is None: mod = get_nbdev_module()
index = mod.index
name = func if is_name else qual_name(func)
while len(name) > 0:
if name in index: return (name,index[name]) if return_all else index[name]
name = '.'.join(name.split('.')[:-1])
You can either pass an object or its name (by default is_name will look if func is a string or not to determine if it should be True or False, but you can override if there is some inconsistent behavior).
If passed a method of a class, the function will return the notebook in which the largest part of the function name was defined in case there is a monkey-matching that defines class.method in a different notebook than class. If return_all=True, the function will return a tuple with the name by which the function was found and the notebook.
For properties defined using property or our own add_props helper, we approximate the name by looking at their getter functions, since we don't seem to have access to the property name itself. If everything fails (a getter cannot be found), we return the name of the object that contains the property. This suffices for source_nb to work.
test_eq(source_nb(notebook2script), '00_export.ipynb')
test_eq(source_nb(DocsTestClass), '00_export.ipynb')
test_eq(source_nb(DocsTestClass.test), '00_export.ipynb')
assert source_nb(int) is None
If someone decides to change a module instead of the notebooks, the following functions help update the notebooks accordingly.
#export
_re_cell = re.compile(r'^# Cell|^# Internal Cell|^# Comes from\s+(\S+), cell')
#export
def _split(code):
lines = code.split('\n')
nbs_path = Config().path("nbs_path").relative_to(Config().config_file.parent)
prefix = '' if nbs_path == Path('.') else f'{nbs_path}/'
default_nb = re.search(f'File to edit: {prefix}(\\S+)\\s+', lines[0]).groups()[0]
s,res = 1,[]
while _re_cell.search(lines[s]) is None: s += 1
e = s+1
while e < len(lines):
while e < len(lines) and _re_cell.search(lines[e]) is None: e += 1
grps = _re_cell.search(lines[s]).groups()
nb = grps[0] or default_nb
content = lines[s+1:e]
while len(content) > 1 and content[-1] == '': content = content[:-1]
res.append((nb, '\n'.join(content)))
s,e = e,e+1
return res
#export
def relimport2name(name, mod_name):
"Unwarps a relative import in `name` according to `mod_name`"
mod_name = str(Path(mod_name))
if mod_name.endswith('.py'): mod_name = mod_name[:-3]
mods = mod_name.split(os.path.sep)
i = last_index(Config().lib_name, mods)
mods = mods[i:]
if name=='.': return '.'.join(mods[:-1])
i = 0
while name[i] == '.': i += 1
return '.'.join(mods[:-i] + [name[i:]])
When we say from
from .submodule import bla
in a module, it needs to be converted to something like
from module.submodule import bla
or
from module1.module2.submodule import bla
depending on where we are. This function deals with those imports renaming.
test_eq(relimport2name('.core', 'nbdev/data.py'), 'nbdev.core')
test_eq(relimport2name('.core', 'home/sgugger/fastai_dev/nbdev/nbdev/data.py'), 'nbdev.core')
test_eq(relimport2name('..core', 'nbdev/vision/data.py'), 'nbdev.core')
test_eq(relimport2name('.transform', 'nbdev/vision/data.py'), 'nbdev.vision.transform')
test_eq(relimport2name('..notebook.core', 'nbdev/data/external.py'), 'nbdev.notebook.core')
#export
#Catches any from .bla import something and catches .bla in group 1, the imported thing(s) in group 2.
_re_loc_import = re.compile(r'(\s*)from(\s+)(\.\S*)(\s+)import(\s+)(.*)$')
_re_loc_import1 = re.compile(r'(\s*)import(\s+)(\.\S*)(.*)$')
#export
def _deal_loc_import(code, fname):
def _replace(m):
s1,s2,mod,s3,s4,obj = m.groups()
return f"{s1}from{s2}{relimport2name(mod, fname)}{s3}import{s4}{obj}"
def _replace1(m):
s1,s2,mod,end = m.groups()
return f"{s1}import{s2}{relimport2name(mod, fname)}{end}"
return '\n'.join([_re_loc_import1.sub(_replace1, _re_loc_import.sub(_replace,line)) for line in code.split('\n')])
#hide
code = \
"""from .core import *
nothing to see
\t from \t .vision \t import \t bla1, \nbla2
import .vision
import .utils as u"""
test_eq(_deal_loc_import(code, 'nbdev/data.py'),
"""from nbdev.core import *
nothing to see
\t from \t nbdev.vision \t import \t bla1, \nbla2
import nbdev.vision
import nbdev.utils as u""")
#export
def _script2notebook(fname, dic, silent=False):
"Put the content of `fname` back in the notebooks it came from."
if os.environ.get('IN_TEST',0): return # don't export if running tests
fname = Path(fname)
with open(fname, encoding='utf8') as f: code = f.read()
splits = _split(code)
rel_name = fname.absolute().resolve().relative_to(Config().path("lib_path"))
key = str(rel_name.with_suffix(''))
assert len(splits)==len(dic[key]), f'"{rel_name}" exported from notebooks should have {len(dic[key])} cells but has {len(splits)}.'
assert all([c1[0]==c2[1]] for c1,c2 in zip(splits, dic[key]))
splits = [(c2[0],c1[0],c1[1]) for c1,c2 in zip(splits, dic[key])]
nb_fnames = {Config().path("nbs_path")/s[1] for s in splits}
for nb_fname in nb_fnames:
nb = read_nb(nb_fname)
for i,f,c in splits:
c = _deal_loc_import(c, str(fname))
if f == nb_fname.name:
flags = split_flags_and_code(nb['cells'][i], str)[0]
nb['cells'][i]['source'] = flags + '\n' + c.replace('', '')
NotebookNotary().sign(nb)
nbformat.write(nb, str(nb_fname), version=4)
if not silent: print(f"Converted {rel_name}.")
Some things are written to the library with a #nbdev_comment comment prefix. e.g. The following cell:
#export
def _not_included_by_default(): pass
_all_=[_not_included_by_default]
would get written to {module}.py as:
def _not_included_by_default(): pass
#nbdev_comment _all_=[_not_included_by_default]
In this case
_all_=[_not_included_by_default] should not be part of the module#hide
dic = notebook2script(silent=True, to_dict=True)
_script2notebook(Config().path("lib_path")/'export.py', dic)
Converted export.py.
#hide
exported = get_nbdev_module().modules
files = nbglob(extension='.py', config_key='lib_path')
files = files.filter(lambda x: str(x.relative_to(Config().path("lib_path"))) in exported)
test_eq(len(files) > 0, True)
#export
@call_parse
def nbdev_update_lib(fname:Param("A python filename or glob to convert", str)=None,
silent:Param("Don't print results", bool_arg)=False):
"Propagates any change in the modules matching `fname` to the notebooks that created them"
if fname and fname.endswith('.ipynb'): raise ValueError("`nbdev_update_lib` operates on .py files. If you wish to convert notebooks instead, see `nbdev_build_lib`.")
if os.environ.get('IN_TEST',0): return
dic = notebook2script(silent=True, to_dict=True)
exported = get_nbdev_module().modules
files = nbglob(fname, extension='.py', config_key='lib_path')
files = files.filter(lambda x: str(x.relative_to(Config().path("lib_path"))) in exported)
files.map(partial(_script2notebook, dic=dic, silent=silent))
If fname is not specified, this will convert all modules and submodules in the lib_folder defined in setting.ini. Otherwise fname can be a single filename or a glob expression.
silent makes the command not print any statement.
#hide
#nbdev_update_lib()
Before making a commit, you may want to check there is no diff between the exported library and the notebooks. You may also want to make this part of your CI, so that you don't accidentally merge a PR that introduces some changes between the two. This function is there to print this diff.
#export
import subprocess
from distutils.dir_util import copy_tree
#export
@call_parse
def nbdev_diff_nbs():
"Prints the diff between an export of the library in notebooks and the actual modules"
cfg = Config()
lib_folder = cfg.path("lib_path")
with tempfile.TemporaryDirectory() as d1, tempfile.TemporaryDirectory() as d2:
copy_tree(cfg.path("lib_path"), d1)
notebook2script(silent=True)
copy_tree(cfg.path("lib_path"), d2)
shutil.rmtree(cfg.path("lib_path"))
shutil.copytree(d1, str(cfg.path("lib_path")))
for d in [d1, d2]:
if (Path(d)/'__pycache__').exists(): shutil.rmtree(Path(d)/'__pycache__')
res = subprocess.run(['diff', '-ru', d1, d2], stdout=subprocess.PIPE)
print(res.stdout.decode('utf-8'))
# nbdev_diff_nbs()
If you receive an output, you'll need to either run notebook2script() or nbdev_update_lib() to fix the difference.
#export
@call_parse
def nbdev_trust_nbs(fname:Param("A notebook name or glob to convert", str)=None,
force_all:Param("Trust even notebooks that haven't changed", bool)=False):
"Trust notebooks matching `fname`"
check_fname = Config().path("nbs_path")/".last_checked"
last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None
files = nbglob(fname)
for fn in files:
if last_checked and not force_all:
last_changed = os.path.getmtime(fn)
if last_changed < last_checked: continue
nb = read_nb(fn)
if not NotebookNotary().check_signature(nb): NotebookNotary().sign(nb)
check_fname.touch(exist_ok=True)
#hide
from nbdev.export import *
notebook2script()
Converted 00_export.ipynb. Converted 01_sync.ipynb. Converted 02_showdoc.ipynb. Converted 03_export2html.ipynb. Converted 04_test.ipynb. Converted 05_merge.ipynb. Converted 06_cli.ipynb. Converted 07_clean.ipynb. Converted 99_search.ipynb. Converted example.ipynb. Converted index.ipynb. Converted nbdev_comments.ipynb. Converted tutorial.ipynb. Converted tutorial_colab.ipynb.