Configuring nbdev and bootstrapping notebook export
#|hide
#|default_exp config
#|export
from datetime import datetime
from fastcore.imports import *
from fastcore.foundation import *
from fastcore.basics import *
from fastcore.imports import *
from fastcore.meta import *
from fastcore.script import *
from fastcore.style import *
from fastcore.xdg import *
from fastcore.xtras import *
import ast,functools
from IPython.display import Markdown
from configparser import ConfigParser
from execnb.nbio import read_nb,NbCell
from pprint import pformat,pprint
from urllib.error import HTTPError
#|hide
from fastcore.test import *
import tempfile
nbdev is heavily customizeable, thanks to the configuration system defined in this module. There are 2 ways to interact with nbdev's config:
nbdev_create_config
creates a config file (if you're starting a new project use nbdev_new
instead)get_config
returns a Config
object.Read on for more about how these work.
#|export
_nbdev_home_dir = 'nbdev' # sub-directory of xdg base dir
_nbdev_cfg_name = 'settings.ini'
#|export
def _git_repo():
try: return repo_details(run('git config --get remote.origin.url'))[1]
except OSError: return
#|hide
test_eq(_git_repo(), 'nbdev')
with tempfile.TemporaryDirectory() as d, working_directory(d): test_is(_git_repo(), None)
#|export
def _apply_defaults(
cfg,
lib_name='%(repo)s', # Package name
branch='master', # Repo default branch
git_url='https://github.com/%(user)s/%(lib_name)s', # Repo URL
custom_sidebar:bool_arg=False, # Use a custom sidebar.yml?
nbs_path='.', # Path to notebooks
lib_path:str=None, # Path to package root (default: `repo` with `-` replaced by `_`)
doc_path='_docs', # Path to rendered docs
tst_flags='notest', # Test flags
version='0.0.1', # Version of this release
doc_host='https://%(user)s.github.io', # Hostname for docs
doc_baseurl='/%(lib_name)s', # Base URL for docs
keywords='nbdev jupyter notebook python', # Package keywords
license='apache2', # License for the package
copyright:str=None, # Copyright for the package, defaults to '`current_year` onwards, `author`'
status='3', # Development status PyPI classifier
min_python='3.7', # Minimum Python version PyPI classifier
audience='Developers', # Intended audience PyPI classifier
language='English', # Language PyPI classifier
recursive:bool_arg=False, # Include subfolders in notebook globs?
black_formatting:bool_arg=False, # Format libraries with black?
readme_nb='index.ipynb', # Notebook to export as repo readme
title='%(lib_name)s', # Quarto website title
allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata
allowed_cell_metadata_keys='', # Preserve the list of keys in cell level metadata
jupyter_hooks=True, # Run Jupyter hooks?
clean_ids=True, # Remove ids from plaintext reprs?
custom_quarto_yml=False, # Use a custom _quarto.yml?
):
"Apply default settings where missing in `cfg`."
if getattr(cfg,'repo',None) is None:
cfg.repo = _git_repo()
if cfg.repo is None:
_parent = Path.cwd()
cfg.repo = _parent.parent.name if _parent.name=='nbs' else _parent.name
if lib_path is None: lib_path = cfg.repo.replace('-', '_')
if copyright is None: copyright = f"{datetime.now().year} ownwards, %(author)s"
for k,v in locals().items():
if k.startswith('_') or k == 'cfg' or cfg.get(k) is not None: continue
cfg[k] = v
return cfg
#|export
def _get_info(owner, repo, default_branch='main', default_kw='nbdev'):
from ghapi.all import GhApi
api = GhApi(owner=owner, repo=repo, token=os.getenv('GITHUB_TOKEN'))
try: r = api.repos.get()
except HTTPError:
msg= [f"""Could not access repo: {owner}/{repo} to find your default branch - `{default_branch} assumed.
Edit `settings.ini` if this is incorrect.
In the future, you can allow nbdev to see private repos by setting the environment variable GITHUB_TOKEN as described here:
https://nbdev.fast.ai/cli.html#Using-nbdev_new-with-private-repos
"""]
print(''.join(msg))
return (default_branch,default_kw,'')
return r.default_branch, default_kw if not r.topics else ' '.join(r.topics), r.description
#|hide
if os.getenv('GITHUB_ACTIONS') != 'true': # GITHUB_TOKEN in actions has limited scope.
_branch, _tags, _descrip = _get_info('fastai', 'fastai')
test_eq(_tags, 'colab deep-learning fastai gpu machine-learning notebooks python pytorch')
test_eq(_branch, 'master')
test_eq(_descrip, 'The fastai deep learning library')
#|export
def _fetch_from_git(raise_err=False):
"Get information for settings.ini from the user."
res={}
try:
url = run('git config --get remote.origin.url')
res['user'],res['repo'] = repo_details(url)
res['branch'],res['keywords'],desc = _get_info(owner=res['user'], repo=res['repo'])
if desc: res['description'] = desc
res['author'] = run('git config --get user.name').strip() # below two lines attempt to pull from global user config
res['author_email'] = run('git config --get user.email').strip()
except OSError as e:
if raise_err: raise(e)
else: res['lib_name'] = res['repo'].replace('-','_')
return res
#|hide
#test_eq(_fetch_from_git(raise_err=True)['lib_name'], 'nbdev')
#|export
def _prompt_user(cfg, inferred):
"Let user input values not in `cfg` or `inferred`."
res = cfg.copy()
for k,v in cfg.items():
inf = inferred.get(k,None)
msg = S.light_blue(k) + ' = '
if v is None:
if inf is None: res[k] = input(f'# Please enter a value for {k}\n'+msg)
else:
res[k] = inf
print(msg+res[k]+' # Automatically inferred from git')
return res
#|hide
# `repo` not printed - already exists in `cfg`
# `user` printed - in `inferred` and not `cfg`
test_stdout(lambda: _prompt_user({'user':None,'repo':'nbdev'},{'user':'fastai'}),
f"{S.light_blue('user')} = fastai # Automatically inferred from git")
#|export
def _cfg2txt(cfg, head, sections, tail=''):
"Render `cfg` with commented sections."
nm = cfg.d.name
res = f'[{nm}]\n'+head
for t,ks in sections.items():
res += f'### {t} ###\n'
for k in ks.split(): res += f"{k} = {cfg._cfg.get(nm, k, raw=True)}\n" # TODO: add `raw` to `Config.get`
res += '\n'
res += tail
return res.strip()
#|hide
d = {'user': 'fastai', 'repo': 'nbdev', 'doc_path': '_docs'}
c = Config('.', 'test_settings.ini', d, save=False)
txt = '''[DEFAULT]
# This is an nbdev settings file.
### Repo ###
user = fastai
repo = nbdev
### nbdev ###
doc_path = _docs'''
res = _cfg2txt(c, '# This is an nbdev settings file.\n\n', {'Repo': 'user repo', 'nbdev': 'doc_path'})
test_eq(res, txt)
#|export
_nbdev_cfg_head = '''# All sections below are required unless otherwise specified.
# See https://github.com/fastai/nbdev/blob/master/settings.ini for examples.
'''
_nbdev_cfg_sections = {'Python library': 'repo lib_name version min_python license',
'nbdev': 'doc_path lib_path nbs_path recursive tst_flags',
'Docs': 'branch custom_sidebar doc_host doc_baseurl git_url title',
'PyPI': 'audience author author_email copyright description keywords language status user'}
_nbdev_cfg_tail = '''### Optional ###
# requirements = fastcore pandas
# dev_requirements =
# console_scripts =
'''
#|export
@call_parse
@delegates(_apply_defaults, but='cfg', verbose=False)
def nbdev_create_config(
repo:str=None, # Repo name
user:str=None, # Repo username
author:str=None, # Package author's name
author_email:str=None, # Package author's email address
description:str=None, # Short summary of the package
path:str='.', # Path to create config file
cfg_name:str=_nbdev_cfg_name, # Name of config file to create
**kwargs
):
"Create a config file."
req = {k:v for k,v in locals().items() if k not in ('path','cfg_name','kwargs')}
inf = _fetch_from_git()
d = _prompt_user(req, inf)
cfg = Config(path, cfg_name, d, save=False)
if cfg.config_file.exists(): warn(f'Config file already exists: {cfg.config_file} and will be used as a base')
cfg = _apply_defaults(cfg, **kwargs)
txt = _cfg2txt(cfg, _nbdev_cfg_head, _nbdev_cfg_sections, _nbdev_cfg_tail)
cfg.config_file.write_text(txt)
cfg_fn = Path(path)/cfg_name
print(f'{cfg_fn} created.')
The table above also serves as a full reference of nbdev's settings (excluding the path
and cfg_name
parameters which decide where the config file is saved). For more about PyPI classifiers, see Classifiers.
#|hide
# NOTE: temporarily disabled tests until we make them more robust
#|notest
#|hide
import nbdev; nbdev.nbdev_export() # Ensure we have the latest command below
#|notest
#|hide
# Ensure we're in an empty tempdir for testing
try: cwd
except NameError: cwd = Path.cwd() # only set once
try: rmtree(tmpdir)
except (NameError, FileNotFoundError): pass
tmpdir = Path(tempfile.mkdtemp())
p = tmpdir/'my-project'
p.mkdir()
os.chdir(p)
You can create a config file by passing all of the required settings via the command line (as well as any optional settings you'd like to override):
#|notest
!nbdev_create_config --repo my-project --user fastai --author fastai --author_email info@fast.ai --description 'A test project'
settings.ini created.
Here's what the first few lines look like:
#|notest
!head -n9 settings.ini
[DEFAULT] # All sections below are required unless otherwise specified. # See https://github.com/fastai/nbdev/blob/master/settings.ini for examples. ### Python library ### repo = my-project lib_name = %(repo)s version = 0.0.1 min_python = 3.7
If you don't provide required settings from the command line, we'll try to to infer them from git and GitHub:
#|notest
#|hide
!rm settings.ini 2>/dev/null
# Initialise a minimal git repo to demonstrate
!git init -q
!git remote add origin git@github.com:fastai/nbdev.git 2> /dev/null
!git config --local user.name fastai
!git config --local user.email info@fast.ai
#|notest
!nbdev_create_config
repo = nbdev # Automatically inferred from git user = fastai # Automatically inferred from git author = fastai # Automatically inferred from git author_email = info@fast.ai # Automatically inferred from git description = Create delightful python projects using Jupyter Notebooks # Automatically inferred from git settings.ini created.
Finally, you'll be asked to manually input any required settings that we couldn't automatically fill in.
#|notest
#|hide
from shutil import rmtree
#|notest
#|hide
os.chdir(cwd) # Go back to original working dir
rmtree(tmpdir)
#|export
def _nbdev_config_file(cfg_name=_nbdev_cfg_name, path=None):
cfg_path = path = Path.cwd() if path is None else Path(path)
while cfg_path != cfg_path.parent and not (cfg_path/cfg_name).exists(): cfg_path = cfg_path.parent
if not (cfg_path/cfg_name).exists(): cfg_path = path
return cfg_path/cfg_name
#|hide
test_eq(_nbdev_config_file(), Path.cwd().parent/'settings.ini')
#|export
def _xdg_config_paths(cfg_name=_nbdev_cfg_name):
xdg_config_paths = reversed([xdg_config_home()]+xdg_config_dirs())
return [o/_nbdev_home_dir/cfg_name for o in xdg_config_paths]
#|export
@functools.lru_cache(maxsize=None)
def get_config(cfg_name=_nbdev_cfg_name, path=None):
"""Return nbdev config.\n\nSearches up from `path` until `cfg_name` is found. User settings are loaded from
`~/.config/nbdev/{cfg_name}`. Unspecified optional settings return defaults."""
cfg_file = _nbdev_config_file(cfg_name, path)
extra_files = _xdg_config_paths(cfg_name)
cfg = Config(cfg_file.parent, cfg_file.name, extra_files=extra_files)
return _apply_defaults(cfg)
See nbdev_create_config
for a full reference of nbdev's settings.
#|hide
try: _get_config,get_config = get_config,get_config.__wrapped__ # Bypass cache during dev and tests
except AttributeError: pass
Let's test it out in nbdev's own repo:
cfg = get_config()
cfg
is a fastcore Config
object, so you can access keys as attributes:
get_config
searches for repo settings.ini
in the current directory, and then in all parent directories, stopping when it is found. Default values for optional settings are applied to the resulting Config
, see nbdev_create_config
for a full reference of nbdev's settings.
p = Path.cwd().parent
test_eq(cfg.lib_name, 'nbdev')
test_eq(cfg.git_url, 'https://github.com/fastai/nbdev')
Its own path and parent are attributes too:
test_eq(cfg.config_path, p)
test_eq(cfg.config_file, p/'settings.ini')
For convenience, use path
to resolve settings that specify project-relative paths:
test_eq(cfg.path('doc_path'), p/'_docs')
test_eq(cfg.path('lib_path'), p/'nbdev')
test_eq(cfg.path('nbs_path'), p/'nbs')
It automatically returns defaults for keys not specified in the config file. Let's create an empty config file to demonstrate:
#|hide
!rm ../tests/test_settings.ini 2>/dev/null
Config('../tests/', 'test_settings.ini', {'repo': 'my-project', 'author': 'fastai'});
%%sh
cat ../tests/test_settings.ini
[DEFAULT] repo = my-project author = fastai
cfg = get_config('test_settings.ini', '../tests/')
test_eq(cfg.repo, 'my-project')
test_eq(cfg.lib_path, 'my_project')
test_eq(cfg.copyright, '2022 ownwards, fastai')
In fact, you can return a default config even if you don't have a settings file. This is to support certain nbdev commands work outside of nbdev repos:
%%sh
rm ../tests/test_settings.ini
cfg = get_config('test_settings.ini', '../tests')
test_eq(cfg.lib_path, 'nbdev')
test_eq(cfg.nbs_path, '.')
You can customise nbdev for all repositories for your user with a ~/.config/nbdev/settings.ini file (by default, although we follow the broader XDG specification).
For example, you could globally disable nbdev's Jupyter hooks by creating the following user settings file (we override XDG_CONFIG_DIR
to demonstrate):
#|hide
from contextlib import contextmanager
#|hide
@contextmanager
def env(k, v):
orig = os.environ.get(k,None)
try:
os.environ[k] = str(v)
yield
finally:
if orig is None: del os.environ[k]
else: os.environ[k] = orig
Config('../tests/user_settings', 'test_settings.ini', {'jupyter_hooks': False})
with env('XDG_CONFIG_DIR', Path('../tests/user_settings').resolve()):
cfg = get_config('test_settings.ini', '../tests')
test_eq(str2bool(cfg.jupyter_hooks), True)
#|hide
!rm -r ../tests/user_settings 2>/dev/null
#|export
def config_key(c, default=None, path=True, missing_ok=None):
"Deprecated: use `get_config().get` or `get_config().path` instead."
warn("`config_key` is deprecated. Use `get_config().get` or `get_config().path` instead.", DeprecationWarning)
return get_config().path(c, default) if path else get_config().get(c, default)
#|export
def create_output(txt, mime):
"Add a cell output containing `txt` of the `mime` text MIME sub-type"
return [{"data": { f"text/{mime}": str(txt).splitlines(True) },
"execution_count": 1, "metadata": {}, "output_type": "execute_result"}]
#|export
def show_src(src, lang='python'): return Markdown(f'```{lang}\n{src}\n```')
show_src("print(create_output('text', 'text/plain'))")
print(create_output('text', 'text/plain'))
#|export
_re_version = re.compile('^__version__\s*=.*$', re.MULTILINE)
_init = '__init__.py'
def update_version(path=None):
"Add or update `__version__` in the main `__init__.py` of the library."
path = Path(path or get_config().path("lib_path"))
fname = path/_init
if not fname.exists(): fname.touch()
version = f'__version__ = "{get_config().version}"'
code = fname.read_text()
if _re_version.search(code) is None: code = version + "\n" + code
else: code = _re_version.sub(version, code)
fname.write_text(code)
def _has_py(fs): return any(1 for f in fs if f.endswith('.py'))
def add_init(path=None):
"Add `__init__.py` in all subdirs of `path` containing python files if it's not there already."
# we add the lowest-level `__init__.py` files first, which ensures _has_py succeeds for parent modules
path = Path(path or get_config().path("lib_path"))
path.mkdir(exist_ok=True)
if not (path/_init).exists(): (path/_init).touch()
for r,ds,fs in os.walk(path, topdown=False):
r = Path(r)
subds = (os.listdir(r/d) for d in ds)
if _has_py(fs) or any(filter(_has_py, subds)) and not (r/_init).exists(): (r/_init).touch()
update_version(path)
Python modules require a __init.py__
file in all directories that are modules. We assume that all directories containing a python file (including in subdirectories of any depth) is a module, and therefore add a __init__.py
to each.
with tempfile.TemporaryDirectory() as d:
d = Path(d)
(d/'a/b').mkdir(parents=True)
(d/'a/b/f.py').touch()
(d/'a/c').mkdir()
add_init(d)
assert not (d/'a/c'/_init).exists(), "Should not add init to dir without py file"
for e in [d, d/'a', d/'a/b']: assert (e/_init).exists(),f"Missing init in {e}"
#|export
def write_cells(cells, hdr, file, offset=0):
"Write `cells` to `file` along with header `hdr` starting at index `offset` (mainly for nbdev internal use)."
for cell in cells:
if cell.source.strip(): file.write(f'\n\n{hdr} {cell.idx_+offset}\n{cell.source}')
#|export
def basic_export_nb(fname, name, dest=None):
"Basic exporter to bootstrap nbdev."
if dest is None: dest = get_config().path('lib_path')
add_init()
fname,dest = Path(fname),Path(dest)
nb = read_nb(fname)
# grab the source from all the cells that have an `export` comment
cells = L(cell for cell in nb.cells if re.match(r'#\s*\|export', cell.source))
# find all the exported functions, to create `__all__`:
trees = cells.map(NbCell.parsed_).concat()
funcs = trees.filter(risinstance((ast.FunctionDef,ast.ClassDef))).attrgot('name')
exp_funcs = [f for f in funcs if f[0]!='_']
# write out the file
with (dest/name).open('w') as f:
f.write(f"# %% auto 0\n__all__ = {exp_funcs}")
write_cells(cells, f"# %% {fname.relpath(dest)}", f)
f.write('\n')
This is a simple exporter with just enough functionality to correctly export this notebook, in order to bootstrap the creation of nbdev itself.
#|hide
#| eval: false
path = Path('../nbdev')
(path/'config.py').unlink(missing_ok=True)
basic_export_nb("01_config.ipynb", 'config.py')
g = exec_new('from nbdev import config')
assert g['config'].add_init
assert 'add_init' in g['config'].__all__