#|default_exp docments
Document parameters using comments.
#|export
from __future__ import annotations
import re
from tokenize import tokenize,COMMENT
from ast import parse,FunctionDef,AsyncFunctionDef,AnnAssign
from io import BytesIO
from textwrap import dedent
from types import SimpleNamespace
from inspect import getsource,isfunction,ismethod,isclass,signature,Parameter
from dataclasses import dataclass, is_dataclass
from fastcore.utils import *
from fastcore.meta import delegates
from fastcore import docscrape
from inspect import isclass,getdoc
#|hide
from nbdev.showdoc import *
from fastcore.test import *
docments provides programmatic access to comments in function parameters and return types. It can be used to create more developer-friendly documentation, CLI, etc tools.
Without docments, if you want to document your parameters, you have to repeat param names in docstrings, since they're already in the function signature. The parameters have to be kept synchronized in the two places as you change your code. Readers of your code have to look back and forth between two places to understand what's happening. So it's more work for you, and for your users.
Furthermore, to have parameter documentation formatted nicely without docments, you have to use special magic docstring formatting, often with odd quirks, which is a pain to create and maintain, and awkward to read in code. For instance, using numpy-style documentation:
def add_np(a:int, b:int=0)->int:
"""The sum of two numbers.
Used to demonstrate numpy-style docstrings.
Parameters
----------
a : int
the 1st number to add
b : int
the 2nd number to add (default: 0)
Returns
-------
int
the result of adding `a` to `b`"""
return a+b
By comparison, here's the same thing using docments:
def add(
a:int, # the 1st number to add
b=0, # the 2nd number to add
)->int: # the result of adding `a` to `b`
"The sum of two numbers."
return a+b
docments also supports numpy-style docstrings, or a mix or numpy-style and docments parameter documentation. The functions in this section help get and parse this information.
#|export
def docstring(sym):
"Get docstring for `sym` for functions ad classes"
if isinstance(sym, str): return sym
res = getdoc(sym)
if not res and isclass(sym): res = getdoc(sym.__init__)
return res or ""
test_eq(docstring(add), "The sum of two numbers.")
#|export
def parse_docstring(sym):
"Parse a numpy-style docstring in `sym`"
docs = docstring(sym)
return AttrDict(**docscrape.NumpyDocString(docstring(sym)))
# parse_docstring(add_np)
#|export
def isdataclass(s):
"Check if `s` is a dataclass but not a dataclass' instance"
return is_dataclass(s) and isclass(s)
#|export
def get_dataclass_source(s):
"Get source code for dataclass `s`"
return getsource(s) if not getattr(s, "__module__") == '__main__' else ""
#|export
def get_source(s):
"Get source code for string, function object or dataclass `s`"
return getsource(s) if isfunction(s) or ismethod(s) else get_dataclass_source(s) if isdataclass(s) else s
#|export
def _parses(s):
"Parse Python code in string, function object or dataclass `s`"
return parse(dedent(get_source(s)))
def _tokens(s):
"Tokenize Python code in string or function object `s`"
s = get_source(s)
return tokenize(BytesIO(s.encode('utf-8')).readline)
_clean_re = re.compile(r'^\s*#(.*)\s*$')
def _clean_comment(s):
res = _clean_re.findall(s)
return res[0] if res else None
def _param_locs(s, returns=True):
"`dict` of parameter line numbers to names"
body = _parses(s).body
if len(body)==1: #or not isinstance(body[0], FunctionDef): return None
defn = body[0]
if isinstance(defn, (FunctionDef, AsyncFunctionDef)):
res = {arg.lineno:arg.arg for arg in defn.args.args}
if returns and defn.returns: res[defn.returns.lineno] = 'return'
return res
elif isdataclass(s):
res = {arg.lineno:arg.target.id for arg in defn.body if isinstance(arg, AnnAssign)}
return res
return None
#|export
empty = Parameter.empty
#|export
def _get_comment(line, arg, comments, parms):
if line in comments: return comments[line].strip()
line -= 1
res = []
while line and line in comments and line not in parms:
res.append(comments[line])
line -= 1
return dedent('\n'.join(reversed(res))) if res else None
def _get_full(anno, name, default, docs):
if anno==empty and default!=empty: anno = type(default)
return AttrDict(docment=docs.get(name), anno=anno, default=default)
#|export
def _merge_doc(dm, npdoc):
if not npdoc: return dm
if not dm.anno or dm.anno==empty: dm.anno = npdoc.type
if not dm.docment: dm.docment = '\n'.join(npdoc.desc)
return dm
def _merge_docs(dms, npdocs):
npparams = npdocs['Parameters']
params = {nm:_merge_doc(dm,npparams.get(nm,None)) for nm,dm in dms.items()}
if 'return' in dms: params['return'] = _merge_doc(dms['return'], npdocs['Returns'])
return params
#|export
def _get_property_name(p):
"Get the name of property `p`"
if hasattr(p, 'fget'):
return p.fget.func.__qualname__ if hasattr(p.fget, 'func') else p.fget.__qualname__
else: return next(iter(re.findall(r'\'(.*)\'', str(p)))).split('.')[-1]
#|export
def get_name(obj):
"Get the name of `obj`"
if hasattr(obj, '__name__'): return obj.__name__
elif getattr(obj, '_name', False): return obj._name
elif hasattr(obj,'__origin__'): return str(obj.__origin__).split('.')[-1] #for types
elif type(obj)==property: return _get_property_name(obj)
else: return str(obj).split('.')[-1]
test_eq(get_name(in_ipython), 'in_ipython')
test_eq(get_name(L.map), 'map')
#|export
def qual_name(obj):
"Get the qualified name of `obj`"
if hasattr(obj,'__qualname__'): return obj.__qualname__
if ismethod(obj): return f"{get_name(obj.__self__)}.{get_name(fn)}"
return get_name(obj)
assert qual_name(docscrape) == 'fastcore.docscrape'
#|export
def _docments(s, returns=True, eval_str=False):
"`dict` of parameter names to 'docment-style' comments in function or string `s`"
nps = parse_docstring(s)
if isclass(s) and not is_dataclass(s): s = s.__init__ # Constructor for a class
comments = {o.start[0]:_clean_comment(o.string) for o in _tokens(s) if o.type==COMMENT}
parms = _param_locs(s, returns=returns) or {}
docs = {arg:_get_comment(line, arg, comments, parms) for line,arg in parms.items()}
if isinstance(s,str): s = eval(s)
sig = signature(s)
res = {arg:_get_full(p.annotation, p.name, p.default, docs) for arg,p in sig.parameters.items()}
if returns: res['return'] = _get_full(sig.return_annotation, 'return', empty, docs)
res = _merge_docs(res, nps)
if eval_str:
hints = type_hints(s)
for k,v in res.items():
if k in hints: v['anno'] = hints.get(k)
return res
#|export
@delegates(_docments)
def docments(elt, full=False, **kwargs):
"Generates a `docment`"
r = {}
params = set(signature(elt).parameters)
params.add('return')
def _update_docments(f, r):
if hasattr(f, '__delwrap__'): _update_docments(f.__delwrap__, r)
r.update({k:v for k,v in _docments(f, **kwargs).items() if k in params
and (v.get('docment', None) or not nested_idx(r, k, 'docment'))})
_update_docments(elt, r)
if not full: r = {k:v['docment'] for k,v in r.items()}
return AttrDict(r)
The returned dict has parameter names as keys, docments as values. The return value comment appears in the return, unless returns=False. Using the add definition above, we get:
def add(
a:int, # the 1st number to add
b=0, # the 2nd number to add
)->int: # the result of adding `a` to `b`
"The sum of two numbers."
return a+b
docments(add)
{ 'a': 'the 1st number to add',
'b': 'the 2nd number to add',
'return': 'the result of adding `a` to `b`'}
If you pass full=True, the values are dict of defaults, types, and docments as values. Note that the type annotation is inferred from the default value, if the annotation is empty and a default is supplied.
docments(add, full=True)
{ 'a': { 'anno': 'int',
'default': <class 'inspect._empty'>,
'docment': 'the 1st number to add'},
'b': { 'anno': <class 'int'>,
'default': 0,
'docment': 'the 2nd number to add'},
'return': { 'anno': 'int',
'default': <class 'inspect._empty'>,
'docment': 'the result of adding `a` to `b`'}}
To evaluate stringified annotations (from python 3.10), use eval_str:
docments(add, full=True, eval_str=True)['a']
{ 'anno': <class 'int'>,
'default': <class 'inspect._empty'>,
'docment': 'the 1st number to add'}
If you need more space to document a parameter, place one or more lines of comments above the parameter, or above the return type. You can mix-and-match these docment styles:
def add(
# The first operand
a:int,
# This is the second of the operands to the *addition* operator.
# Note that passing a negative value here is the equivalent of the *subtraction* operator.
b:int,
)->int: # The result is calculated using Python's builtin `+` operator.
"Add `a` to `b`"
return a+b
docments(add)
{ 'a': 'The first operand',
'b': 'This is the second of the operands to the *addition* operator.\n'
'Note that passing a negative value here is the equivalent of the '
'*subtraction* operator.',
'return': "The result is calculated using Python's builtin `+` operator."}
Docments works with async functions, too:
async def add_async(
# The first operand
a:int,
# This is the second of the operands to the *addition* operator.
# Note that passing a negative value here is the equivalent of the *subtraction* operator.
b:int,
)->int: # The result is calculated using Python's builtin `+` operator.
"Add `a` to `b`"
return a+b
test_eq(docments(add_async), docments(add))
You can also use docments with classes and methods:
class Adder:
"An addition calculator"
def __init__(self,
a:int, # First operand
b:int, # 2nd operand
): self.a,self.b = a,b
def calculate(self
)->int: # Integral result of addition operator
"Add `a` to `b`"
return a+b
docments(Adder)
{'a': 'First operand', 'b': '2nd operand', 'return': None}
docments(Adder.calculate)
{'return': 'Integral result of addition operator', 'self': None}
docments can also be extracted from numpy-style docstrings:
print(add_np.__doc__)
The sum of two numbers.
Used to demonstrate numpy-style docstrings.
Parameters
----------
a : int
the 1st number to add
b : int
the 2nd number to add (default: 0)
Returns
-------
int
the result of adding `a` to `b`
docments(add_np)
{ 'a': 'the 1st number to add',
'b': 'the 2nd number to add (default: 0)',
'return': 'the result of adding `a` to `b`'}
You can even mix and match docments and numpy parameters:
def add_mixed(a:int, # the first number to add
b
)->int: # the result
"""The sum of two numbers.
Parameters
----------
b : int
the 2nd number to add (default: 0)"""
return a+b
docments(add_mixed, full=True)
{ 'a': { 'anno': 'int',
'default': <class 'inspect._empty'>,
'docment': 'the first number to add'},
'b': { 'anno': 'int',
'default': <class 'inspect._empty'>,
'docment': 'the 2nd number to add (default: 0)'},
'return': { 'anno': 'int',
'default': <class 'inspect._empty'>,
'docment': 'the result'}}
You can use docments with dataclasses, however if the class was defined in online notebook, docments will not contain parameters' comments. This is because the source code is not available in the notebook. After converting the notebook to a module, the docments will be available. Thus, documentation will have correct parameters' comments.
#|hide
class _F:
@classmethod
def class_method(cls,
foo:str, # docment for parameter foo
):...
test_eq(docments(_F.class_method), {'foo': 'docment for parameter foo', 'return': None})
Docments even works with delegates:
from fastcore.meta import delegates
def _a(a:int=2): return a # First
@delegates(_a)
def _b(b:str, **kwargs): return b, (_a(**kwargs)) # Second
docments(_b)
{'a': 'First', 'b': 'Second', 'return': None}
#|hide
def _c(b:str, # Second
a:int=2): return b, a # Third
@delegates(_c)
def _d(c:int, # First
b:str, **kwargs
)->int:
return c, _c(b, **kwargs)
#|hide
test_eq(docments(_c, full=True)['b']['docment'],'Second')
test_eq(docments(_d, full=True)['b']['docment'],'Second')
_argset = {'a', 'b', 'c', 'return'}
test_eq(docments(_d, full=True).keys() & _argset, _argset) # _d has the args a,b,c and return
#|hide
import nbdev; nbdev.nbdev_export()