#hide
#default_exp export2html
#default_cls_lvl 3
from nbdev.showdoc import show_doc

#export
from nbdev.imports import *
from nbdev.sync import *
from nbdev.export import *
from nbdev.export import _mk_flag_re
from nbdev.showdoc import *
from nbdev.template import *
from fastcore.foundation import *

from html.parser import HTMLParser
from nbconvert.preprocessors import ExecutePreprocessor, Preprocessor
from nbconvert import HTMLExporter,MarkdownExporter
import traitlets

#export
class HTMLParseAttrs(HTMLParser):
    "Simple HTML parser which stores any attributes in `attrs` dict"
    def handle_starttag(self, tag, attrs): self.tag,self.attrs = tag,dict(attrs)

    def attrs2str(self):
        "Attrs as string"
        return ' '.join([f'{k}="{v}"' for k,v in self.attrs.items()])

    def show(self):
        "Tag with updated attrs"
        return f'<{self.tag} {self.attrs2str()} />'

    def __call__(self, s):
        "Parse `s` and store attrs"
        self.feed(s)
        return self.attrs

h = HTMLParseAttrs()
t = h('<img src="src" alt="alt" width="700" caption="cap" />')
test_eq(t['width'], '700')
test_eq(t['src'  ], 'src')

t['width'] = '600'
test_eq(h.show(), '<img src="src" alt="alt" width="600" caption="cap" />')

t['max-width'] = t.pop('width')
test_eq(h.show(), '<img src="src" alt="alt" caption="cap" max-width="600" />')

#export
def remove_widget_state(cell):
    "Remove widgets in the output of `cells`"
    if cell['cell_type'] == 'code' and 'outputs' in cell:
        cell['outputs'] = [l for l in cell['outputs']
                           if not ('data' in l and 'application/vnd.jupyter.widget-view+json' in l.data)]
    return cell

#export
# Note: `_re_show_doc` will catch show_doc even if it's commented out etc
_re_show_doc = re.compile(r"""
# Catches any show_doc and get the first argument in group 1
^\s*show_doc # line can start with any amount of whitespace followed by show_doc
\s*\(\s*     # Any number of whitespace, opening (, any number of whitespace
([^,\)\s]*)  # Catching group for any character but a comma, a closing ) or a whitespace
[,\)\s]      # A comma, a closing ) or a whitespace
""", re.MULTILINE | re.VERBOSE)

_re_hide_input = [
    _mk_flag_re('export', (0,1), "Cell that has `#export"),
    _mk_flag_re('(hide_input|hide-input)', 0, "Cell that has `#hide_input` or `#hide-input`")]

_re_hide_output = _mk_flag_re('(hide_output|hide-output)', 0, "Cell that has `#hide_output` or `#hide-output`")

#export
def upd_metadata(cell, key, value=True):
    "Sets `key` to `value` on the `metadata` of `cell` without replacing metadata"
    cell.setdefault('metadata',{})[key] = value

#export
def hide_cells(cell):
    "Hide inputs of `cell` that need to be hidden"
    if check_re_multi(cell, [_re_show_doc, *_re_hide_input]): upd_metadata(cell, 'hide_input')
    elif check_re(cell, _re_hide_output): upd_metadata(cell, 'hide_output')
    return cell

for source in ['show_doc(read_nb)', '# export\nfrom local.core import *', '# hide_input\n2+2',
              'line1\n show_doc (read_nb) \nline3', '# export with.mod\nfrom local.core import *']:
    cell = {'cell_type': 'code', 'source': source}
    cell1 = hide_cells(cell.copy())
    assert 'metadata' in cell1
    assert 'hide_input' in cell1['metadata']
    assert cell1['metadata']['hide_input']

flag = '# exports'
cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core2 import *'}
test_eq(hide_cells(cell.copy()), cell)

for source in ['# hide-output\nfrom local.core import *', '# hide_output\n2+2']:
    cell = {'cell_type': 'code', 'source': source}
    cell1 = hide_cells(cell.copy())
    assert 'metadata' in cell1
    assert 'hide_output' in cell1['metadata']
    assert cell1['metadata']['hide_output']

cell = {'cell_type': 'code', 'source': '# hide-outputs\nfrom local.core import *'}
test_eq(hide_cells(cell.copy()), cell)

#export
def clean_exports(cell):
    "Remove all flags from code `cell`s"
    if cell['cell_type'] == 'code': cell['source'] = split_flags_and_code(cell, str)[1]
    return cell

flag = '# exports'
cell = {'cell_type': 'code', 'source': f'{flag}\nfrom local.core import *'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
cell['cell_type'] = 'markdown'
test_eq(clean_exports(cell.copy()), cell)
cell = {'cell_type': 'code', 'source': f'{flag} core\nfrom local.core import *'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': 'from local.core import *'})
    
cell = {'cell_type': 'code', 'source': f'# comment \n# exports\nprint("something")'}
test_eq(clean_exports(cell.copy()), {'cell_type': 'code', 'source': '# exports\nprint("something")'})

#export
def treat_backticks(cell):
    "Add links to backticks words in `cell`"
    if cell['cell_type'] == 'markdown': cell['source'] = add_doc_links(cell['source'])
    return cell

cell = {'cell_type': 'markdown', 'source': 'This is a `DocsTestClass`'}
test_eq(treat_backticks(cell), {'cell_type': 'markdown',
    'source': 'This is a [`DocsTestClass`](/export.html#DocsTestClass)'})

#export
_re_nb_link = re.compile(r"""
# Catches any link to a local notebook and keeps the title in group 1, the link without .ipynb in group 2
\[          # Opening [
([^\]]*)    # Catching group for any character except ]
\]\(        # Closing ], opening (
([^http]    # Catching group that must not begin by html (local notebook)
[^\)]*)     # and containing anything but )
.ipynb\)    # .ipynb and closing )
""", re.VERBOSE)

#export
_re_block_notes = re.compile(r"""
# Catches any pattern > Title: content with title in group 1 and content in group 2
^\s*>\s*     # > followed by any number of whitespace
([^:]*)      # Catching group for any character but :
:\s*         # : then any number of whitespace
([^\n]*)     # Catching group for anything but a new line character
(?:\n|$)     # Non-catching group for either a new line or the end of the text
""", re.VERBOSE | re.MULTILINE)

#export
def _to_html(text):
    return text.replace("'", "&#8217;")

#export
def add_jekyll_notes(cell):
    "Convert block quotes to jekyll notes in `cell`"
    styles = Config().get('jekyll_styles', 'note,warning,tip,important').split(',')
    def _inner(m):
        title,text = m.groups()
        if title.lower() not in styles: return f"> {title}:{text}"
        return '{% include '+title.lower()+".html content=\'"+_to_html(text)+"\' %}"
    if cell['cell_type'] == 'markdown':
        cell['source'] = _re_block_notes.sub(_inner, cell['source'])
    return cell

#hide
for w in ['Warning', 'Note', 'Important', 'Tip', 'Bla']:
    cell = {'cell_type': 'markdown', 'source': f"> {w}: This is my final {w.lower()}!"}
    res = '{% include '+w.lower()+'.html content=\'This is my final '+w.lower()+'!\' %}'
    if w != 'Bla': test_eq(add_jekyll_notes(cell), {'cell_type': 'markdown', 'source': res})
    else: test_eq(add_jekyll_notes(cell), cell)

#hide
cell = {'cell_type': 'markdown', 'source': f"> This is a link, don't break me! https://my.link.com"}
test_eq(add_jekyll_notes(cell.copy()), cell)

#export
_re_image = re.compile(r"""
# Catches any image file used, either with `![alt](image_file)` or `<img src="image_file">`
^(!\[           #   Beginning of line (since re.MULTILINE is passed) followed by ![ in a catching group
[^\]]*          #   Anything but ]
\]\()           #   Closing ] and opening (, end of the first catching group
[ \t]*          #   Whitespace before the image path
([^\) \t]*)     #   Catching block with any character that is not ) or whitespace
(\)| |\t)       #   Catching group with closing ) or whitespace
|               # OR
^(<img\ [^>]*>) #   Catching group with <img some_html_code>
""", re.MULTILINE | re.VERBOSE)

m=_re_image.search('![Alt](images/logo.png)')
test_eq(m.groups(), ('![Alt](', 'images/logo.png', ')', None))
# using ) or whitespace to close the group means we don't need a special case for captions
m=_re_image.search('![Alt](images/logo.png "caption (something)")')
test_eq(m.groups(), ('![Alt](', 'images/logo.png', '', None))

#export
def _img2jkl(d, h, jekyll=True):
    if not jekyll: return '<img ' + h.attrs2str() + '>'
    if 'width' in d: d['max-width'] = d.pop('width')
    if 'src' in d:   d['file'] = d.pop('src')
    return '{% include image.html ' + h.attrs2str() + ' %}'

#export
def _is_real_image(src):
    return not (src.startswith('http://') or src.startswith('https://') or src.startswith('data:image/'))

#export
def copy_images(cell, fname, dest, jekyll=True):
    "Copy images referenced in `cell` from `fname` parent folder to `dest` folder"
    def _rep_src(m):
        grps = m.groups()
        if grps[3] is not None:
            h = HTMLParseAttrs()
            dic = h(grps[3])
            src = dic['src']
        else: src = grps[1]
        if _is_real_image(src):
            os.makedirs((Path(dest)/src).parent, exist_ok=True)
            shutil.copy(Path(fname).parent/src, Path(dest)/src)
            src = Config().doc_baseurl + src
        if grps[3] is not None:
            dic['src'] = src
            return _img2jkl(dic, h, jekyll=jekyll)
        else: return f"{grps[0]}{src}{grps[2]}"
    if cell['cell_type'] == 'markdown': cell['source'] = _re_image.sub(_rep_src, cell['source'])
    return cell

dest_img = Config().doc_path/'images'/'logo.png'
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.png)'}
try:
    copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
    test_eq(cell["source"], 'Text\n![Alt](/images/logo.png)')
    #Image has been copied
    assert dest_img.exists()
    cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](images/logo.png "caption (something)")'}
    copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
    test_eq(cell["source"], 'Text\n![Alt](/images/logo.png "caption (something)")')
finally: dest_img.unlink()

#hide
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](https://site.logo.png)'}
copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
test_eq(cell["source"], 'Text\n![Alt](https://site.logo.png)')
cell = {'cell_type': 'markdown', 'source':'Text\n![Alt](https://site.logo.png "caption")'}
copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
test_eq(cell["source"], 'Text\n![Alt](https://site.logo.png "caption")')

#hide
cell = {'cell_type': 'markdown', 'source': 'Text\n<img src="images/logo.png" alt="alt" width="600" caption="cap" />'}
try:
    copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
    test_eq(cell["source"], 'Text\n{% include image.html alt="alt" caption="cap" max-width="600" file="/images/logo.png" %}')
    assert dest_img.exists()
finally: dest_img.unlink()

#hide
cell = {'cell_type': 'markdown', 'source': 'Text\n<img src="http://site.logo.png" alt="alt" width="600" caption="cap" />'}
copy_images(cell, Path('01_export.ipynb'), Config().doc_path)
test_eq(cell["source"], 'Text\n{% include image.html alt="alt" caption="cap" max-width="600" file="http://site.logo.png" %}')

#export
def _relative_to(path1, path2):
    p1,p2 = Path(path1).absolute().parts,Path(path2).absolute().parts
    i=0
    while i <len(p1) and i<len(p2) and p1[i] == p2[i]: i+=1
    p1,p2 = p1[i:],p2[i:]
    return os.path.sep.join(['..' for _ in p2] + list(p1))

#hide
test_eq(_relative_to(Path('images/logo.png'), Config().doc_path), '../nbs/images/logo.png')
test_eq(_relative_to(Path('images/logo.png'), Config().doc_path.parent), 'nbs/images/logo.png')

#export
def adapt_img_path(cell, fname, dest, jekyll=True):
    "Adapt path of images referenced in `cell` from `fname` to work in folder `dest`"
    def _rep(m):
        gps = m.groups()
        if gps[0] is not None:
            start,img,end = gps[:3]
            if not (img.startswith('http:/') or img.startswith('https:/')):
                img = _relative_to(fname.parent/img, dest)
            return f'{start}{img}{end}'
        else:
            h = HTMLParseAttrs()
            dic = h(gps[3])
            if not (dic['src'].startswith('http:/') or dic['src'].startswith('https:/')):
                dic['src'] = _relative_to(fname.parent/dic['src'], dest)
            return _img2jkl(dic, h, jekyll=jekyll)
    if cell['cell_type'] == 'markdown': cell['source'] = _re_image.sub(_rep, cell['source'])
    return cell

cell = {'cell_type': 'markdown', 'source': 'Text\n![Alt](images/logo.png)'}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], 'Text\n![Alt](nbs/images/logo.png)')

cell = {'cell_type': 'markdown', 'source': 'Text\n![Alt](http://site.logo.png)'}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], 'Text\n![Alt](http://site.logo.png)')

cell = {'cell_type': 'markdown', 
        'source': 'Text\n<img alt="Logo" src="images/logo.png" width="600"/>'}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], 'Text\n{% include image.html alt="Logo" max-width="600" file="nbs/images/logo.png" %}')

cell = {'cell_type': 'markdown', 
        'source': 'Text\n<img alt="Logo" src="https://site.image.png" width="600"/>'}
cell1 = adapt_img_path(cell, Path('01_export.ipynb'), Path('.').absolute().parent)
test_eq(cell1['source'], 'Text\n{% include image.html alt="Logo" max-width="600" file="https://site.image.png" %}')

#export
_re_latex = re.compile(r'^(\$\$.*\$\$)$', re.MULTILINE)

#export
def escape_latex(cell):
    if cell['cell_type'] != 'markdown': return cell
    cell['source'] = _re_latex.sub(r'{% raw %}\n\1\n{% endraw %}', cell['source'])
    return cell

cell = {'cell_type': 'markdown', 
        'source': 'lala\n$$equation$$\nlala'}
cell = escape_latex(cell)
test_eq(cell['source'], 'lala\n{% raw %}\n$$equation$$\n{% endraw %}\nlala')

#export
_re_cell_to_collapse_closed = _mk_flag_re('(collapse|collapse_hide|collapse-hide)', 0, "Cell with #collapse or #collapse_hide")
_re_cell_to_collapse_open = _mk_flag_re('(collapse_show|collapse-show)', 0, "Cell with #collapse_show")
_re_cell_to_collapse_output = _mk_flag_re('(collapse_output|collapse-output)', 0, "Cell with #collapse_output")

#export
def collapse_cells(cell):
    "Add a collapse button to inputs or outputs of `cell` in either the open or closed position"
    if check_re(cell, _re_cell_to_collapse_closed): upd_metadata(cell,'collapse_hide')
    elif check_re(cell, _re_cell_to_collapse_open): upd_metadata(cell,'collapse_show')
    elif check_re(cell, _re_cell_to_collapse_output): upd_metadata(cell,'collapse_output')
    return cell

#hide
for flag in [
    ('collapse_hide', '#collapse'),
    ('collapse_hide', '# collapse_hide'),
    ('collapse_hide', ' # collapse-hide'),
    ('collapse_show', '#collapse_show'),
    ('collapse_show', '#collapse-show'),
    ('collapse_output', ' #collapse_output'),
    ('collapse_output', '#collapse-output')]:
    cell = nbformat.v4.new_code_cell(f'#comment\n{flag[1]} \ndef some_code')
    test_eq(True, collapse_cells(cell)['metadata'][flag[0]])

#hide
# check that we can't collapse both input and output
cell = nbformat.v4.new_code_cell(f'#hide-input\n#collapse_output \ndef some_code')
test_eq({'hide_input': True, 'collapse_output': True}, hide_cells(collapse_cells(cell))['metadata'])

#collapse_input open
print('This code cell is not collapsed by default but you can collapse it to hide it from view!')
print("Note that the output always shows with `%collapse_input`.")

#collapse_input
print('The code cell that produced this output is collapsed by default but you can expand it!')

#collapse_output
print('The input of this cell is visible as usual.\nHowever, the OUTPUT of this cell is collapsed by default but you can expand it!')

#export
_re_hide = _mk_flag_re('hide', 0, 'Cell with #hide')
_re_cell_to_remove = _mk_flag_re('(default_exp|exporti)', (0,1), 'Cell with #default_exp or #exporti')
_re_default_cls_lvl = _mk_flag_re('default_cls_lvl', 1, "Cell with #default_cls_lvl")

#export
def remove_hidden(cells):
    "Remove in `cells` the ones with a flag `#hide`, `#default_exp`, `#default_cls_lvl` or `#exporti`"
    _hidden = lambda c: check_re(c, _re_hide, code_only=False) or check_re(c, _re_cell_to_remove)
    return L(cells).filter(_hidden, negate=True)

cells = [{'cell_type': 'code', 'source': source, 'hide': hide} for hide, source in [
    (False, '# export\nfrom local.core import *'), 
    (False, '# exporti mod file'), # Note: this used to get removed but we're more strict now
    (True, '# hide\nfrom local.core import *'),
    (False, '# hide_input\nfrom local.core import *'),
    (False, '#exports\nsuper code'),
    (True, '#default_exp notebook.export'),
    (False, 'show_doc(read_nb)'),
    (False, '#hide (last test of to_concat)'),
    (True, '# exporti\n1 + 1')]] + [
    {'cell_type': 'markdown', 'source': source, 'hide': hide} for hide, source in [
    (False, '#hide_input\nnice'), 
    (True, '#hide\n\nto hide')]]

for a,b in zip([cell for cell in cells if not cell['hide']], remove_hidden(cells)):
    test_eq(a,b)

#export
def find_default_level(cells):
    "Find in `cells` the default class level."
    res = L(cells).map_first(check_re_multi, pats=_re_default_cls_lvl)
    return int(res.groups()[0]) if res else 2

tst_nb = read_nb('00_export.ipynb')
test_eq(find_default_level(tst_nb['cells']), 3)

#export
_re_export = _mk_flag_re("exports?", (0,1), "Line with #export or #exports with or without module name")

#export
def nb_code_cell(source):
    "A code cell (as a dict) containing `source`"
    return {'cell_type': 'code', 'execution_count': None, 'metadata': {}, 'outputs': [], 'source': source}

#export
def _show_doc_cell(name, cls_lvl=None):
    return nb_code_cell(f"show_doc({name}{'' if cls_lvl is None else f', default_cls_level={cls_lvl}'})")

def add_show_docs(cells, cls_lvl=None):
    "Add `show_doc` for each exported function or class"
    documented = []
    for cell in cells:
        m = check_re(cell, _re_show_doc)
        if not m: continue
        documented.append(m.group(1))

    def _documented(name): return name in documented

    for cell in cells:
        res.append(cell)
        if check_re(cell, _re_export):
            for n in export_names(cell['source'], func_only=True):
                if not _documented(n): res.insert(len(res)-1, _show_doc_cell(n, cls_lvl=cls_lvl))
    return res

#export
def _show_doc_cell(name, cls_lvl=None):
    return nb_code_cell(f"show_doc({name}{'' if cls_lvl is None else f', default_cls_level={cls_lvl}'})")

def add_show_docs(cells, cls_lvl=None):
    "Add `show_doc` for each exported function or class"
    documented = L(cells).map_filter(check_re, pat=_re_show_doc).map(Self.group(1))
    res = []
    for cell in cells:
        res.append(cell)
        if check_re(cell, _re_export):
            for n in export_names(cell['source'], func_only=True):
                if not n in documented: res.insert(len(res)-1, _show_doc_cell(n, cls_lvl=cls_lvl))
    return res

for i,cell in enumerate(tst_nb['cells']):
    if cell['source'].startswith('#export\ndef read_nb'): break
tst_cells = [c.copy() for c in tst_nb['cells'][i-1:i+1]]
added_cells = add_show_docs(tst_cells, cls_lvl=3)
test_eq(len(added_cells), 3)
test_eq(added_cells[0], tst_nb['cells'][i-1])
test_eq(added_cells[2], tst_nb['cells'][i])
test_eq(added_cells[1], _show_doc_cell('read_nb', cls_lvl=3))
test_eq(added_cells[1]['source'], 'show_doc(read_nb, default_cls_level=3)')

for flag in ['#export', '#exports']:
    for show_doc_source in [
            ('show_doc(my_func)', 'show_doc(my_func, title_level=3)')]:
        #Check show_doc isn't added if it was already there.
        tst_cells1 = [{'cell_type':'code', 'source': f'{flag}\ndef my_func(x):\n    return x'},
                      {'cell_type':'code', 'source': show_doc_source[0]}]
        test_eq(add_show_docs(tst_cells1), tst_cells1)
        #Check show_doc is added
        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))
        tst_cells1 = [{'cell_type':'code', 'source': f'{flag} with.mod\ndef my_func(x):\n    return x'},
                      {'cell_type':'markdown', 'source': 'Some text'},
                      {'cell_type':'code', 'source': show_doc_source[1]}]
        test_eq(add_show_docs(tst_cells1), tst_cells1)
        #Check show_doc is added when using mod export
        test_eq(len(add_show_docs(tst_cells1[:-1])), len(tst_cells1))

#export
_re_fake_header = re.compile(r"""
# Matches any fake header (one that ends with -)
\#+    # One or more #
\s+    # One or more of whitespace
.*     # Any char
-\s*   # A dash followed by any number of white space
$      # End of text
""", re.VERBOSE)

#export
def remove_fake_headers(cells):
    "Remove in `cells` the fake header"
    return [c for c in cells if c['cell_type']=='code' or _re_fake_header.search(c['source']) is None]

cells = [{'cell_type': 'markdown',
          'metadata': {},
          'source': '### Fake-'}] + tst_nb['cells'][:10]
cells1 = remove_fake_headers(cells)
test_eq(len(cells1), len(cells)-1)
test_eq(cells1[0], cells[1])

#export
def remove_empty(cells):
    "Remove in `cells` the empty cells"
    return [c for c in cells if len(c['source']) >0]

#export
_re_title_summary = re.compile(r"""
# Catches the title and summary of the notebook, presented as # Title > summary, with title in group 1 and summary in group 2
^\s*       # Beginning of text followe by any number of whitespace
\#\s+      # # followed by one or more of whitespace
([^\n]*)   # Catching group for any character except a new line
\n+        # One or more new lines
>[ ]*       # > followed by any number of whitespace
([^\n]*)   # Catching group for any character except a new line
""", re.VERBOSE)

_re_title_only = re.compile(r"""
# Catches the title presented as # Title without a summary
^\s*       # Beginning of text followe by any number of whitespace
\#\s+      # # followed by one or more of whitespace
([^\n]*)   # Catching group for any character except a new line
(?:\n|$)    # New line or end of text
""", re.VERBOSE)

_re_properties = re.compile(r"""
^-\s+      # Beginnig of a line followed by - and at least one space
(.*?)      # Any pattern (shortest possible)
\s*:\s*    # Any number of whitespace, :, any number of whitespace
(.*?)$     # Any pattern (shortest possible) then end of line
""", re.MULTILINE | re.VERBOSE)

_re_mdlinks = re.compile(r"\[(.+)]\((.+)\)", re.MULTILINE)

#export
def _md2html_links(s):
    'Converts markdown links to html links'
    return _re_mdlinks.sub(r"<a href='\2'>\1</a>", s)

#export
def get_metadata(cells):
    "Find the cell with title and summary in `cells`."
    for i,cell in enumerate(cells):
        if cell['cell_type'] == 'markdown':
            match = _re_title_summary.match(cell['source'])
            if match:
                cells.pop(i)
                attrs = {k:v for k,v in _re_properties.findall(cell['source'])}
                return {'keywords': 'fastai',
                        'summary' : _md2html_links(match.groups()[1]),
                        'title'   : match.groups()[0],
                        **attrs}
            elif _re_title_only.search(cell['source']) is not None:
                title = _re_title_only.search(cell['source']).groups()[0]
                cells.pop(i)
                attrs = {k:v for k,v in _re_properties.findall(cell['source'])}
                return {'keywords': 'fastai',
                        'title'   : title,
                        **attrs}

    return {'keywords': 'fastai',
            'title'   : 'Title'}

tst_nb = read_nb('00_export.ipynb')
test_eq(get_metadata(tst_nb['cells']), {
    'keywords': 'fastai',
    'summary': 'The functions that transform notebooks in a library',
    'title': 'Export to modules'})

#The cell with the metada is popped out, so if we do it a second time we get the default.
test_eq(get_metadata(tst_nb['cells']), {'keywords': 'fastai', 'title'   : 'Title'})

#hide
#test with title only
test_eq(get_metadata([{'cell_type': 'markdown', 'source': '# Awesome title'}]), 
        {'keywords': 'fastai', 'title': 'Awesome title'})

#hide
text = r"""
[This](https://nbdev.fast.ai) goes to docs.
This [one:here](00_export.ipynb) goes to a local nb. 
\n[And-this](http://dev.fast.ai/) goes to fastai docs
"""
res = """
<a href='https://nbdev.fast.ai'>This</a> goes to docs.\nThis <a href='00_export.ipynb'>one:here</a> goes to a local nb. \n\\n<a href='http://dev.fast.ai/'>And-this</a> goes to fastai docs
"""
test_eq(_md2html_links(text), res)

#hide
cells = [{'cell_type': 'markdown', 'source': "# Title\n\n> s\n\n- toc: false"}]
test_eq(get_metadata(cells), {'keywords': 'fastai', 'summary': 's', 'title': 'Title', 'toc': 'false'})

#export
_re_mod_export = _mk_flag_re("export[s]?", 1,
    "Matches any line with #export or #exports with a module name and catches it in group 1")

def _gather_export_mods(cells):
    res = []
    for cell in cells:
        tst = check_re(cell, _re_mod_export)
        if tst is not None: res.append(tst.groups()[0])
    return res

#hide
cells = [
    {'cell_type': 'markdown', 'source': '#export ignored'},
    {'cell_type': 'code', 'source': '#export'},
    {'cell_type': 'code', 'source': '#export normal'},
    {'cell_type': 'code', 'source': '# exports show'},
    {'cell_type': 'code', 'source': '# exporti hidden'},
    {'cell_type': 'code', 'source': '#export\n@call_parse'},
    {'cell_type': 'code', 'source': '#export \n@delegates(concurrent.futures.ProcessPoolExecutor)'}
]

test_eq(_gather_export_mods(cells), ['normal', 'show'])

#export
# match any cell containing a zero indented import from the current lib
_re_lib_import = ReLibName(r"^from LIB_NAME\.", re.MULTILINE)
# match any cell containing a zero indented import
_re_import = re.compile(r"^from[ \t]+\S+[ \t]+import|^import[ \t]", re.MULTILINE)
# match any cell containing a zero indented call to notebook2script
_re_notebook2script = re.compile(r"^notebook2script\(", re.MULTILINE)

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (True, 'show_doc(Tensor.p)'),
    (True, ' show_doc(Tensor.p)'),
    (True, 'if somthing:\n    show_doc(Tensor.p)'),
    (False, '# show_doc(Tensor.p)'),
    (True, '# comment \n show_doc(Tensor.p)'),
    (True, '"""\nshow_doc(Tensor.p)\n"""'),
    (True, 'import torch\nshow_doc(Tensor.p)'),
    (False,'class Ex(ExP):\n"An `ExP` that ..."\ndef preprocess_cell(self, cell, resources, index):\n'),
    (False, 'from somewhere import something'),
    (False, 'from '),
    (False, 'import re'),
    (False, 'import '),
    (False, 'try: from PIL import Image\except: pass'),
    (False, 'from PIL import Image\n@patch\ndef p(x:Image):\n    pass'),
    (False, '@patch\ndef p(x:Image):\n    pass\nfrom PIL import Image')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re_multi(cell, [_re_show_doc, _re_lib_import.re])), f'expected {exp} for {cell}'

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (False, 'show_doc(Tensor.p)'),
    (True, 'import torch\nshow_doc(Tensor.p)'),
    (False,'class Ex(ExP):\n"An `ExP` that ..."\ndef preprocess_cell(self, cell, resources, index):\n'),
    (False, ' from somewhere import something'),
    (True, 'from somewhere import something'),
    (False, 'from '),
    (False, 'select * \nfrom database'),
    (False, ' import re'),
    (True, 'import re'),
    (True, 'import '),
    (False, 'try: from PIL import Image\except: pass'),
    (True, 'from PIL import Image\n@patch\ndef p(x:Image):\n    pass'),
    (True, '@patch\ndef p(x:Image):\n    pass\nfrom PIL import Image')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re(cell, _re_import)), f'expected {exp} for {cell}'

#hide
for cell in [nbformat.v4.new_code_cell(s, metadata={'exp': exp}) for exp,s in [
    (False, 'show_doc(Tensor.p)'),
    (False, 'notebook2script'),
    (False, '#notebook2script()'),
    (True, 'notebook2script()'),
    (True, 'notebook2script(anything at all)')]]:
    exp = cell.metadata.exp
    assert exp == bool(check_re(cell, _re_notebook2script)), f'expected {exp} for {cell}'

#export
def _non_comment_code(s):
    if re.match(r'\s*#', s): return False
    if _re_import.findall(s) or _re_lib_import.re.findall(s): return False
    return re.match(r'\s*\w', s)

#export
class ExecuteShowDocPreprocessor(ExecutePreprocessor):
    "An `ExecutePreprocessor` that only executes `show_doc` and `import` cells"
    def preprocess_cell(self, cell, resources, index):
        if not check_re(cell, _re_notebook2script):
            if check_re(cell, _re_show_doc):
                return super().preprocess_cell(cell, resources, index)
            elif check_re_multi(cell, [_re_import, _re_lib_import.re]):
                if check_re_multi(cell, [_re_export, 'show_doc', '^\s*#\s*import']):
#                     r = list(filter(_non_comment_code, cell['source'].split('\n')))
#                     if r: print("You have import statements mixed with other code", r)
                    return super().preprocess_cell(cell, resources, index)
#                 try: return super().preprocess_cell(cell, resources, index)
#                 except: pass
        return cell, resources

#export
def _import_show_doc_cell(mods=None):
    "Add an import show_doc cell."
    source = f"from nbdev.showdoc import show_doc"
    if mods is not None:
        for mod in mods: source += f"\nfrom {Config().lib_name}.{mod} import *"
    return {'cell_type': 'code',
            'execution_count': None,
            'metadata': {'hide_input': True},
            'outputs': [],
            'source': source}

def execute_nb(nb, mod=None, metadata=None, show_doc_only=True):
    "Execute `nb` (or only the `show_doc` cells) with `metadata`"
    mods = ([] if mod is None else [mod]) + _gather_export_mods(nb['cells'])
    nb['cells'].insert(0, _import_show_doc_cell(mods))
    ep_cls = ExecuteShowDocPreprocessor if show_doc_only else ExecutePreprocessor
    ep = ep_cls(timeout=600, kernel_name='python3')
    metadata = metadata or {}
    pnb = nbformat.from_dict(nb)
    ep.preprocess(pnb, metadata)
    return pnb

#export
_re_cite = re.compile(r"(\\cite{)([^}]*)(})", re.MULTILINE | re.VERBOSE) # Catches citations used with `\cite{}`

#export
def _textcite2link(text):
    citations = _re_cite.finditer(text)
    out = []
    start_pos = 0
    for cit_group in citations:
        cit_pos_st =  cit_group.span()[0]
        cit_pos_fin =  cit_group.span()[1]
        out.append(text[start_pos:cit_pos_st])
        out.append('[')
        cit_group = cit_group[2].split(',')
        for i, cit in enumerate(cit_group):
            cit=cit.strip()
            out.append(f"""<a class="latex_cit" id="call-{cit}" href="#cit-{cit}">{cit}</a>""")
            if i != len(cit_group) - 1:
                out.append(',')
        out.append(']')
        start_pos = cit_pos_fin
    out.append(text[start_pos:])
    return ''.join(out)

#export
def cite2link(cell):
    '''Creates links from \cite{} to Refenrence section generated by jupyter_latex_envs'''
    if cell['cell_type'] == 'markdown': cell['source'] = _textcite2link(cell['source'])
    return cell

#hide
cell = {'cell_type': 'markdown', 'source': r"""This is cited multireference \cite{Frob1, Frob3}.
And single \cite{Frob2}."""}
expected=r"""This is cited multireference [<a class="latex_cit" id="call-Frob1" href="#cit-Frob1">Frob1</a>,<a class="latex_cit" id="call-Frob3" href="#cit-Frob3">Frob3</a>].
And single [<a class="latex_cit" id="call-Frob2" href="#cit-Frob2">Frob2</a>]."""
test_eq(cite2link(cell)["source"], expected)

#slow
fake_nb = {k:v for k,v in tst_nb.items() if k != 'cells'}
fake_nb['cells'] = [tst_nb['cells'][0].copy()] + added_cells
fake_nb = execute_nb(fake_nb, mod='export')
assert len(fake_nb['cells'][-2]['outputs']) > 0

#export
def write_tmpl(tmpl, nms, cfg, dest):
    "Write `tmpl` to `dest` (if missing) filling in `nms` in template using dict `cfg`"
    if dest.exists(): return
    vs = {o:cfg.d[o] for o in nms.split()}
    outp = tmpl.format(**vs)
    dest.write_text(outp)

#export
def write_tmpls():
    "Write out _config.yml and _data/topnav.yml using templates"
    cfg = Config()
    path = Path(cfg.get('doc_src_path', cfg.doc_path))
    write_tmpl(config_tmpl, 'user lib_name title copyright description', cfg, path/'_config.yml')
    write_tmpl(topnav_tmpl, 'host git_url', cfg, path/'_data'/'topnav.yml')
    write_tmpl(makefile_tmpl, 'nbs_path lib_name', cfg, cfg.config_file.parent/'Makefile')

__file__ = Config().lib_path/'export2html.py'

#export
def nbdev_exporter(cls=HTMLExporter, template_file=None):
    cfg = traitlets.config.Config()
    exporter = cls(cfg)
    exporter.exclude_input_prompt=True
    exporter.exclude_output_prompt=True
    exporter.anchor_link_text = ' '
    exporter.template_file = 'jekyll.tpl' if template_file is None else template_file
    exporter.template_path.append(str(Path(__file__).parent/'templates'))
    return exporter

#export
process_cells = [remove_fake_headers, remove_hidden, remove_empty]
process_cell  = [hide_cells, collapse_cells, remove_widget_state, add_jekyll_notes, escape_latex, cite2link]

#export
def _nb2htmlfname(nb_path, dest=None):
    if dest is None: dest = Config().doc_path
    return Path(dest)/re_digits_first.sub('', nb_path.with_suffix('.html').name)

#hide
test_eq(_nb2htmlfname(Path('00a_export.ipynb')), Config().doc_path/'export.html')
test_eq(_nb2htmlfname(Path('export.ipynb')), Config().doc_path/'export.html')
test_eq(_nb2htmlfname(Path('00ab_export_module_1.ipynb')), Config().doc_path/'export_module_1.html')
test_eq(_nb2htmlfname(Path('export.ipynb'), '.'), Path('export.html'))

#export
def convert_nb(fname, cls=HTMLExporter, template_file=None, exporter=None, dest=None, execute=True):
    "Convert a notebook `fname` to html file in `dest_path`."
    fname = Path(fname).absolute()
    nb = read_nb(fname)
    meta_jekyll = get_metadata(nb['cells'])
    meta_jekyll['nb_path'] = str(fname.relative_to(Config().lib_path.parent))
    cls_lvl = find_default_level(nb['cells'])
    mod = find_default_export(nb['cells'])
    nb['cells'] = compose(*process_cells,partial(add_show_docs, cls_lvl=cls_lvl))(nb['cells'])
    _func = compose(partial(copy_images, fname=fname, dest=Config().doc_path), *process_cell, treat_backticks)
    nb['cells'] = [_func(c) for c in nb['cells']]
    if execute: nb = execute_nb(nb, mod=mod)
    nb['cells'] = [clean_exports(c) for c in nb['cells']]
    if exporter is None: exporter = nbdev_exporter(cls=cls, template_file=template_file)
    with open(_nb2htmlfname(fname, dest=dest),'w') as f:
        f.write(exporter.from_notebook_node(nb, resources=meta_jekyll)[0])

#export
def _notebook2html(fname, cls=HTMLExporter, template_file=None, exporter=None, dest=None, execute=True):
    time.sleep(random.random())
    print(f"converting: {fname}")
    try:
        convert_nb(fname, cls=cls, template_file=template_file, exporter=exporter, dest=dest, execute=execute)
        return True
    except Exception as e:
        print(e)
        return False

#export
def notebook2html(fname=None, force_all=False, n_workers=None, cls=HTMLExporter, template_file=None,
                  exporter=None, dest=None, pause=0, execute=True):
    "Convert all notebooks matching `fname` to html files"
    if fname is None:
        files = [f for f in Config().nbs_path.glob('**/*.ipynb')
                 if not f.name.startswith('_') and not '/.' in f.as_posix()]
    else:
        p = Path(fname)
        files = list(p.parent.glob(p.name))
    if len(files)==1:
        force_all = True
        if n_workers is None: n_workers=0
    if not force_all:
        # only rebuild modified files
        files,_files = [],files.copy()
        for fname in _files:
            fname_out = _nb2htmlfname(Path(fname).absolute(), dest=dest)
            if not fname_out.exists() or os.path.getmtime(fname) >= os.path.getmtime(fname_out):
                files.append(fname)
    if len(files)==0: print("No notebooks were modified")
    else:
        passed = parallel(_notebook2html, files, n_workers=n_workers, cls=cls,
                          template_file=template_file, exporter=exporter, dest=dest, pause=pause, execute=execute)
        if not all(passed):
            msg = "Conversion failed on the following:\n"
            print(msg + '\n'.join([f.name for p,f in zip(passed,files) if not p]))

#hide
# Test when an argument is given to notebook2html
p1 = Path('/tmp/sync.html')
if p1.exists(): p1.unlink()
notebook2html('01_sync.ipynb', dest='/tmp');
assert p1.exists()

#slow
#hide
# Test when no argument is given to notebook2html
dest_files = [_nb2htmlfname(f, dest='/tmp') for f in Config().nbs_path.glob('**/*.ipynb')
              if not f.name.startswith('_') and not 'checkpoint' in f.name]
[f.unlink() for f in dest_files if f.exists()]
notebook2html(fname=None, dest='/tmp');
for f in dest_files: assert f.exists(), f

#hide
# # Test Error handling
# try: notebook2html('../README.md');
# except Exception as e: pass
# else: assert False, 'An error should be raised when a non-notebook file is passed to notebook2html!'

#hide
#notebook2html(force_all=True)

#export
def convert_md(fname, dest_path, img_path='docs/images/', jekyll=True):
    "Convert a notebook `fname` to a markdown file in `dest_path`."
    fname = Path(fname).absolute()
    if not img_path: img_path = fname.stem + '_files/'
    Path(img_path).mkdir(exist_ok=True, parents=True)
    nb = read_nb(fname)
    meta_jekyll = get_metadata(nb['cells'])
    try: meta_jekyll['nb_path'] = str(fname.relative_to(Config().lib_path.parent))
    except: meta_jekyll['nb_path'] = str(fname)
    nb['cells'] = compose(*process_cells)(nb['cells'])
    nb['cells'] = [compose(partial(adapt_img_path, fname=fname, dest=dest_path, jekyll=jekyll), *process_cell)(c)
                   for c in nb['cells']]
    fname = Path(fname).absolute()
    dest_name = fname.with_suffix('.md').name
    exp = nbdev_exporter(cls=MarkdownExporter, template_file='jekyll-md.tpl' if jekyll else 'md.tpl')
    export = exp.from_notebook_node(nb, resources=meta_jekyll)
    md = export[0]
    for ext in ['png', 'svg']:
        md = re.sub(r'!\['+ext+'\]\((.+)\)', '!['+ext+'](' + img_path + '\\1)', md)
    with (Path(dest_path)/dest_name).open('w') as f: f.write(md)
    if hasattr(export[1]['outputs'], 'items'):
        for n,o in export[1]['outputs'].items():
            with open(Path(dest_path)/img_path/n, 'wb') as f: f.write(o)

#hide
def _test_md(fn):
    fn,dest = Path(fn),Path().absolute().parent
    try: convert_md(fn, dest, jekyll=False)
    finally: (dest/f'{fn.stem}.md').unlink()

#hide
_test_md('index.ipynb')
# `export[1]['outputs']` will be a `str` if the notebook has no markdown cells to convert.
# e.g. the nb could have a single jekyll markdown cell or just code cells ...
_test_md(f'../tests/single-cell-index.ipynb')

#export
_re_att_ref = re.compile(r' *!\[(.*)\]\(attachment:image.png(?: "(.*)")?\)')

t = '![screenshot](attachment:image.png)'
test_eq(_re_att_ref.match(t).groups(), ('screenshot', None))

t = '![screenshot](attachment:image.png "Deploying to Binder")'
test_eq(_re_att_ref.match(t).groups(), ('screenshot', "Deploying to Binder"))

#export
try: from PIL import Image
except: pass # Only required for _update_att_ref

#export
_tmpl_img = '<img alt="{title}" width="{width}" caption="{title}" id="{id}" src="{name}">'

def _update_att_ref(line, path, img):
    m = _re_att_ref.match(line)
    if not m: return line
    alt,title = m.groups()
    w = img.size[0]
    if alt=='screenshot': w //= 2
    if not title: title = "TK: add title"
    return _tmpl_img.format(title=title, width=str(w), id='TK: add it', name=str(path))

#export
def _nb_detach_cell(cell, dest, use_img):
    att,src = cell['attachments'],cell['source']
    mime,img = first(first(att.values()).items())
    ext = mime.split('/')[1]
    for i in range(99999):
        p = dest/(f'att_{i:05d}.{ext}')
        if not p.exists(): break
    img = b64decode(img)
    p.write_bytes(img)
    del(cell['attachments'])
    if use_img:  return [_update_att_ref(o,p,Image.open(p)) for o in src]
    else: return [o.replace('attachment:image.png', str(p)) for o in src]

#export
def nb_detach_cells(path_nb, dest=None, replace=True, use_img=False):
    "Export cell attachments to `dest` and update references"
    path_nb = Path(path_nb)
    if not dest: dest = f'{path_nb.stem}_files'
    dest = Path(dest)
    dest.mkdir(exist_ok=True, parents=True)
    j = json.load(path_nb.open())
    atts = [o for o in j['cells'] if 'attachments' in o]
    for o in atts: o['source'] = _nb_detach_cell(o, dest, use_img)
    if atts and replace: json.dump(j, path_nb.open('w'))
    if not replace: return j

#export
import time,random,warnings

#export
def _leaf(k,v):
    url = 'external_url' if "http" in v else 'url'
    #if url=='url': v=v+'.html'
    return {'title':k, url:v, 'output':'web,pdf'}

#export
_k_names = ['folders', 'folderitems', 'subfolders', 'subfolderitems']
def _side_dict(title, data, level=0):
    k_name = _k_names[level]
    level += 1
    res = [(_side_dict(k, v, level) if isinstance(v,dict) else _leaf(k,v))
        for k,v in data.items()]
    return ({k_name:res} if not title
            else res if title.startswith('empty')
            else {'title': title, 'output':'web', k_name: res})

#export
_re_catch_title = re.compile('^title\s*:\s*(\S+.*)$', re.MULTILINE)

#export
def _get_title(fname):
    "Grabs the title of html file `fname`"
    with open(fname, 'r') as f: code = f.read()
    src =  _re_catch_title.search(code)
    return fname.stem if src is None else src.groups()[0]

#hide
test_eq(_get_title(Config().doc_path/'export.html'), "Export to modules")

#export
def _create_default_sidebar():
    "Create the default sidebar for the docs website"
    dic = {"Overview": "/"}
    files = [f for f in Config().nbs_path.glob('**/*.ipynb') if not f.name.startswith('_')]
    fnames = [_nb2htmlfname(f) for f in sorted(files)]
    titles = [_get_title(f) for f in fnames if 'index' not in f.stem!='index']
    if len(titles) > len(set(titles)): print(f"Warning: Some of your Notebooks use the same title ({titles}).")
    dic.update({_get_title(f):f'{f.name}' for f in fnames if f.stem!='index'})
    return dic

#export
def create_default_sidebar():
    "Create the default sidebar for the docs website"
    dic = {Config().lib_name: _create_default_sidebar()}
    json.dump(dic, open(Config().doc_path/'sidebar.json', 'w'), indent=2)

#export
def make_sidebar():
    "Making sidebar for the doc website form the content of `doc_folder/sidebar.json`"
    cfg = Config()
    if not (cfg.doc_path/'sidebar.json').exists() or cfg.get('custom_sidebar', 'False') == 'False':
        create_default_sidebar()
    sidebar_d = json.load(open(cfg.doc_path/'sidebar.json', 'r'))
    res = _side_dict('Sidebar', sidebar_d)
    res = {'entries': [res]}
    res_s = yaml.dump(res, default_flow_style=False)
    res_s = res_s.replace('- subfolders:', '  subfolders:').replace(' - - ', '   - ')
    res_s = f"""
#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# Instead edit {'../../sidebar.json'}
"""+res_s
    open(cfg.doc_path/'_data/sidebars/home_sidebar.yml', 'w').write(res_s)

#hide
from nbdev.export import *
notebook2script()