#|default_exp merge
Fix merge conflicts in jupyter notebooks
#|export
from nbdev.imports import *
from nbdev.config import *
from nbdev.export import *
from nbdev.sync import *
from execnb.nbio import *
from fastcore.script import *
from fastcore import shutil
import subprocess
from difflib import SequenceMatcher
#|hide
from nbdev.doclinks import nbdev_export
from fastcore.test import *
When working with jupyter notebooks (which are json files behind the scenes) and GitHub, it is very common that a merge conflict (that will add new lines in the notebook source file) will break some notebooks you are working on. This module defines the function nbdev_fix
to fix those notebooks for you, and attempt to automatically merge standard conflicts. The remaining ones will be delimited by markdown cells like this:
<<<<<< HEAD
# local code here
======
# remote code here
>>>>>> a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35
Below is an example of broken notebook. The json format is broken by the lines automatically added by git. Such a file can't be opened in jupyter notebook.
broken = Path('..//tests/example.ipynb.broken')
tst_nb = broken.read_text()
print(tst_nb)
{ "cells": [ { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ <<<<<<< HEAD "z=3\n", ======= "z=2\n", >>>>>>> a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35 "z" ] }, { "cell_type": "code", "execution_count": 7, "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, <<<<<<< HEAD "execution_count": 7, ======= "execution_count": 5, >>>>>>> a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35 "metadata": {}, "output_type": "execute_result" } ], "source": [ "x=3\n", "y=3\n", "x+y" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 2 }
Note that in this example, the second conflict is easily solved: it just concerns the execution count of the second cell and can be solved by choosing either option without really impacting your notebook. This is the kind of conflict we will fix automatically. The first conflict is more complicated as it spans across two cells and there is a cell present in one version, not the other. Such a conflict (and generally the ones where the inputs of the cells change form one version to the other) aren't automatically fixed, but we will return a proper json file where the annotations introduced by git will be placed in markdown cells.
The approach we use is to first "unpatch" the conflicted file, regenerating the two files it was originally created from. Then we redo the diff process, but using cells instead of text lines.
#|export
_BEG,_MID,_END = '<'*7,'='*7,'>'*7
conf_re = re.compile(rf'^{_BEG}\s+(\S+)\n(.*?)^{_MID}\n(.*?)^{_END}\s+([\S ]+)\n', re.MULTILINE|re.DOTALL)
def _unpatch_f(before, cb1, cb2, c, r):
if cb1 is not None and cb1 != cb2: raise Exception(f'Branch mismatch: {cb1}/{cb2}')
r.append(before)
r.append(c)
return cb2
#|export
def unpatch(s:str):
"Takes a string with conflict markers and returns the two original files, and their branch names"
*main,last = conf_re.split(s)
r1,r2,c1b,c2b = [],[],None,None
for before,c1_branch,c1,c2,c2_branch in chunked(main, 5):
c1b = _unpatch_f(before, c1b, c1_branch, c1, r1)
c2b = _unpatch_f(before, c2b, c2_branch, c2, r2)
return ''.join(r1+[last]), ''.join(r2+[last]), c1b, c2b
The result of "unpatching" our conflicted test notebook is the two original notebooks it would have been created from. Each of these original notebooks will contain valid JSON:
a,b,branch1,branch2 = unpatch(tst_nb)
dict2nb(loads(a))
{ 'cells': [ { 'cell_type': 'code',
'execution_count': 6,
'idx_': 0,
'metadata': {},
'outputs': [ { 'data': {'text/plain': ['3']},
'execution_count': 6,
'metadata': {},
'output_type': 'execute_result'}],
'source': 'z=3\nz'},
{ 'cell_type': 'code',
'execution_count': 5,
'idx_': 1,
'metadata': {},
'outputs': [ { 'data': {'text/plain': ['6']},
'execution_count': 7,
'metadata': {},
'output_type': 'execute_result'}],
'source': 'x=3\ny=3\nx+y'},
{ 'cell_type': 'code',
'execution_count': None,
'idx_': 2,
'metadata': {},
'outputs': [],
'source': ''}],
'metadata': { 'kernelspec': { 'display_name': 'Python 3',
'language': 'python',
'name': 'python3'}},
'nbformat': 4,
'nbformat_minor': 2}
dict2nb(loads(b))
{ 'cells': [ { 'cell_type': 'code',
'execution_count': 6,
'idx_': 0,
'metadata': {},
'outputs': [ { 'data': {'text/plain': ['3']},
'execution_count': 6,
'metadata': {},
'output_type': 'execute_result'}],
'source': 'z=2\nz'},
{ 'cell_type': 'code',
'execution_count': 5,
'idx_': 1,
'metadata': {},
'outputs': [ { 'data': {'text/plain': ['6']},
'execution_count': 5,
'metadata': {},
'output_type': 'execute_result'}],
'source': 'x=3\ny=3\nx+y'},
{ 'cell_type': 'code',
'execution_count': None,
'idx_': 2,
'metadata': {},
'outputs': [],
'source': ''}],
'metadata': { 'kernelspec': { 'display_name': 'Python 3',
'language': 'python',
'name': 'python3'}},
'nbformat': 4,
'nbformat_minor': 2}
branch1,branch2
('HEAD', 'a7ec1b0bfb8e23b05fd0a2e6cafcb41cd0fb1c35')
#|export
def _make_md(code): return [dict(source=f'`{code}`', cell_type="markdown", metadata={})]
def _make_conflict(a,b, branch1, branch2):
return _make_md(f'{_BEG} {branch1}') + a+_make_md(_MID)+b + _make_md(f'{_END} {branch2}')
def _merge_cells(a, b, brancha, branchb, theirs):
matches = SequenceMatcher(None, a, b).get_matching_blocks()
res,prev_sa,prev_sb,conflict = [],0,0,False
for sa,sb,sz in matches:
ca,cb = a[prev_sa:sa],b[prev_sb:sb]
if ca or cb:
res += _make_conflict(ca, cb, brancha, branchb)
conflict = True
if sz: res += b[sb:sb+sz] if theirs else a[sa:sa+sz]
prev_sa,prev_sb = sa+sz,sb+sz
return res,conflict
#|export
@call_parse
def nbdev_fix(nbname:str, # Notebook filename to fix
outname:str=None, # Filename of output notebook (defaults to `nbname`)
nobackup:bool_arg=True, # Do not backup `nbname` to `nbname`.bak if `outname` not provided
theirs:bool=False, # Use their outputs and metadata instead of ours
noprint:bool=False): # Do not print info about whether conflicts are found
"Create working notebook from conflicted notebook `nbname`"
nbname = Path(nbname)
if not nobackup and not outname: shutil.copy(nbname, nbname.with_suffix('.ipynb.bak'))
nbtxt = nbname.read_text()
a,b,branch1,branch2 = unpatch(nbtxt)
ac,bc = dict2nb(loads(a)),dict2nb(loads(b))
dest = bc if theirs else ac
cells,conflict = _merge_cells(ac.cells, bc.cells, branch1, branch2, theirs=theirs)
dest.cells = cells
write_nb(dest, ifnone(outname, nbname))
if not noprint:
if conflict: print("One or more conflict remains in the notebook, please inspect manually.")
else: print("Successfully merged conflicts!")
return conflict
This begins by optionally backing the notebook fname
to fname.bak
in case something goes wrong. Then it parses the broken json, solving conflicts in cells. Every conflict that only involves metadata or outputs of cells will be solved automatically by using the local (theirs==False
) or the remote (theirs==True
) branch. Otherwise, or for conflicts involving the inputs of cells, the json will be repaired by including the two version of the conflicted cell(s) with markdown cells indicating the conflicts. You will be able to open the notebook again and search for the conflicts (look for <<<<<<<
) then fix them as you wish.
A message will be printed indicating whether the notebook was fully merged or if conflicts remain.
nbdev_fix(broken, outname='tmp.ipynb')
chk = read_nb('tmp.ipynb')
test_eq(len(chk.cells), 7)
os.unlink('tmp.ipynb')
One or more conflict remains in the notebook, please inspect manually.
#|export
def _git_branch_merge():
try: return only(v for k,v in os.environ.items() if k.startswith('GITHEAD'))
except ValueError: return
#|export
def _git_rebase_head():
for d in ('apply','merge'):
d = Path(f'.git/rebase-{d}')
if d.is_dir():
cmt = (d/'orig-head').read_text()
msg = run(f'git show-branch --no-name {cmt}')
return f'{cmt[:7]} ({msg})'
#|export
def _git_merge_file(base, ours, theirs):
"`git merge-file` with expected labels depending on if a `merge` or `rebase` is in-progress"
l_theirs = _git_rebase_head() or _git_branch_merge() or 'THEIRS'
cmd = f"git merge-file -L HEAD -L BASE -L '{l_theirs}' {ours} {base} {theirs}"
return subprocess.run(cmd, shell=True, capture_output=True, text=True)
#|export
@call_parse
def nbdev_merge(base:str, ours:str, theirs:str, path:str):
"Git merge driver for notebooks"
if not _git_merge_file(base, ours, theirs).returncode: return
theirs = str2bool(os.environ.get('THEIRS', False))
return nbdev_fix.__wrapped__(ours, theirs=theirs)
This implements a git merge driver for notebooks that automatically resolves conflicting metadata and outputs, and splits remaining conflicts as separate cells so that the notebook can be viewed and fixed in Jupyter. The easiest way to install it is by running nbdev_install_hooks
.
This works by first running Git's default merge driver, and then nbdev_fix
if there are still conflicts. You can set nbdev_fix
's theirs
argument using the THEIRS
environment variable, for example:
THEIRS=True git merge branch
#|hide
import nbdev.doclinks; nbdev.nbdev_export()