Notebook

In [1]:

%install-location $cwd/swift-install
%install '.package(url: "https://github.com/mxcl/Path.swift", from: "0.16.1")' Path

Installing packages:
	.package(url: "https://github.com/mxcl/Path.swift", from: "0.16.1")
		Path
With SwiftPM flags: []
Working in: /tmp/tmp8x665_rc/swift-install
warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
Fetching https://github.com/mxcl/Path.swift
Completed resolution in 0.65s
Cloning https://github.com/mxcl/Path.swift
Resolving https://github.com/mxcl/Path.swift at 0.16.3
warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)
/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[1/10] Compiling Path Path+StringConvertibles.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[2/10] Compiling Path Extensions.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[3/10] Compiling Path Path+Attributes.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[4/10] Compiling Path Path->Bool.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[5/10] Compiling Path Path+Codable.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[6/10] Compiling Path Path+ls.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[7/10] Compiling Path Path+CommonDirectories.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[8/10] Compiling Path Path+FileManager.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[9/10] Compiling Path Path.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[10/11] Merging module Path
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[11/12] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
[12/13] Merging module jupyterInstalledPackages
/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)
/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)
/home/sgugger/swift/usr/bin/swift-autolink-extract: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift-autolink-extract)
[13/13] Linking libjupyterInstalledPackages.so
Initializing Swift...
Installation complete!

In [2]:

import Foundation
import Path

In [3]:

let path = Path.cwd.parent
print(path)

/home/sgugger/git/swiftai

In [4]:

public extension String {
    func findFirst(pat: String) -> Range<String.Index>? {
        return range(of: pat, options: .regularExpression)
    }
    func hasMatch(pat: String) -> Bool {
        return findFirst(pat:pat) != nil
    }
    func withMaj() -> String {
        return prefix(1).capitalized + dropFirst()
    }
}

Notebook to script¶

In [5]:

public func nbNameToScriptName(_ name: String) -> String {
    var splits = name.components(separatedBy: "_")
    splits = splits[1...].map { $0.withMaj() }
    return splits.joined(separator: "")
}

In [6]:

public func readNb(_ fname: Path) -> [String:Any] {
    let data = try! Data(contentsOf: fname.url)
    return try! JSONSerialization.jsonObject(with: data, options: .allowFragments) as! [String: Any]
}

In [7]:

public func notebookToScript(_ fname: Path, dest: Path?=nil) {
    let newName = nbNameToScriptName(fname.basename(dropExtension: true))+".swift"
    let destFname = (dest ?? fname.parent) / newName
    let cells = readNb(fname)["cells"] as! [[String:Any]]
        var module = """
/*
This file was autogenerated from \(fname.basename())
        
If you edit it, be sure that:
  1. there is no diff between this file and the corresponding notebook prior to editing
  2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook
        
Run *** when you are done to update the notebooks with your change.
*/
        
"""
    for (i,cell) in cells.enumerated() {
        if let source = cell["source"] as? [String], !source.isEmpty, source[0].hasMatch(pat: #"^\s*//\s*export\s*$"#) {
            module.append("\n//cell\(i)\n\(source[1...].joined())\n")
        }
    }
    try! module.write(to: destFname, encoding: .utf8)
}

In [9]:

let fname = path/"/nbs/00_load_data.ipynb"
let dest = path/"Sources/SwiftAI"

In [10]:

notebookToScript(fname, dest: dest)

In [11]:

public func makeLibrary(_ nbFolder: Path, dest: Path?=nil){
    for entry in try! nbFolder.ls()  where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #"^\d+[a-z]*_.*ipynb$"#) {
        print("Converting \(entry.path.basename())")
        notebookToScript(entry.path, dest: dest)
    }
}

In [12]:

let nbFolder = path/"nbs"

In [13]:

makeLibrary(nbFolder, dest:dest)

Converting 02a_why_sqrt5.ipynb
Converting 10_mixup_ls.ipynb
Converting 05_anneal.ipynb
Converting 09_optimizer.ipynb
Converting 04_callbacks.ipynb
Converting 08_data_block.ipynb
Converting 08a_heterogeneous_dictionary.ipynb
Converting 06_cuda.ipynb
Converting 01_matmul.ipynb
Converting 02_fully_connected.ipynb
Converting 01a_fastai_layers.ipynb
Converting 11_imagenette.ipynb
Converting 05b_early_stopping.ipynb
Converting 00_load_data.ipynb
Converting 03_minibatch_training.ipynb
Converting 07_batchnorm.ipynb

Script to notebook¶

In [13]:

let fname = dest/"LoadData.swift"

In [14]:

public func readScript(_ fname: Path) -> String {
    let data = try! Data(contentsOf: fname.url)
    return String(data: data, encoding: .utf8)!
}

In [15]:

public func writeNotebook(_ nbFname: Path, nbData: [String: Any]) {
    let outData = try! JSONSerialization.data(withJSONObject: nbData, options: .prettyPrinted)  
    let jsonString = String(data: outData, encoding: .utf8)! 
    do { try jsonString.write(to: nbFname, encoding: .utf8) }
    catch { "Couldn't save notebook" }
}

In [18]:

public func scriptToNotebook(_ fname: Path, nbFolder: Path){
    let code = readScript(fname)
    let codeCells = code.components(separatedBy: "//cell")
    let nbName = codeCells[0].components(separatedBy: "\n")[1].components(separatedBy: "from ")[1]
    let nbFname = nbFolder / nbName
    var jsonData = readNb(nbFname)
    var cells = jsonData["cells"] as! [[String:Any]]
    for c in codeCells[1...] {
        var lines = c.components(separatedBy: "\n")
        let idx: Int = Int(lines[0])!
        var i = lines.count-1
        while lines[i].isEmpty { i -= 1}
        if i > 1 {
            for i in 1...(i-1) { lines[i].append("\n") }
        }
        lines[0] = "// export\n"
        cells[idx]["source"] = Array(lines[...i])
    }
    jsonData["cells"] = cells
    writeNotebook(nbFname, nbData: jsonData)
}

In [19]:

scriptToNotebook(fname, nbFolder: nbFolder)

In [20]:

public func updateNotebooks(_ scriptsFolder: Path, nbFolder: Path) {
    for entry in try! scriptsFolder.ls()  where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #".swift$"#) {
        print("Updating nb from \(entry.path.basename())")
        scriptToNotebook(entry.path, nbFolder: nbFolder)
    }
}

In [21]:

updateNotebooks(dest, nbFolder: nbFolder)

Updating nb from WhySqrt5.swift
Updating nb from Anneal.swift
Updating nb from Callbacks.swift
Updating nb from Optimizer.swift
Updating nb from MixupLs.swift
Updating nb from DataBlock.swift
Updating nb from FastaiLayers.swift
Updating nb from FullyConnected.swift
Updating nb from HeterogeneousDictionary.swift
Updating nb from Batchnorm.swift
Updating nb from MinibatchTraining.swift
Updating nb from LoadData.swift
Updating nb from Matmul.swift
Updating nb from Imagenette.swift
Updating nb from EarlyStopping.swift
Updating nb from Cuda.swift

Diffing¶

In [ ]:

public func diffNbScript(_ nbFname: Path, dest: Path){
    let newName = nbNameToScriptName(fname.basename(dropExtension: true))+".swift"
    let destFname = (dest ?? fname.parent) / newName
    let cells = readNb(nbFname)["cells"] as! [[String:Any]]
    
    let data = try! Data(contentsOf: fname.url)
    let code: String = String(data: data, encoding: .utf8)!
    let codeCells = code.components(separatedBy: "//cell")
    let nbName = codeCells[0].components(separatedBy: "\n")[1].components(separatedBy: "from ")[1]
    let nbData = try! Data(contentsOf: nbFname.url)
    var jsonData = try! JSONSerialization.jsonObject(with: nbData, options: .allowFragments) as! [String: Any]
    var cells = jsonData["cells"] as! [[String:Any]]
    for c in codeCells[1...] {
        var lines = c.components(separatedBy: "\n")
        let idx: Int = Int(lines[0])!
        var i = lines.count-1
        while lines[i].isEmpty { i -= 1}
        if i > 1 {
            for i in 1...(i-1) { lines[i].append("\n") }
        }
        lines[0] = "// export\n"
        cells[idx]["source"] = Array(lines[...i])
    }
    jsonData["cells"] = cells
    let outData = try! JSONSerialization.data(withJSONObject: jsonData, options: .prettyPrinted)  
    let jsonString = String(data: outData, encoding: .utf8)! 
    do { try jsonString.write(to: nbFname, encoding: .utf8) }
    catch { "Couldn't save notebook" }
}