%install-location $cwd/swift-install
%install '.package(path: "$cwd/FastaiNotebook_09_optimizer")' FastaiNotebook_09_optimizer
Installing packages: .package(path: "/home/sgugger/git/course-v3/nbs/swift/FastaiNotebook_09_optimizer") FastaiNotebook_09_optimizer With SwiftPM flags: [] Working in: /tmp/tmp5t4bwpzq/swift-install warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) Updating https://github.com/mxcl/Path.swift Updating https://github.com/saeta/Just Updating https://github.com/latenitesoft/NotebookExport Completed resolution in 2.36s warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[1/15] Compiling FastaiNotebook_09_optimizer 08a_heterogeneous_dictionary.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [2/15] Compiling FastaiNotebook_09_optimizer 01_matmul.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [3/15] Compiling FastaiNotebook_09_optimizer 03_minibatch_training.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [4/15] Compiling FastaiNotebook_09_optimizer 02_fully_connected.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [5/15] Compiling FastaiNotebook_09_optimizer 05b_early_stopping.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [6/15] Compiling FastaiNotebook_09_optimizer 06_cuda.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [7/15] Compiling FastaiNotebook_09_optimizer 05_anneal.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [8/15] Compiling FastaiNotebook_09_optimizer 02a_why_sqrt5.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [9/15] Compiling FastaiNotebook_09_optimizer 00_load_data.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [10/15] Compiling FastaiNotebook_09_optimizer 08_data_block.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [11/15] Compiling FastaiNotebook_09_optimizer 04_callbacks.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [12/15] Compiling FastaiNotebook_09_optimizer 09_optimizer.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [13/15] Compiling FastaiNotebook_09_optimizer 01a_fastai_layers.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [14/15] Compiling FastaiNotebook_09_optimizer 07_batchnorm.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [15/16] Merging module FastaiNotebook_09_optimizer /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[16/17] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) [17/18] Merging module jupyterInstalledPackages /home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift) /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc) /home/sgugger/swift/usr/bin/swift-autolink-extract: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift-autolink-extract) [18/18] Linking libjupyterInstalledPackages.so Initializing Swift... Installation complete!
// export
import Path
import TensorFlow
import FastaiNotebook_09_optimizer
%include "EnableIPythonDisplay.swift"
IPythonDisplay.shell.enable_matplotlib("inline")
('inline', 'module://ipykernel.pylab.backend_inline')
//TODO: switch to imagenette when possible to train
let data = mnistDataBunch(flat: true)
let (n,m) = (60000,784)
let c = 10
let nHid = 50
func modelInit() -> BasicModel {return BasicModel(nIn: m, nHid: nHid, nOut: c)}
let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: sgdOpt(lr: 0.1), modelInit: modelInit)
let recorder = learner.makeDefaultDelegates(metrics: [accuracy])
learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))
learner.fit(1)
Epoch 0: [0.35363615, 0.9073]
//export
extension RandomDistribution {
// Returns a batch of samples.
func next<G: RandomNumberGenerator>(
_ count: Int, using generator: inout G
) -> [Sample] {
var result: [Sample] = []
for _ in 0..<count {
result.append(next(using: &generator))
}
return result
}
// Returns a batch of samples, using the global Threefry RNG.
func next(_ count: Int) -> [Sample] {
return next(count, using: &ThreefryRandomNumberGenerator.global)
}
}
Mixup requires one-hot encoded targets since we don't have a loss function with no reduction.
//export
extension Learner {
public class MixupDelegate: Delegate {
private var distribution: BetaDistribution
public init(alpha: Float = 0.4){
distribution = BetaDistribution(alpha: alpha, beta: alpha)
}
override public func batchWillStart(learner: Learner) {
if let xb = learner.currentInput {
if let yb = learner.currentTarget as? Tensor<Float>{
var lambda = Tensor<Float>(distribution.next(Int(yb.shape[0])))
lambda = max(lambda, 1-lambda)
let shuffle = Raw.randomShuffle(value: Tensor<Int32>(0..<Int32(yb.shape[0])))
let xba = Raw.gather(params: xb, indices: shuffle)
let yba = Raw.gather(params: yb, indices: shuffle)
lambda = lambda.expandingShape(at: 1)
learner.currentInput = lambda * xb + (1-lambda) * xba
learner.currentTarget = (lambda * yb + (1-lambda) * yba) as? Label
}
}
}
}
public func makeMixupDelegate(alpha: Float = 0.4) -> MixupDelegate {
return MixupDelegate(alpha: alpha)
}
}
let (n,m) = (60000,784)
let c = 10
let nHid = 50
We need to one-hot encode the targets
var train1 = data.train.innerDs.map { DataBatch<TF,TF>(xb: $0.xb,
yb: Raw.oneHot(indices: $0.yb, depth: TI(10), onValue: TF(1), offValue: TF(0))) }
var valid1 = data.valid.innerDs.map { DataBatch<TF,TF>(xb: $0.xb,
yb: Raw.oneHot(indices: $0.yb, depth: TI(10), onValue: TF(1), offValue: TF(0))) }
let data1 = DataBunch(train: train1, valid: valid1, trainLen: data.train.dsCount,
validLen: data.valid.dsCount, bs: data.train.bs)
func modelInit() -> BasicModel {return BasicModel(nIn: m, nHid: nHid, nOut: c)}
func accuracyFloat(_ out: TF, _ targ: TF) -> TF {
return TF(out.argmax(squeezingAxis: 1) .== targ.argmax(squeezingAxis: 1)).mean()
}
let learner = Learner(data: data1, lossFunc: softmaxCrossEntropy, optFunc: sgdOpt(lr: 0.1), modelInit: modelInit)
let recorder = learner.makeRecorder()
learner.delegates = [learner.makeTrainEvalDelegate(), learner.makeShowProgress(),
learner.makeAvgMetric(metrics: [accuracyFloat]), recorder,
learner.makeMixupDelegate(alpha: 0.2)]
learner.fit(2)
Epoch 0: [0.5300334, 0.9163]
Epoch 1: [0.507523, 0.926]
//export
@differentiable(wrt: out)
public func labelSmoothingCrossEntropy(_ out: TF, _ targ: TI, ε: Float = 0.1) -> TF {
let c = out.shape[1]
let loss = softmaxCrossEntropy(logits: out, labels: targ)
let logPreds = logSoftmax(out)
return (1-ε) * loss - (ε / Float(c)) * logPreds.mean()
}
@differentiable(wrt: out)
func lossFunc(_ out: TF, _ targ: TI) -> TF { return labelSmoothingCrossEntropy(out, targ, ε: 0.1) }
let learner = Learner(data: data, lossFunc: lossFunc, optFunc: sgdOpt(lr: 0.1), modelInit: modelInit)
let recorder = learner.makeDefaultDelegates(metrics: [accuracy])
learner.delegates.append(learner.makeNormalize(mean: mnistStats.mean, std: mnistStats.std))
learner.fit(2)
Epoch 0: [0.2789237, 0.9376]
Epoch 1: [0.28466254, 0.9394]
import NotebookExport
let exporter = NotebookExport(Path.cwd/"10_mixup_ls.ipynb")
print(exporter.export(usingPrefix: "FastaiNotebook_"))
success