Classifying handwritten digits using the SVD¶
Load data from the MNIST data set¶
In [33]:
using Flux
using Flux.Data.MNIST
In [34]:
labels = MNIST.labels();
images = MNIST.images();
In [35]:
using Pkg
Pkg.add("ImageShow")
Resolving package versions... Updating `~/.julia/environments/v1.0/Project.toml` [no changes] Updating `~/.julia/environments/v1.0/Manifest.toml` [no changes]
In [36]:
using ImageShow
In [100]:
size(images)
Out[100]:
(60000,)
In [102]:
size(images[8])
Out[102]:
(28, 28)
In [103]:
28*28
Out[103]:
784
Let's see what the images look like:
In [37]:
images[1:5]
Out[37]:
 |  |  |  |  |
(a vector displayed as a row to save space)
In [104]:
reshape(images[1:25], 5, 5)
Out[104]:
|  |  |  |  |
|  |  |  |  |
|  |  |  |  |
|  |  |  |  |
|  |  |  |  |
In [39]:
using Interact
In [107]:
images[17]
Out[107]:
In [108]:
labels[17]
Out[108]:
2
In [109]:
@manipulate for i in eachindex(images)
hbox(images[i], hskip(1em), string("label: ", labels[i]))
end
Out[109]:
In [ ]:
An image is just data¶
In [110]:
a = vec(images[2][10, :]);
In [111]:
b = round.(Int, 256*images[2][10, i].val for i in 1:28);
In [112]:
a[1]
Out[112]:
In [113]:
b[1]
Out[113]:
0
In [114]:
using Pkg
Pkg.add("Interact")
using Interact
Resolving package versions... Updating `~/.julia/environments/v1.0/Project.toml` [no changes] Updating `~/.julia/environments/v1.0/Manifest.toml` [no changes]
In [115]:
@manipulate for j in 1:28
a = vec(images[2][j, :])
b = round.(Int, 256*images[2][j, i].val for i in 1:28);
hbox([vbox(a[i], string(" ", b[i], " " )) for i in 1:28])
end
Out[115]:
In [48]:
@manipulate for j in 1:28
a = vec(images[2][j, :])
b = round.(Int, 256*images[2][j, i].val for i in 1:28);
hbox([vbox(a[i], string(b[i])) for i in 1:28])
end
Out[48]:
In [49]:
@manipulate for start in 1:28^2 - 28
a = vec(images[2][start:start+27])
b = round.(Int, 256*images[2][i].val for i in start:start+27);
hbox([vbox(a[i], hbox(hskip(0.5em), string(b[i]))) for i in 1:28])
end
Out[49]:
Collect all the data into a matrix¶
Let's collect all the data into a matrix. We take 50000 images as our training set:
In [116]:
training_size = 50000
training_images = images[1:training_size]
training_labels = labels[1:training_size];
For example, let's look at the images of 0s:
In [117]:
images_0 = training_images[training_labels .== 0];
images_0[1:100]
Out[117]:
| |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
(a vector displayed as a row to save space)
In [52]:
length(images_0)
Out[52]:
4932
Let's check that the numerical SVD gives the correct result for the 0 images:
In [118]:
function vectorize(image)
return round.(Int, 256*image[i].val for i in eachindex(image))
end
Out[118]:
vectorize (generic function with 1 method)
In [119]:
show(vectorize(images[2]))
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 85, 85, 85, 86, 85, 85, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 57, 199, 247, 253, 253, 253, 254, 253, 253, 200, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 179, 253, 254, 253, 231, 224, 146, 226, 250, 253, 253, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 166, 253, 253, 191, 112, 25, 0, 0, 0, 147, 253, 253, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 239, 254, 241, 63, 0, 0, 0, 0, 0, 0, 48, 230, 254, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 164, 239, 253, 234, 71, 0, 0, 0, 0, 0, 0, 0, 29, 216, 253, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 253, 254, 253, 209, 19, 0, 0, 0, 0, 0, 0, 0, 85, 253, 253, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 225, 253, 254, 180, 84, 28, 0, 0, 0, 0, 0, 0, 0, 179, 253, 234, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 228, 253, 253, 191, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 253, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 239, 254, 254, 254, 114, 75, 0, 0, 0, 0, 0, 0, 0, 0, 114, 254, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 253, 253, 253, 253, 254, 121, 0, 0, 0, 0, 0, 0, 0, 48, 239, 224, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 253, 240, 203, 253, 229, 21, 0, 0, 0, 0, 0, 0, 7, 166, 254, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 253, 234, 84, 96, 47, 0, 0, 0, 0, 0, 0, 7, 132, 253, 163, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 238, 253, 253, 190, 79, 0, 0, 0, 0, 0, 0, 136, 253, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 254, 254, 256, 254, 254, 254, 254, 256, 254, 254, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 122, 168, 169, 244, 253, 253, 253, 254, 253, 187, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 166, 196, 196, 197, 149, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
In [55]:
matrix_0s = reduce(hcat, vectorize.(images_0));
In [120]:
using LinearAlgebra
U, Σ, V = svd(matrix_0s)
Out[120]:
SVD{Float64,Float64,Array{Float64,2}}([-1.11022e-16 -8.32667e-17 … 0.263017 0.108898; 0.0 1.11022e-16 … 0.759198 -0.217008; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0], [1.53412e5, 53182.6, 43389.3, 33227.4, 26282.7, 23954.3, 22724.6, 21385.5, 19505.4, 19000.8 … 1.32435e-11, 1.32435e-11, 1.32435e-11, 1.32435e-11, 1.32435e-11, 1.32435e-11, 1.32435e-11, 1.32435e-11, 8.05781e-12, 1.62903e-12], [-0.0146172 -0.0155562 … -0.0130666 -0.00895334; -0.00861048 -0.00649652 … 0.00223801 0.00682512; … ; 0.0 0.649508 … 0.00523472 0.00408817; -0.9741 0.00190592 … 0.00231786 0.000850163])
In [121]:
U * Diagonal(Σ) * V' ≈ matrix_0s
Out[121]:
true
In [122]:
sum(Σ.^2) ≈ sum(matrix_0s.^2)
Out[122]:
true
In [124]:
using Plots
scatter(log10.(Σ))
Out[124]:
Let's look at the "average 0":
In [125]:
function normalize_image(image)
image2 = abs.(image)
m = maximum(image2)
image2 = image2 ./ m
return Gray.(reshape(image2, 28, 28))
end
Out[125]:
normalize_image (generic function with 1 method)
In [126]:
using Statistics
normalize_image(mean(matrix_0s, dims=2))
Out[126]:
Applying SVD to all images¶
In [127]:
big_image_matrix = reduce(hcat, vectorize.(training_images));
In [128]:
size(big_image_matrix)
Out[128]:
(784, 50000)
In [129]:
big_image_matrix
Out[129]:
784×50000 Array{Int64,2}:
0 0 0 0 0 0 0 0 0 0 0 0 0 … 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 … 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 … 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
⋮ ⋮ ⋮ ⋱ ⋮ ⋮
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 … 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 … 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Extract data for each digit into separate matrices¶
In [130]:
single_digits = [ big_image_matrix[:, training_labels .== i] for i in 0:9 ]
Out[130]:
10-element Array{Array{Int64,2},1}:
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
[0 0 … 0 0; 0 0 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]
In [131]:
Z = svd.(single_digits);
In [67]:
Us = first.(Z)
Out[67]:
10-element Array{Array{Float64,2},1}:
[-1.11022e-16 -8.32667e-17 … 0.263017 0.108898; 0.0 1.11022e-16 … 0.759198 -0.217008; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[3.33067e-16 2.22045e-16 … 0.12277 0.243441; 0.0 1.66533e-16 … 0.669715 0.0243274; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[2.22045e-16 -5.55112e-17 … 0.00295705 -0.607003; 0.0 1.94289e-16 … -0.518739 -0.190468; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-1.11022e-16 -5.55112e-17 … 0.217468 -0.467104; 0.0 -2.77556e-16 … 0.00632763 0.044892; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[1.11022e-16 0.0 … -0.0842133 -0.364336; 0.0 1.249e-16 … -0.00156071 0.0946388; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-3.33067e-16 -1.38778e-17 … -0.0427051 0.0213372; 0.0 -8.67362e-18 … 0.00325293 -0.281373; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-1.11022e-16 -6.245e-17 … 0.413816 -0.033716; 0.0 3.46945e-17 … 0.0728744 -0.229194; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[0.0 1.66533e-16 … -0.0205796 0.59204; 0.0 -1.66533e-16 … 0.0452746 -0.58315; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[2.22045e-16 5.55112e-17 … 0.272366 -0.0480318; 0.0 -2.77556e-16 … -0.522072 -0.261654; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[1.11022e-16 -1.94289e-16 … 0.371535 -0.288321; 0.0 2.22045e-16 … -0.532185 -0.089791; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
In [68]:
using Statistics
In [69]:
U[:, 1]
Out[69]:
784-element Array{Float64,1}:
-1.1102230246251565e-16
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
⋮
-4.4300954484669486e-7
-3.839416055338021e-6
0.0
0.0
0.0
-3.919018919865072e-6
-7.235111852058591e-6
0.0
0.0
0.0
0.0
0.0
In [70]:
norm(U[:, 1])
Out[70]:
1.0000000000000004
In [71]:
show(U[:, 1])
[-1.11022e-16, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -9.29612e-7, -3.66035e-6, 0.0, 0.0, 0.0, 0.0, 0.0, -2.14453e-6, -5.4438e-6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.93344e-6, -1.69527e-6, -8.79028e-6, 0.0, -5.53573e-6, -1.44148e-5, 0.0, 0.0, 0.0, 0.0, 0.0, -2.10329e-6, -3.75293e-6, -1.66112e-6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -2.21336e-6, -6.41111e-6, -1.55236e-5, -2.46301e-5, 0.0, -5.65138e-6, -2.41218e-6, -3.76726e-7, -9.35918e-6, -2.16633e-5, -2.17482e-5, -3.89486e-5, -1.94463e-5, -2.60095e-5, -4.87944e-5, -8.54256e-5, -0.000109922, -0.000149057, -0.000114416, -0.000112289, -6.30066e-5, -7.76734e-6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -7.63227e-7, -1.20028e-5, -1.64947e-5, -1.23905e-5, -5.31149e-5, -7.318e-5, -0.000105579, -0.000188515, -0.00035423, -0.000693559, -0.00124441, -0.00228696, -0.00393148, -0.00643971, -0.00901028, -0.011483, -0.0124281, -0.0115601, -0.00864732, -0.00506708, -0.00189222, -0.000378161, -7.24689e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.8247e-5, -3.41593e-5, -7.73528e-5, -0.000323475, -0.000503755, -0.000709758, -0.00136158, -0.0031869, -0.00602337, -0.010341, -0.0165778, -0.0240642, -0.0316041, -0.0379046, -0.0420571, -0.0420043, -0.0380997, -0.0300962, -0.0197713, -0.00957706, -0.00262734, -0.000280652, 0.0, 0.0, 0.0, 0.0, 0.0, -4.62144e-6, -3.31196e-5, -9.53852e-5, -0.000300368, -0.000741996, -0.0014655, -0.00313203, -0.00628747, -0.0116083, -0.0189902, -0.0286608, -0.039182, -0.0495202, -0.0585094, -0.0649597, -0.0680562, -0.0678249, -0.0641733, -0.0557702, -0.041597, -0.0244011, -0.00846455, -0.00105439, -1.77147e-6, 0.0, 0.0, 0.0, 0.0, -1.71758e-5, -3.29213e-5, -0.000249343, -0.000794528, -0.00208671, -0.00471621, -0.00930322, -0.0169816, -0.0270502, -0.0391318, -0.0517022, -0.0633458, -0.072956, -0.078419, -0.081074, -0.0816774, -0.0814694, -0.0803967, -0.0761313, -0.0649242, -0.0438516, -0.0192477, -0.0027257, -1.68309e-5, 0.0, 0.0, 0.0, 0.0, -3.96306e-6, -5.08748e-5, -0.000639662, -0.00212392, -0.00562587, -0.0116444, -0.021046, -0.0332099, -0.0468138, -0.0603366, -0.0713853, -0.0785998, -0.0816232, -0.0807239, -0.0775694, -0.0748041, -0.0754344, -0.0793643, -0.0842819, -0.0820606, -0.0640721, -0.0333849, -0.00554569, -8.41123e-5, 0.0, 0.0, 0.0, 0.0, 0.0, -0.000103407, -0.00161836, -0.00526386, -0.0122792, -0.023592, -0.0376687, -0.0528606, -0.0662507, -0.0761521, -0.0802371, -0.0787691, -0.0727135, -0.0644194, -0.0561241, -0.0517143, -0.0538872, -0.0646572, -0.0810112, -0.0907599, -0.0801031, -0.0479187, -0.00941692, -0.000175352, 0.0, 0.0, 0.0, 0.0, 0.0, -0.000209664, -0.00319526, -0.0108353, -0.0233625, -0.0393488, -0.0558426, -0.0699283, -0.0779686, -0.0794927, -0.0743699, -0.06381, -0.0507933, -0.0385716, -0.0296786, -0.0260057, -0.030622, -0.0454542, -0.0703136, -0.0911852, -0.0896554, -0.0597438, -0.0132559, -0.000237477, 0.0, 0.0, 0.0, -7.76936e-6, -5.54094e-6, -0.000477831, -0.00615264, -0.0196747, -0.0382017, -0.0567742, -0.0715031, -0.0793624, -0.0790663, -0.0706828, -0.0568964, -0.040957, -0.026535, -0.0166672, -0.011505, -0.0103315, -0.0160061, -0.0324903, -0.0613318, -0.0879917, -0.0928002, -0.0666086, -0.0157871, -0.000273446, 0.0, 0.0, 0.0, -1.52978e-5, -1.75684e-5, -0.000801455, -0.0110538, -0.031182, -0.0543092, -0.0718141, -0.0806081, -0.0797445, -0.0699647, -0.0540765, -0.0365375, -0.0211988, -0.0111318, -0.00604216, -0.00401295, -0.00473737, -0.0108131, -0.0285155, -0.0578675, -0.0857163, -0.0923542, -0.0670753, -0.016546, -0.000229425, 0.0, 0.0, 0.0, -6.26367e-6, -1.97159e-5, -0.00118142, -0.0170954, -0.044853, -0.0692063, -0.0817528, -0.0825871, -0.0732662, -0.0564384, -0.037645, -0.0210178, -0.00982994, -0.00464825, -0.0024225, -0.00197926, -0.00390463, -0.0118281, -0.0312013, -0.0600443, -0.085089, -0.089544, -0.0614098, -0.0152337, -0.000191247, 0.0, 0.0, 0.0, 0.0, -5.51044e-6, -0.00142415, -0.0229579, -0.0570214, -0.0804279, -0.0873724, -0.0799514, -0.0633885, -0.0428989, -0.0242524, -0.0113031, -0.00483948, -0.00255735, -0.00161092, -0.00222982, -0.00594399, -0.0172717, -0.0393505, -0.0662895, -0.0856561, -0.0830268, -0.0515994, -0.0122627, -0.000126578, 0.0, 0.0, 0.0, -4.66584e-6, -1.13046e-5, -0.00178888, -0.028588, -0.065624, -0.0873468, -0.0890781, -0.0759516, -0.0546552, -0.0321253, -0.015854, -0.00713906, -0.00338257, -0.00206523, -0.00222535, -0.00455262, -0.0119142, -0.0280872, -0.0508991, -0.0744074, -0.084823, -0.0722581, -0.0395586, -0.00875546, -6.77532e-5, 0.0, 0.0, 0.0, -1.28051e-5, -1.07611e-5, -0.0018051, -0.0305761, -0.0693246, -0.0894301, -0.089377, -0.0732191, -0.0490075, -0.0269034, -0.0130881, -0.00663814, -0.00389516, -0.00337284, -0.00533968, -0.011112, -0.0234095, -0.0425965, -0.0640269, -0.0794032, -0.0786719, -0.0580672, -0.0273984, -0.00543074, -4.98625e-5, 0.0, 0.0, 0.0, -6.06559e-6, -7.04193e-7, -0.0014824, -0.0289104, -0.0654539, -0.0875614, -0.0897999, -0.0740719, -0.0507841, -0.0296994, -0.0169159, -0.0107839, -0.00885046, -0.0102422, -0.0152158, -0.0248967, -0.0406006, -0.0588688, -0.0741359, -0.0785696, -0.0669438, -0.0423903, -0.0169052, -0.0029277, -1.45265e-5, 0.0, 0.0, 0.0, 0.0, -5.97485e-6, -0.00114682, -0.0238799, -0.0558715, -0.0804283, -0.0881724, -0.0789697, -0.0608886, -0.0421107, -0.0290834, -0.0233797, -0.0222899, -0.0251676, -0.0324303, -0.0444993, -0.0590743, -0.0713426, -0.0766738, -0.0691189, -0.0508247, -0.0273129, -0.00883312, -0.00122867, -2.48006e-6, 0.0, 0.0, 0.0, 0.0, -1.07705e-5, -0.000862209, -0.0170581, -0.0427619, -0.0665565, -0.0808641, -0.082096, -0.0735237, -0.0608269, -0.0505345, -0.0446263, -0.043336, -0.0471373, -0.05434, -0.0639132, -0.0718571, -0.0750034, -0.0685773, -0.0534073, -0.0335805, -0.0152007, -0.00381763, -0.000496152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.000495944, -0.0102147, -0.0274943, -0.0491865, -0.0663364, -0.0769399, -0.0787075, -0.075821, -0.0712549, -0.0673621, -0.0662786, -0.0682265, -0.0711643, -0.0736357, -0.0731428, -0.065965, -0.0524, -0.0356127, -0.0188329, -0.00690243, -0.00135797, -0.00016971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.000210135, -0.00501333, -0.0148127, -0.0298559, -0.0461895, -0.0604066, -0.0703424, -0.0756486, -0.0776478, -0.0772138, -0.0767381, -0.0755141, -0.0728516, -0.0682966, -0.0598518, -0.0476365, -0.0336641, -0.0195033, -0.00840084, -0.00255081, -0.000421394, -5.6797e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -9.74303e-5, -0.00196203, -0.00621762, -0.0144075, -0.0258992, -0.0380752, -0.0496125, -0.0588842, -0.0642859, -0.0665632, -0.0661114, -0.0632071, -0.0580823, -0.0499376, -0.0396573, -0.0282319, -0.0169218, -0.00811972, -0.00282778, -0.000739964, -0.000143172, -3.33553e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -3.43133e-5, -0.000431934, -0.00170332, -0.00460061, -0.0097588, -0.0165772, -0.024146, -0.0316038, -0.037841, -0.0415021, -0.0420442, -0.0396092, -0.0346017, -0.0275498, -0.0193135, -0.0116986, -0.0058467, -0.00233375, -0.000715049, -0.000184592, -7.54188e-5, -2.55503e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.61041e-5, -7.76787e-5, -0.000217645, -0.000609343, -0.00176425, -0.00340562, -0.00550206, -0.00810407, -0.0107946, -0.0131075, -0.0142463, -0.0140244, -0.012007, -0.00874721, -0.00541626, -0.00281527, -0.00108326, -0.000391725, -0.000137165, -5.31258e-5, -3.87769e-5, -1.30418e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.59217e-6, -1.59079e-5, -1.28356e-5, -3.7236e-5, -8.02628e-5, -4.67486e-5, -5.53104e-5, -0.000125478, -0.000214883, -0.000324334, -0.000424952, -0.000485672, -0.000427529, -0.000327826, -0.000219973, -0.000119802, -4.89548e-5, -1.32894e-5, -1.19655e-5, -1.46681e-5, -9.71593e-6, -8.12396e-6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.60957e-6, -8.55516e-7, 0.0, 0.0, 0.0, -2.11074e-5, -1.68472e-5, -1.93646e-6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.40393e-5, -3.99817e-5, -1.47388e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -5.67752e-6, -1.2055e-5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -4.4301e-7, -3.83942e-6, 0.0, 0.0, 0.0, -3.91902e-6, -7.23511e-6, 0.0, 0.0, 0.0, 0.0, 0.0]
Thresholding an image:
In [132]:
@manipulate for θ in 0.0:0.01:0.12
image = abs.(U[:, 1]) .> θ
Gray.(reshape(image, 28, 28))
end
Out[132]:
In [73]:
Us
Out[73]:
10-element Array{Array{Float64,2},1}:
[-1.11022e-16 -8.32667e-17 … 0.263017 0.108898; 0.0 1.11022e-16 … 0.759198 -0.217008; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[3.33067e-16 2.22045e-16 … 0.12277 0.243441; 0.0 1.66533e-16 … 0.669715 0.0243274; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[2.22045e-16 -5.55112e-17 … 0.00295705 -0.607003; 0.0 1.94289e-16 … -0.518739 -0.190468; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-1.11022e-16 -5.55112e-17 … 0.217468 -0.467104; 0.0 -2.77556e-16 … 0.00632763 0.044892; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[1.11022e-16 0.0 … -0.0842133 -0.364336; 0.0 1.249e-16 … -0.00156071 0.0946388; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-3.33067e-16 -1.38778e-17 … -0.0427051 0.0213372; 0.0 -8.67362e-18 … 0.00325293 -0.281373; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[-1.11022e-16 -6.245e-17 … 0.413816 -0.033716; 0.0 3.46945e-17 … 0.0728744 -0.229194; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[0.0 1.66533e-16 … -0.0205796 0.59204; 0.0 -1.66533e-16 … 0.0452746 -0.58315; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[2.22045e-16 5.55112e-17 … 0.272366 -0.0480318; 0.0 -2.77556e-16 … -0.522072 -0.261654; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
[1.11022e-16 -1.94289e-16 … 0.371535 -0.288321; 0.0 2.22045e-16 … -0.532185 -0.089791; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
Let's look at all the singular vectors for all images:
In [74]:
@manipulate for digit in slider(0:9, value=0), which_sing_vec in slider(1:10, value=1)
image = Us[digit+1][:,which_sing_vec]
normalize_image(image)
end
Out[74]:
Predicting labels from images of digits¶
Let's use the SVD to try to predict a label for an image that we haven't seen yet:
In [75]:
image = images[training_size + 1] # was not included in the training
Out[75]:
In [88]:
M = [U[:,1:3] for U in Us] # first n sing vecs for each digit
Out[88]:
10-element Array{Array{Float64,2},1}:
[-1.11022e-16 -8.32667e-17 -1.80411e-16; 0.0 1.11022e-16 -2.77556e-17; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[3.33067e-16 2.22045e-16 -1.38778e-16; 0.0 1.66533e-16 1.11022e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[2.22045e-16 -5.55112e-17 5.55112e-17; 0.0 1.94289e-16 -5.55112e-17; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[-1.11022e-16 -5.55112e-17 5.55112e-17; 0.0 -2.77556e-16 2.22045e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[1.11022e-16 0.0 1.94289e-16; 0.0 1.249e-16 -1.66533e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[-3.33067e-16 -1.38778e-17 5.55112e-17; 0.0 -8.67362e-18 -1.94289e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[-1.11022e-16 -6.245e-17 -6.93889e-17; 0.0 3.46945e-17 -1.94289e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[0.0 1.66533e-16 0.0; 0.0 -1.66533e-16 0.0; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[2.22045e-16 5.55112e-17 -8.32667e-17; 0.0 -2.77556e-16 2.498e-16; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
[1.11022e-16 -1.94289e-16 1.38778e-16; 0.0 2.22045e-16 -8.32667e-17; … ; 0.0 0.0 0.0; 0.0 0.0 0.0]
In [89]:
first_sing_vecs = reduce(hcat, [U[:,1] for U in Us])
Out[89]:
784×10 Array{Float64,2}:
-1.11022e-16 3.33067e-16 2.22045e-16 … 2.22045e-16 1.11022e-16
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 … 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 … 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
⋮ ⋱
-4.4301e-7 0.0 -3.147e-5 0.0 0.0
-3.83942e-6 -8.10781e-7 -2.42268e-5 0.0 0.0
0.0 -3.49319e-5 -3.07186e-5 0.0 0.0
0.0 -1.94591e-5 -1.85176e-5 … 0.0 0.0
0.0 0.0 0.0 0.0 0.0
-3.91902e-6 0.0 0.0 0.0 0.0
-7.23511e-6 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 … 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
0.0 0.0 0.0 0.0 0.0
In [90]:
my_softmax(v) = exp.(v) / sum(exp.(v))
Out[90]:
my_softmax (generic function with 1 method)
In [91]:
v = abs.(first_sing_vecs' * vec(Float64.(image)))
my_softmax(v)
Out[91]:
10-element Array{Float64,1}:
0.03382072710940378
0.15392804472033755
0.12316884381500334
0.2824501000116202
0.027879155990251033
0.12441004437229447
0.0474266712208733
0.02423256810297591
0.1465310863057624
0.0361527583514779
In [92]:
M = [U[:, 1:1] for U in Us] # first n sing vecs for each digit
my_softmax(norm.([MM' * vec(Float64.(image)) for MM in M]))
Out[92]:
10-element Array{Float64,1}:
0.033820727109403786
0.15392804472033758
0.12316884381500348
0.28245010001161996
0.027879155990251016
0.1244100443722945
0.04742667122087331
0.024232568102975912
0.14653108630576242
0.03615275835147791
In [93]:
@manipulate for num_sing_vecs in 1:784
M = [U[:, 1:num_sing_vecs] for U in Us] # first n sing vecs for each digit
v = my_softmax(norm.([MM' * vec(Float64.(image)) for MM in M]))
hbox(v, plot(0:9, v, m=:o), findmax(v))
end
Out[93]:
In [96]:
data = []
for num_sing_vecs in 1:15
M = [U[:, 1:num_sing_vecs] for U in Us] # first n sing vecs for each digit
v = my_softmax(norm.([MM' * vec(Float64.(image)) for MM in M]))
push!(data, v[4])
end
plot(data, m=:o)
scatter!([argmax(data)], [maximum(data)])
Out[96]:
In [97]:
argmax(data)
Out[97]:
6
In [98]:
softmax(norm.([MM' * vec(Float64.(image)) for MM in M]))
Out[98]:
10-element Array{Float64,1}:
0.06244873229934407
0.08153335605992665
0.14844813863333253
0.3010347736526581
0.0248050897247611
0.11181027499468853
0.06919557999052471
0.025800589190491042
0.13536138735349518
0.03956207810077807
In [99]:
vec(images[1])
Out[99]:
In [ ]: