In [1]:
ENV["MOCHA_CUDA_DEVICE"]=1
ENV["MOCHA_USE_NATIVE_EXT"] = "false"
ENV["MOCHA_USE_CUDA"] = "true"
Out[1]:
"true"
In [2]:
using Mocha
using UnicodePlots
Configuring Mocha...
WARNING: Method definition info(Any...) in module Base at util.jl:334 overwritten in module Logging at /home/uniwa/students2/students/20361362/linux/.julia/v0.4/Logging/src/Logging.jl:61.
WARNING: Method definition warn(Any...) in module Base at util.jl:364 overwritten in module Logging at /home/uniwa/students2/students/20361362/linux/.julia/v0.4/Logging/src/Logging.jl:61.
 * CUDA       enabled [DEV=1] (MOCHA_USE_CUDA environment variable detected)
 * Native Ext disabled by default
Mocha configured, continue loading module...
WARNING: parseint(s) is deprecated, use parse(Int,s) instead.
 in depwarn at deprecated.jl:73
 in parseint at deprecated.jl:50
 in parseEnvInt at /home/uniwa/students2/students/20361362/linux/.julia/v0.4/Mocha/src/config.jl:12
 in include at ./boot.jl:261
 in include_from_node1 at ./loading.jl:320
 in include at ./boot.jl:261
 in include_from_node1 at ./loading.jl:320
 in require at ./loading.jl:259
 in include_string at loading.jl:282
 in execute_request_0x535c5df2 at /home/uniwa/students2/students/20361362/linux/.julia/v0.4/IJulia/src/execute_request.jl:183
 in eventloop at /home/uniwa/students2/students/20361362/linux/.julia/v0.4/IJulia/src/IJulia.jl:143
 in anonymous at task.jl:447
while loading /home/uniwa/students2/students/20361362/linux/.julia/v0.4/Mocha/src/config.jl, in expression starting on line 27
DefaultBackend = Mocha.GPUBackend
In [4]:
function draw_line!(rise,run, out=zeros(Float64,(10,10)))
    len = minimum(size(out))÷2-1
    if abs(run)>abs(rise)
        range = run>0 ? (0:1:len) : (0:-1:-len)
        for x in range
            y = -trunc(Int64,(rise/run)*x)
            out[end÷2+y,end÷2+x]=1 #Julia is backwards for column first
        end
    else
        range = rise>0 ? (0:1:len) : (0:-1:-len)
        for y in range
            x = -trunc(Int64,(run/rise)*y)
            out[end÷2+y,end÷2+x]=1 #Julia is backwards for column first
        end
    end

    return out
end
Out[4]:
draw_line! (generic function with 2 methods)
In [5]:
function show_results_summary(pred, actual)
    errors = abs(pred.-actual)
    mean_error = mean(errors)
    @show mean_error
    @show std(errors)
    @show minimum(errors)
    @show median(errors)
    @show maximum(errors)
    
    accurancy= mean([==y for (,y) in zip(pred, actual)])
    accurancy_todefault_epsilon= mean([ŷ≈y for (,y) in zip(pred, actual)])
    accurancy_to_point001= mean([isapprox(,y, atol=0.001) for (,y) in zip(pred, actual)])
    accurancy_to_point01= mean([isapprox(,y, atol=0.01) for (,y) in zip(pred, actual)])
    accurancy_to_point1= mean([isapprox(,y, atol=0.1) for (,y) in zip(pred, actual)])
    @show accurancy
    @show accurancy_todefault_epsilon
    @show accurancy_to_point001
    @show accurancy_to_point01
    @show accurancy_to_point1
    nothing
end
Out[5]:
show_results_summary (generic function with 1 method)
In [6]:
function random_angle_rise_run()
    quadrents=[(1,1,0),(1,-1,π),(-1,-1,-π),(-1,1,0)]
    quadrent=rand(quadrents)
    angle = rand(Float32)*π/2
    if angle==π/2
        rise,run = (1,0)
    else
        rise= num(Rational(tan(angle)))
        run = den(Rational(tan(angle)))
    end
    rise*=quadrent[1]
    run*=quadrent[2]
    angle=angle*quadrent[1]*quadrent[2]+quadrent[3]

    @assert(isapprox(angle,atan2(rise,run), atol=0.0001), "$angle != $(atan2(rise,run))")
    angle,rise,run
end

random_angle_rise_run()
Out[6]:
(-0.21413377f0,-3648505,16777216)
In [7]:
function make_data(n_cases, img_width, img_height)
    X = Matrix{Float64}(img_width*img_height,n_cases)
    Y_angles = Vector{Float64}(n_cases)
    
    for ii in 1:n_cases
        angle,rise,run = random_angle_rise_run()        
        img = draw_line!(rise,run, zeros(Float64,(img_height,img_width)))
        X[:,ii] = img[:]
        Y_angles[ii]=angle
    end
    X,Y_angles'
end
Out[7]:
make_data (generic function with 1 method)
In [8]:
#Define various encoders for angles
encode_scaled(Y_angles)=((Y_angles+π)/2π)
decode_scaled(Y_scaled)=(Y_scaled[:]*2π-π)'

encode_cossin(Y_angles) = [cos(Y_angles); sin(Y_angles)]
decode_cossin(Y_cossin) = atan2(Y_cossin[2,:], Y_cossin[1,:])'

function encode_binned(Y_angles,n_bins = 500)
    Ys = encode_scaled(Y_angles)
    ret = zeros(n_bins, length(Ys))
    for (ii,y) in enumerate(Ys)
        idx=1+round(Int64, y*(n_bins-1))
        ret[idx,ii]=1
    end
    ret
end

function decode_binned(Y_codes,n_bins = 500)
    ret = Vector{Float64}(size(Y_codes,2))
    for ii in 1:size(Y_codes,2)
        idx = indmax(Y_codes[:,ii])[1]
        ret[ii]=(idx.-1)./(n_bins-1)
    end
    decode_scaled(ret)
end
Out[8]:
decode_binned (generic function with 2 methods)
In [9]:
function eval_angular_encoding(encode, decode; img_width=101,img_height=101, cases=1000, max_iter=1000, softmax=false)
##Prep Data
    xs,ys_angles = make_data(cases,img_width,img_height);
    xs_test, ys_test_angles = make_data(cases,img_width,img_height); #Since it is syntetic data we can generate a testset as largeas the training set
    
    ys_code = encode(ys_angles)
    ys_test_code = encode(ys_test_angles)
    #@assert(softmax || decode(ys_code)≈ys_angles) #if it is softmax, then it is coded in a form that is not conging to decode perfect
    #Train Net
    @show size(ys_code)
    
    data_layer = MemoryDataLayer(batch_size=50, data=Array[xs, ys_code], tops=[:data, :label])
    ip_layer   = InnerProductLayer(name="ip", output_dim=500, tops=[:ip], bottoms=[:data], neuron=Neurons.Sigmoid())
    label_dim = size(data_layer.data[data_layer.tops.==:label][1],1)
    @show label_dim
    pred_layer = pred_layer = InnerProductLayer(name="pred", output_dim=label_dim, tops=[:pred], bottoms=[:ip])
    loss_layer = SquareLossLayer(name="sqloss", bottoms=[:pred, :label])
    
    if softmax #overwrite it
        #pred_layer = SoftmaxLayer(name="pred",tops=[:pred], bottoms=[:ip])
        #loss_layer = MultinomialLogisticLossLayer(bottoms=[:pred, :label])
        loss_layer = SoftlabelSoftmaxLossLayer(name="smloss", bottoms=[:pred, :label])
    end
    backend = DefaultBackend()
    init(backend)

    net = Net("Training", backend, [data_layer,ip_layer,pred_layer,loss_layer])
    method = SGD()
    params = make_solver_parameters(method, max_iter=max_iter, lr_policy=LRPolicy.Fixed(0.01), mom_policy=MomPolicy.Fixed(0.9))
    solver = Solver(method, params)
    add_coffee_break(solver, TrainingSummary(), every_n_iter=100)
    solve(solver, net) #Train the net on the training data

    #Evaluate Net
    println("-------Evaluating on Test-------")
    data_layer_test = MemoryDataLayer(batch_size=cases, data=Array[xs_test, ys_test_code], tops=[:data, :label])
    test_net = Net("Test", backend, [data_layer_test,ip_layer,pred_layer])

    forward_epoch(test_net) #Run the net on the test data 
    pred_code = to_array(test_net.output_blobs[:pred])
    pred=decode(pred_code)
    
    
    #shutdown(backend)
    (pred,ys_test_angles)
end
Out[9]:
eval_angular_encoding (generic function with 1 method)
In [9]:
@time pred_binned, actual_binned = eval_angular_encoding(encode_binned, decode_binned; softmax=true)
size(ys_code) = (500,1000)
label_dim = 500
30-Jun 13:22:03:INFO:root:Constructing net Training on Mocha.CPUBackend...
30-Jun 13:22:03:INFO:root:Topological sorting 4 layers...
30-Jun 13:22:03:INFO:root:Setup layers...
30-Jun 13:22:03:INFO:root:Network constructed!
30-Jun 13:22:03:DEBUG:root:#DEBUG Checking network topology for back-propagation
30-Jun 13:22:04:DEBUG:root:Init network Training
30-Jun 13:22:04:DEBUG:root:Init parameter weight for layer ip
30-Jun 13:22:04:DEBUG:root:Init parameter bias for layer ip
30-Jun 13:22:04:DEBUG:root:Init parameter weight for layer pred
30-Jun 13:22:04:DEBUG:root:Init parameter bias for layer pred
30-Jun 13:22:04:DEBUG:root:#DEBUG Initializing coffee breaks
30-Jun 13:22:04:INFO:root: TRAIN iter=000000 obj_val=6.32109459
30-Jun 13:22:05:DEBUG:root:#DEBUG Entering solver loop
30-Jun 13:22:10:INFO:root: TRAIN iter=000100 obj_val=5.90896460
30-Jun 13:22:16:INFO:root: TRAIN iter=000200 obj_val=5.81646692
30-Jun 13:22:22:INFO:root: TRAIN iter=000300 obj_val=5.72751290
30-Jun 13:22:27:INFO:root: TRAIN iter=000400 obj_val=5.63788346
30-Jun 13:22:33:INFO:root: TRAIN iter=000500 obj_val=5.54463511
30-Jun 13:22:39:INFO:root: TRAIN iter=000600 obj_val=5.44539300
30-Jun 13:22:44:INFO:root: TRAIN iter=000700 obj_val=5.33807684
30-Jun 13:22:50:INFO:root: TRAIN iter=000800 obj_val=5.22093573
30-Jun 13:22:56:INFO:root: TRAIN iter=000900 obj_val=5.09273098
30-Jun 13:23:01:INFO:root: TRAIN iter=001000 obj_val=4.95288227
-------Evaluating on Test-------
30-Jun 13:23:01:INFO:root:Constructing net Test on Mocha.CPUBackend...
30-Jun 13:23:01:INFO:root:Topological sorting 3 layers...
30-Jun 13:23:01:INFO:root:Setup layers...
30-Jun 13:23:01:DEBUG:root:InnerProductLayer(ip): sharing weights and bias
30-Jun 13:23:01:DEBUG:root:InnerProductLayer(pred): sharing weights and bias
30-Jun 13:23:01:INFO:root:Network constructed!
 61.688237 seconds (381.43 M allocations: 12.276 GB, 2.01% gc time)
Out[9]:
(
1x1000 Array{Float64,2}:
 -2.31055  1.51728  2.94013  -2.33573  …  -0.875113  0.295902  0.0188873,

1x1000 Array{Float64,2}:
 -2.29266  1.54422  2.96279  -2.35595  …  -0.867362  0.29379  -0.0191173)
In [12]:
#Time CPU_with_OpenMP = 61.688237 seconds (381.43 M allocations: 12.276 GB, 2.01% gc time)
#Time CPU = 58.084438 seconds (381.30 M allocations: 12.270 GB, 2.02% gc time)
#Time GTX960 = 27.612926 seconds (825.41 k allocations: 4.685 GB, 1.43% gc time)
#Time Tesla K40 = 13.499364 seconds (9.65 M allocations: 5.052 GB, 2.34% gc time)

println(" = 500 bins; 1,000 Iter ")
show_results_summary(pred_binned, actual_binned)
 = 500 bins; 1,000 Iter 
mean_error = 0.4711263341561127
std(errors) = 1.1881991421080138
minimum(errors) = 1.827016860245223e-6
median(errors) = 0.05121685329339731
maximum(errors) = 6.074969396458127
accurancy = 0.0
accurancy_todefault_epsilon = 0.0
accurancy_to_point001 = 0.021
accurancy_to_point01 = 0.219
accurancy_to_point1 = 0.596
In [13]:
@time pred_scaled_1k, actual_scaled_1k = eval_angular_encoding(encode_scaled, decode_scaled)
size(ys_code) = (1,1000)
label_dim = 1
30-Jun 13:23:40:INFO:root:Constructing net Training on Mocha.CPUBackend...
30-Jun 13:23:40:INFO:root:Topological sorting 4 layers...
30-Jun 13:23:40:INFO:root:Setup layers...
30-Jun 13:23:40:INFO:root:Network constructed!
30-Jun 13:23:40:DEBUG:root:#DEBUG Checking network topology for back-propagation
30-Jun 13:23:40:DEBUG:root:Init network Training
30-Jun 13:23:40:DEBUG:root:Init parameter weight for layer ip
30-Jun 13:23:40:DEBUG:root:Init parameter bias for layer ip
30-Jun 13:23:40:DEBUG:root:Init parameter weight for layer pred
30-Jun 13:23:40:DEBUG:root:Init parameter bias for layer pred
30-Jun 13:23:41:DEBUG:root:#DEBUG Initializing coffee breaks
30-Jun 13:23:41:INFO:root: TRAIN iter=000000 obj_val=0.05882566
30-Jun 13:23:41:DEBUG:root:#DEBUG Entering solver loop
30-Jun 13:23:44:INFO:root: TRAIN iter=000100 obj_val=0.04859836
30-Jun 13:23:47:INFO:root: TRAIN iter=000200 obj_val=0.01595876
30-Jun 13:23:50:INFO:root: TRAIN iter=000300 obj_val=0.00866565
30-Jun 13:23:53:INFO:root: TRAIN iter=000400 obj_val=0.00701754
30-Jun 13:23:56:INFO:root: TRAIN iter=000500 obj_val=0.00648604
30-Jun 13:23:59:INFO:root: TRAIN iter=000600 obj_val=0.00631583
30-Jun 13:24:02:INFO:root: TRAIN iter=000700 obj_val=0.00628356
30-Jun 13:24:05:INFO:root: TRAIN iter=000800 obj_val=0.00629137
30-Jun 13:24:08:INFO:root: TRAIN iter=000900 obj_val=0.00630042
30-Jun 13:24:10:INFO:root: TRAIN iter=001000 obj_val=0.00629985
-------Evaluating on Test-------
30-Jun 13:24:10:INFO:root:Constructing net Test on Mocha.CPUBackend...
30-Jun 13:24:10:INFO:root:Topological sorting 3 layers...
30-Jun 13:24:10:INFO:root:Setup layers...
30-Jun 13:24:10:DEBUG:root:InnerProductLayer(ip): sharing weights and bias
30-Jun 13:24:10:DEBUG:root:InnerProductLayer(pred): sharing weights and bias
30-Jun 13:24:10:INFO:root:Network constructed!
 30.487064 seconds (955.99 k allocations: 4.836 GB, 1.64% gc time)
Out[13]:
(
1x1000 Array{Float64,2}:
 -1.23567  -0.496112  -1.96872  -1.93343  …  -2.10888  1.87745  -0.87746,

1x1000 Array{Float64,2}:
 -1.3738  -0.618581  -2.2956  -1.82246  …  -2.51013  1.80805  -1.02263)
In [14]:
#Time CPU = 37.183113 seconds (957.24 k allocations: 4.836 GB, 1.13% gc time)
#Time CPU_with_OpenMP =  30.487064 seconds (955.99 k allocations: 4.836 GB, 1.64% gc time)
#Time GTX960 = 26.231840 seconds (1.19 M allocations: 4.491 GB, 1.60% gc time)
#time Tesla K40 = 7.613501 seconds (1.06 M allocations: 4.486 GB, 4.11% gc time)
println(" = scaled to 0-1; 1,000 iter ")
show_results_summary(pred_scaled_1k, actual_scaled_1k)
 = scaled to 0-1; 1,000 iter 
mean_error = 0.22252844859352533
std(errors) = 0.4878383766698216
minimum(errors) = 1.8213413039980253e-5
median(errors) = 0.12910339816685834
maximum(errors) = 4.928355124833342
accurancy = 0.0
accurancy_todefault_epsilon = 0.0
accurancy_to_point001 = 0.003
accurancy_to_point01 = 0.042
accurancy_to_point1 = 0.359
In [15]:
@time pred_cossin_1k, actual_cossin_1k = eval_angular_encoding(encode_cossin, decode_cossin)
size(ys_code) = (2,1000)
label_dim = 2
30-Jun 13:24:27:INFO:root:Constructing net Training on Mocha.CPUBackend...
30-Jun 13:24:27:INFO:root:Topological sorting 4 layers...
30-Jun 13:24:27:INFO:root:Setup layers...
30-Jun 13:24:27:INFO:root:Network constructed!
30-Jun 13:24:27:DEBUG:root:#DEBUG Checking network topology for back-propagation
30-Jun 13:24:27:DEBUG:root:Init network Training
30-Jun 13:24:27:DEBUG:root:Init parameter weight for layer ip
30-Jun 13:24:28:DEBUG:root:Init parameter bias for layer ip
30-Jun 13:24:28:DEBUG:root:Init parameter weight for layer pred
30-Jun 13:24:28:DEBUG:root:Init parameter bias for layer pred
30-Jun 13:24:28:DEBUG:root:#DEBUG Initializing coffee breaks
30-Jun 13:24:28:INFO:root: TRAIN iter=000000 obj_val=0.56657965
30-Jun 13:24:28:DEBUG:root:#DEBUG Entering solver loop
30-Jun 13:24:31:INFO:root: TRAIN iter=000100 obj_val=0.11403108
30-Jun 13:24:34:INFO:root: TRAIN iter=000200 obj_val=0.01661780
30-Jun 13:24:36:INFO:root: TRAIN iter=000300 obj_val=0.00841437
30-Jun 13:24:39:INFO:root: TRAIN iter=000400 obj_val=0.00383931
30-Jun 13:24:43:INFO:root: TRAIN iter=000500 obj_val=0.00159342
30-Jun 13:24:47:INFO:root: TRAIN iter=000600 obj_val=0.00071809
30-Jun 13:24:50:INFO:root: TRAIN iter=000700 obj_val=0.00040033
30-Jun 13:24:54:INFO:root: TRAIN iter=000800 obj_val=0.00027489
30-Jun 13:24:57:INFO:root: TRAIN iter=000900 obj_val=0.00021550
30-Jun 13:25:01:INFO:root: TRAIN iter=001000 obj_val=0.00018167
-------Evaluating on Test-------
30-Jun 13:25:01:INFO:root:Constructing net Test on Mocha.CPUBackend...
30-Jun 13:25:01:INFO:root:Topological sorting 3 layers...
30-Jun 13:25:01:INFO:root:Setup layers...
30-Jun 13:25:01:DEBUG:root:InnerProductLayer(ip): sharing weights and bias
30-Jun 13:25:01:DEBUG:root:InnerProductLayer(pred): sharing weights and bias
30-Jun 13:25:01:INFO:root:Network constructed!
 33.969061 seconds (890.57 k allocations: 4.834 GB, 1.55% gc time)
Out[15]:
(
1000x1 Array{Float64,2}:
  0.506251
 -0.706846
 -1.69701 
  1.69604 
 -0.359154
  1.30174 
  2.69727 
  1.48109 
 -1.21916 
 -2.45308 
 -2.06127 
  1.08498 
  0.281024
  ⋮       
 -1.97779 
  0.490637
 -1.91589 
  2.60628 
  2.00374 
  0.998328
  1.49201 
  0.705631
 -1.23383 
  1.74181 
  2.29165 
 -3.13403 ,

1x1000 Array{Float64,2}:
 0.52814  -0.770912  -1.76656  1.7068  …  1.79144  2.31057  -3.12977)
In [17]:
#Time CPU =  32.189825 seconds (890.98 k allocations: 4.834 GB, 1.35% gc time)
#Time CPU_with_OpenMP =  33.969061 seconds (890.57 k allocations: 4.834 GB, 1.55% gc time)
#Time GTX960 = 26.175433 seconds (830.20 k allocations: 4.475 GB, 1.49% gc time)
#Time Tesla K40 =  7.570600 seconds (964.25 k allocations: 4.482 GB, 3.57% gc time)
println(" = Sin/Cos; 1,000 iter ")
show_results_summary(pred_cossin_1k, actual_cossin_1k)
 = Sin/Cos; 1,000 iter 
mean_error = 2.0999147180485234
std(errors) = 1.4859679090402775
minimum(errors) = 9.664462125025253e-7
median(errors) = 1.8440767072115745
maximum(errors) = 6.259330736645284
accurancy = 0.0
accurancy_todefault_epsilon = 0.0
accurancy_to_point001 = 0.037
accurancy_to_point01 = 0.371
accurancy_to_point1 = 0.989
In [18]:
@time pred_scaled_10k, actual_scaled_10k = eval_angular_encoding(encode_scaled, decode_scaled, max_iter=10_000)
size(ys_code) = (1,1000)
label_dim = 1
30-Jun 13:25:18:INFO:root:Constructing net Training on Mocha.CPUBackend...
30-Jun 13:25:18:INFO:root:Topological sorting 4 layers...
30-Jun 13:25:18:INFO:root:Setup layers...
30-Jun 13:25:18:INFO:root:Network constructed!
30-Jun 13:25:18:DEBUG:root:#DEBUG Checking network topology for back-propagation
30-Jun 13:25:18:DEBUG:root:Init network Training
30-Jun 13:25:18:DEBUG:root:Init parameter weight for layer ip
30-Jun 13:25:18:DEBUG:root:Init parameter bias for layer ip
30-Jun 13:25:18:DEBUG:root:Init parameter weight for layer pred
30-Jun 13:25:18:DEBUG:root:Init parameter bias for layer pred
30-Jun 13:25:18:DEBUG:root:#DEBUG Initializing coffee breaks
30-Jun 13:25:18:INFO:root: TRAIN iter=000000 obj_val=0.22203671
30-Jun 13:25:18:DEBUG:root:#DEBUG Entering solver loop
30-Jun 13:25:21:INFO:root: TRAIN iter=000100 obj_val=0.02317332
30-Jun 13:25:25:INFO:root: TRAIN iter=000200 obj_val=0.00744551
30-Jun 13:25:28:INFO:root: TRAIN iter=000300 obj_val=0.00388474
30-Jun 13:25:32:INFO:root: TRAIN iter=000400 obj_val=0.00216964
30-Jun 13:25:35:INFO:root: TRAIN iter=000500 obj_val=0.00126046
30-Jun 13:25:39:INFO:root: TRAIN iter=000600 obj_val=0.00077038
30-Jun 13:25:42:INFO:root: TRAIN iter=000700 obj_val=0.00049547
30-Jun 13:25:46:INFO:root: TRAIN iter=000800 obj_val=0.00033653
30-Jun 13:25:49:INFO:root: TRAIN iter=000900 obj_val=0.00024135
30-Jun 13:25:53:INFO:root: TRAIN iter=001000 obj_val=0.00018180
30-Jun 13:25:56:INFO:root: TRAIN iter=001100 obj_val=0.00014282
30-Jun 13:25:59:INFO:root: TRAIN iter=001200 obj_val=0.00011630
30-Jun 13:26:03:INFO:root: TRAIN iter=001300 obj_val=0.00009770
30-Jun 13:26:07:INFO:root: TRAIN iter=001400 obj_val=0.00008438
30-Jun 13:26:10:INFO:root: TRAIN iter=001500 obj_val=0.00007466
30-Jun 13:26:13:INFO:root: TRAIN iter=001600 obj_val=0.00006747
30-Jun 13:26:17:INFO:root: TRAIN iter=001700 obj_val=0.00006207
30-Jun 13:26:21:INFO:root: TRAIN iter=001800 obj_val=0.00005793
30-Jun 13:26:25:INFO:root: TRAIN iter=001900 obj_val=0.00005473
30-Jun 13:26:28:INFO:root: TRAIN iter=002000 obj_val=0.00005220
30-Jun 13:26:32:INFO:root: TRAIN iter=002100 obj_val=0.00005018
30-Jun 13:26:36:INFO:root: TRAIN iter=002200 obj_val=0.00004854
30-Jun 13:26:40:INFO:root: TRAIN iter=002300 obj_val=0.00004719
30-Jun 13:26:43:INFO:root: TRAIN iter=002400 obj_val=0.00004608
30-Jun 13:26:47:INFO:root: TRAIN iter=002500 obj_val=0.00004514
30-Jun 13:26:50:INFO:root: TRAIN iter=002600 obj_val=0.00004436
30-Jun 13:26:53:INFO:root: TRAIN iter=002700 obj_val=0.00004368
30-Jun 13:26:56:INFO:root: TRAIN iter=002800 obj_val=0.00004311
30-Jun 13:26:59:INFO:root: TRAIN iter=002900 obj_val=0.00004261
30-Jun 13:27:02:INFO:root: TRAIN iter=003000 obj_val=0.00004217
30-Jun 13:27:05:INFO:root: TRAIN iter=003100 obj_val=0.00004179
30-Jun 13:27:08:INFO:root: TRAIN iter=003200 obj_val=0.00004145
30-Jun 13:27:12:INFO:root: TRAIN iter=003300 obj_val=0.00004115
30-Jun 13:27:15:INFO:root: TRAIN iter=003400 obj_val=0.00004087
30-Jun 13:27:18:INFO:root: TRAIN iter=003500 obj_val=0.00004063
30-Jun 13:27:21:INFO:root: TRAIN iter=003600 obj_val=0.00004041
30-Jun 13:27:24:INFO:root: TRAIN iter=003700 obj_val=0.00004021
30-Jun 13:27:26:INFO:root: TRAIN iter=003800 obj_val=0.00004002
30-Jun 13:27:29:INFO:root: TRAIN iter=003900 obj_val=0.00003986
30-Jun 13:27:32:INFO:root: TRAIN iter=004000 obj_val=0.00003970
30-Jun 13:27:34:INFO:root: TRAIN iter=004100 obj_val=0.00003956
30-Jun 13:27:37:INFO:root: TRAIN iter=004200 obj_val=0.00003943
30-Jun 13:27:40:INFO:root: TRAIN iter=004300 obj_val=0.00003932
30-Jun 13:27:43:INFO:root: TRAIN iter=004400 obj_val=0.00003921
30-Jun 13:27:45:INFO:root: TRAIN iter=004500 obj_val=0.00003911
30-Jun 13:27:48:INFO:root: TRAIN iter=004600 obj_val=0.00003902
30-Jun 13:27:51:INFO:root: TRAIN iter=004700 obj_val=0.00003894
30-Jun 13:27:53:INFO:root: TRAIN iter=004800 obj_val=0.00003887
30-Jun 13:27:56:INFO:root: TRAIN iter=004900 obj_val=0.00003880
30-Jun 13:27:58:INFO:root: TRAIN iter=005000 obj_val=0.00003874
30-Jun 13:28:01:INFO:root: TRAIN iter=005100 obj_val=0.00003868
30-Jun 13:28:04:INFO:root: TRAIN iter=005200 obj_val=0.00003863
30-Jun 13:28:08:INFO:root: TRAIN iter=005300 obj_val=0.00003858
30-Jun 13:28:11:INFO:root: TRAIN iter=005400 obj_val=0.00003854
30-Jun 13:28:15:INFO:root: TRAIN iter=005500 obj_val=0.00003850
30-Jun 13:28:19:INFO:root: TRAIN iter=005600 obj_val=0.00003846
30-Jun 13:28:23:INFO:root: TRAIN iter=005700 obj_val=0.00003843
30-Jun 13:28:26:INFO:root: TRAIN iter=005800 obj_val=0.00003840
30-Jun 13:28:30:INFO:root: TRAIN iter=005900 obj_val=0.00003838
30-Jun 13:28:33:INFO:root: TRAIN iter=006000 obj_val=0.00003835
30-Jun 13:28:37:INFO:root: TRAIN iter=006100 obj_val=0.00003833
30-Jun 13:28:40:INFO:root: TRAIN iter=006200 obj_val=0.00003831
30-Jun 13:28:44:INFO:root: TRAIN iter=006300 obj_val=0.00003830
30-Jun 13:28:48:INFO:root: TRAIN iter=006400 obj_val=0.00003828
30-Jun 13:28:52:INFO:root: TRAIN iter=006500 obj_val=0.00003827
30-Jun 13:28:56:INFO:root: TRAIN iter=006600 obj_val=0.00003825
30-Jun 13:29:00:INFO:root: TRAIN iter=006700 obj_val=0.00003824
30-Jun 13:29:03:INFO:root: TRAIN iter=006800 obj_val=0.00003823
30-Jun 13:29:06:INFO:root: TRAIN iter=006900 obj_val=0.00003822
30-Jun 13:29:10:INFO:root: TRAIN iter=007000 obj_val=0.00003821
30-Jun 13:29:13:INFO:root: TRAIN iter=007100 obj_val=0.00003820
30-Jun 13:29:17:INFO:root: TRAIN iter=007200 obj_val=0.00003820
30-Jun 13:29:21:INFO:root: TRAIN iter=007300 obj_val=0.00003819
30-Jun 13:29:25:INFO:root: TRAIN iter=007400 obj_val=0.00003818
30-Jun 13:29:28:INFO:root: TRAIN iter=007500 obj_val=0.00003817
30-Jun 13:29:32:INFO:root: TRAIN iter=007600 obj_val=0.00003817
30-Jun 13:29:35:INFO:root: TRAIN iter=007700 obj_val=0.00003816
30-Jun 13:29:39:INFO:root: TRAIN iter=007800 obj_val=0.00003816
30-Jun 13:29:43:INFO:root: TRAIN iter=007900 obj_val=0.00003815
30-Jun 13:29:46:INFO:root: TRAIN iter=008000 obj_val=0.00003814
30-Jun 13:29:49:INFO:root: TRAIN iter=008100 obj_val=0.00003814
30-Jun 13:29:53:INFO:root: TRAIN iter=008200 obj_val=0.00003813
30-Jun 13:29:57:INFO:root: TRAIN iter=008300 obj_val=0.00003812
30-Jun 13:30:00:INFO:root: TRAIN iter=008400 obj_val=0.00003812
30-Jun 13:30:04:INFO:root: TRAIN iter=008500 obj_val=0.00003811
30-Jun 13:30:07:INFO:root: TRAIN iter=008600 obj_val=0.00003810
30-Jun 13:30:11:INFO:root: TRAIN iter=008700 obj_val=0.00003810
30-Jun 13:30:13:INFO:root: TRAIN iter=008800 obj_val=0.00003809
30-Jun 13:30:16:INFO:root: TRAIN iter=008900 obj_val=0.00003808
30-Jun 13:30:19:INFO:root: TRAIN iter=009000 obj_val=0.00003807
30-Jun 13:30:22:INFO:root: TRAIN iter=009100 obj_val=0.00003806
30-Jun 13:30:25:INFO:root: TRAIN iter=009200 obj_val=0.00003805
30-Jun 13:30:28:INFO:root: TRAIN iter=009300 obj_val=0.00003804
30-Jun 13:30:30:INFO:root: TRAIN iter=009400 obj_val=0.00003803
30-Jun 13:30:33:INFO:root: TRAIN iter=009500 obj_val=0.00003802
30-Jun 13:30:36:INFO:root: TRAIN iter=009600 obj_val=0.00003801
30-Jun 13:30:40:INFO:root: TRAIN iter=009700 obj_val=0.00003800
30-Jun 13:30:42:INFO:root: TRAIN iter=009800 obj_val=0.00003799
30-Jun 13:30:45:INFO:root: TRAIN iter=009900 obj_val=0.00003798
30-Jun 13:30:49:INFO:root: TRAIN iter=010000 obj_val=0.00003797
-------Evaluating on Test-------
30-Jun 13:30:49:INFO:root:Constructing net Test on Mocha.CPUBackend...
30-Jun 13:30:49:INFO:root:Topological sorting 3 layers...
30-Jun 13:30:49:INFO:root:Setup layers...
30-Jun 13:30:49:DEBUG:root:InnerProductLayer(ip): sharing weights and bias
30-Jun 13:30:49:DEBUG:root:InnerProductLayer(pred): sharing weights and bias
30-Jun 13:30:49:INFO:root:Network constructed!
331.426570 seconds (3.40 M allocations: 39.112 GB, 0.64% gc time)
Out[18]:
(
1x1000 Array{Float64,2}:
 -1.69483  1.29428  -1.69285  -0.393985  …  -0.419886  0.855957  -0.673332,

1x1000 Array{Float64,2}:
 -1.73139  1.25579  -1.73625  -0.30835  …  -0.485388  0.857431  -0.746863)
In [19]:
#Time CPU = 340.415832 seconds (3.40 M allocations: 39.112 GB, 0.55% gc time)
#Time CPU_with_OpenMP = 331.426570 seconds (3.40 M allocations: 39.112 GB, 0.64% gc time)
#time GTX960 = 251.337496 seconds (4.26 M allocations: 38.784 GB, 1.01% gc time)
#time Tesla K40 = 67.574583 seconds (4.12 M allocations: 38.778 GB, 2.39% gc time)
println(" = scaled to 0-1; 10,000 iter ")
show_results_summary(pred_scaled_10k, actual_scaled_10k)
 = scaled to 0-1; 10,000 iter 
mean_error = 0.10858464291503868
std(errors) = 0.2807570441989076
minimum(errors) = 1.917062086853605e-6
median(errors) = 0.05629081429380989
maximum(errors) = 3.7358848230052435
accurancy = 0.0
accurancy_todefault_epsilon = 0.0
accurancy_to_point001 = 0.008
accurancy_to_point01 = 0.082
accurancy_to_point1 = 0.725
In [10]:
@time pred_cossin_10k, actual_cossin_10k = eval_angular_encoding(encode_cossin, decode_cossin;max_iter=10_000)
size(ys_code) = (2,1000)
label_dim = 2
30-Jun 13:32:43:INFO:root:Initializing CuDNN backend...
30-Jun 13:32:44:INFO:root:CuDNN backend initialized!
30-Jun 13:32:44:INFO:root:Constructing net Training on Mocha.GPUBackend...
30-Jun 13:32:44:INFO:root:Topological sorting 4 layers...
30-Jun 13:32:44:INFO:root:Setup layers...
30-Jun 13:32:45:INFO:root:Network constructed!
30-Jun 13:32:45:DEBUG:root:#DEBUG Checking network topology for back-propagation
30-Jun 13:32:45:DEBUG:root:Init network Training
30-Jun 13:32:45:DEBUG:root:Init parameter weight for layer ip
30-Jun 13:32:45:DEBUG:root:Init parameter bias for layer ip
30-Jun 13:32:45:DEBUG:root:Init parameter weight for layer pred
30-Jun 13:32:45:DEBUG:root:Init parameter bias for layer pred
30-Jun 13:32:45:DEBUG:root:#DEBUG Initializing coffee breaks
30-Jun 13:32:45:INFO:root: TRAIN iter=000000 obj_val=0.72353330
30-Jun 13:32:46:DEBUG:root:#DEBUG Entering solver loop
30-Jun 13:32:46:INFO:root: TRAIN iter=000100 obj_val=0.10527111
30-Jun 13:32:47:INFO:root: TRAIN iter=000200 obj_val=0.02965303
30-Jun 13:32:48:INFO:root: TRAIN iter=000300 obj_val=0.01149488
30-Jun 13:32:48:INFO:root: TRAIN iter=000400 obj_val=0.00391103
30-Jun 13:32:49:INFO:root: TRAIN iter=000500 obj_val=0.00132583
30-Jun 13:32:50:INFO:root: TRAIN iter=000600 obj_val=0.00060305
30-Jun 13:32:50:INFO:root: TRAIN iter=000700 obj_val=0.00042884
30-Jun 13:32:51:INFO:root: TRAIN iter=000800 obj_val=0.00036293
30-Jun 13:32:52:INFO:root: TRAIN iter=000900 obj_val=0.00030879
30-Jun 13:32:52:INFO:root: TRAIN iter=001000 obj_val=0.00025920
30-Jun 13:32:53:INFO:root: TRAIN iter=001100 obj_val=0.00021741
30-Jun 13:32:54:INFO:root: TRAIN iter=001200 obj_val=0.00018417
30-Jun 13:32:54:INFO:root: TRAIN iter=001300 obj_val=0.00015839
30-Jun 13:32:55:INFO:root: TRAIN iter=001400 obj_val=0.00013848
30-Jun 13:32:56:INFO:root: TRAIN iter=001500 obj_val=0.00012297
30-Jun 13:32:56:INFO:root: TRAIN iter=001600 obj_val=0.00011069
30-Jun 13:32:57:INFO:root: TRAIN iter=001700 obj_val=0.00010077
30-Jun 13:32:58:INFO:root: TRAIN iter=001800 obj_val=0.00009262
30-Jun 13:32:58:INFO:root: TRAIN iter=001900 obj_val=0.00008583
30-Jun 13:32:59:INFO:root: TRAIN iter=002000 obj_val=0.00008010
30-Jun 13:33:00:INFO:root: TRAIN iter=002100 obj_val=0.00007523
30-Jun 13:33:00:INFO:root: TRAIN iter=002200 obj_val=0.00007107
30-Jun 13:33:01:INFO:root: TRAIN iter=002300 obj_val=0.00006749
30-Jun 13:33:02:INFO:root: TRAIN iter=002400 obj_val=0.00006440
30-Jun 13:33:02:INFO:root: TRAIN iter=002500 obj_val=0.00006171
30-Jun 13:33:03:INFO:root: TRAIN iter=002600 obj_val=0.00005936
30-Jun 13:33:04:INFO:root: TRAIN iter=002700 obj_val=0.00005731
30-Jun 13:33:04:INFO:root: TRAIN iter=002800 obj_val=0.00005550
30-Jun 13:33:05:INFO:root: TRAIN iter=002900 obj_val=0.00005390
30-Jun 13:33:06:INFO:root: TRAIN iter=003000 obj_val=0.00005248
30-Jun 13:33:06:INFO:root: TRAIN iter=003100 obj_val=0.00005121
30-Jun 13:33:07:INFO:root: TRAIN iter=003200 obj_val=0.00005007
30-Jun 13:33:08:INFO:root: TRAIN iter=003300 obj_val=0.00004905
30-Jun 13:33:08:INFO:root: TRAIN iter=003400 obj_val=0.00004813
30-Jun 13:33:09:INFO:root: TRAIN iter=003500 obj_val=0.00004730
30-Jun 13:33:10:INFO:root: TRAIN iter=003600 obj_val=0.00004654
30-Jun 13:33:10:INFO:root: TRAIN iter=003700 obj_val=0.00004585
30-Jun 13:33:11:INFO:root: TRAIN iter=003800 obj_val=0.00004523
30-Jun 13:33:12:INFO:root: TRAIN iter=003900 obj_val=0.00004465
30-Jun 13:33:12:INFO:root: TRAIN iter=004000 obj_val=0.00004413
30-Jun 13:33:13:INFO:root: TRAIN iter=004100 obj_val=0.00004365
30-Jun 13:33:14:INFO:root: TRAIN iter=004200 obj_val=0.00004321
30-Jun 13:33:14:INFO:root: TRAIN iter=004300 obj_val=0.00004280
30-Jun 13:33:15:INFO:root: TRAIN iter=004400 obj_val=0.00004243
30-Jun 13:33:16:INFO:root: TRAIN iter=004500 obj_val=0.00004209
30-Jun 13:33:16:INFO:root: TRAIN iter=004600 obj_val=0.00004177
30-Jun 13:33:17:INFO:root: TRAIN iter=004700 obj_val=0.00004148
30-Jun 13:33:18:INFO:root: TRAIN iter=004800 obj_val=0.00004121
30-Jun 13:33:18:INFO:root: TRAIN iter=004900 obj_val=0.00004097
30-Jun 13:33:19:INFO:root: TRAIN iter=005000 obj_val=0.00004074
30-Jun 13:33:20:INFO:root: TRAIN iter=005100 obj_val=0.00004052
30-Jun 13:33:20:INFO:root: TRAIN iter=005200 obj_val=0.00004033
30-Jun 13:33:21:INFO:root: TRAIN iter=005300 obj_val=0.00004015
30-Jun 13:33:22:INFO:root: TRAIN iter=005400 obj_val=0.00003998
30-Jun 13:33:22:INFO:root: TRAIN iter=005500 obj_val=0.00003982
30-Jun 13:33:23:INFO:root: TRAIN iter=005600 obj_val=0.00003967
30-Jun 13:33:24:INFO:root: TRAIN iter=005700 obj_val=0.00003954
30-Jun 13:33:24:INFO:root: TRAIN iter=005800 obj_val=0.00003941
30-Jun 13:33:25:INFO:root: TRAIN iter=005900 obj_val=0.00003929
30-Jun 13:33:26:INFO:root: TRAIN iter=006000 obj_val=0.00003918
30-Jun 13:33:26:INFO:root: TRAIN iter=006100 obj_val=0.00003908
30-Jun 13:33:27:INFO:root: TRAIN iter=006200 obj_val=0.00003898
30-Jun 13:33:28:INFO:root: TRAIN iter=006300 obj_val=0.00003889
30-Jun 13:33:28:INFO:root: TRAIN iter=006400 obj_val=0.00003881
30-Jun 13:33:29:INFO:root: TRAIN iter=006500 obj_val=0.00003873
30-Jun 13:33:30:INFO:root: TRAIN iter=006600 obj_val=0.00003865
30-Jun 13:33:30:INFO:root: TRAIN iter=006700 obj_val=0.00003858
30-Jun 13:33:31:INFO:root: TRAIN iter=006800 obj_val=0.00003852
30-Jun 13:33:32:INFO:root: TRAIN iter=006900 obj_val=0.00003845
30-Jun 13:33:32:INFO:root: TRAIN iter=007000 obj_val=0.00003839
30-Jun 13:33:33:INFO:root: TRAIN iter=007100 obj_val=0.00003834
30-Jun 13:33:34:INFO:root: TRAIN iter=007200 obj_val=0.00003828
30-Jun 13:33:34:INFO:root: TRAIN iter=007300 obj_val=0.00003823
30-Jun 13:33:35:INFO:root: TRAIN iter=007400 obj_val=0.00003818
30-Jun 13:33:36:INFO:root: TRAIN iter=007500 obj_val=0.00003813
30-Jun 13:33:36:INFO:root: TRAIN iter=007600 obj_val=0.00003809
30-Jun 13:33:37:INFO:root: TRAIN iter=007700 obj_val=0.00003805
30-Jun 13:33:38:INFO:root: TRAIN iter=007800 obj_val=0.00003801
30-Jun 13:33:38:INFO:root: TRAIN iter=007900 obj_val=0.00003797
30-Jun 13:33:39:INFO:root: TRAIN iter=008000 obj_val=0.00003793
30-Jun 13:33:40:INFO:root: TRAIN iter=008100 obj_val=0.00003789
30-Jun 13:33:40:INFO:root: TRAIN iter=008200 obj_val=0.00003786
30-Jun 13:33:41:INFO:root: TRAIN iter=008300 obj_val=0.00003782
30-Jun 13:33:42:INFO:root: TRAIN iter=008400 obj_val=0.00003779
30-Jun 13:33:42:INFO:root: TRAIN iter=008500 obj_val=0.00003776
30-Jun 13:33:43:INFO:root: TRAIN iter=008600 obj_val=0.00003773
30-Jun 13:33:44:INFO:root: TRAIN iter=008700 obj_val=0.00003770
30-Jun 13:33:44:INFO:root: TRAIN iter=008800 obj_val=0.00003767
30-Jun 13:33:45:INFO:root: TRAIN iter=008900 obj_val=0.00003764
30-Jun 13:33:46:INFO:root: TRAIN iter=009000 obj_val=0.00003762
30-Jun 13:33:46:INFO:root: TRAIN iter=009100 obj_val=0.00003759
30-Jun 13:33:47:INFO:root: TRAIN iter=009200 obj_val=0.00003757
30-Jun 13:33:48:INFO:root: TRAIN iter=009300 obj_val=0.00003754
30-Jun 13:33:48:INFO:root: TRAIN iter=009400 obj_val=0.00003752
30-Jun 13:33:49:INFO:root: TRAIN iter=009500 obj_val=0.00003750
30-Jun 13:33:50:INFO:root: TRAIN iter=009600 obj_val=0.00003747
30-Jun 13:33:50:INFO:root: TRAIN iter=009700 obj_val=0.00003745
30-Jun 13:33:51:INFO:root: TRAIN iter=009800 obj_val=0.00003743
30-Jun 13:33:52:INFO:root: TRAIN iter=009900 obj_val=0.00003741
30-Jun 13:33:52:INFO:root: TRAIN iter=010000 obj_val=0.00003739
-------Evaluating on Test-------
30-Jun 13:33:53:INFO:root:Constructing net Test on Mocha.GPUBackend...
30-Jun 13:33:53:INFO:root:Topological sorting 3 layers...
30-Jun 13:33:53:INFO:root:Setup layers...
30-Jun 13:33:53:DEBUG:root:InnerProductLayer(ip): sharing weights and bias
30-Jun 13:33:53:DEBUG:root:InnerProductLayer(pred): sharing weights and bias
30-Jun 13:33:53:INFO:root:Network constructed!
 72.630029 seconds (13.26 M allocations: 39.162 GB, 2.45% gc time)
Out[10]:
(
1000x1 Array{Float64,2}:
 -0.296786
 -1.55355 
 -1.39308 
 -2.80827 
 -0.580536
  0.719799
 -0.273527
  2.24399 
 -1.28489 
 -0.990405
 -0.513425
 -1.80072 
 -2.64894 
  ⋮       
 -0.397097
  3.12607 
  2.63425 
 -0.156399
  1.34117 
  2.27049 
 -2.76929 
  2.50708 
  0.855685
 -2.3646  
 -2.96012 
 -2.75884 ,

1x1000 Array{Float64,2}:
 -0.304842  -1.54747  -1.37505  -2.8153  …  -2.35986  -2.95677  -2.75746)
In [12]:
#time Tesla K40 = 72.630029 seconds (13.26 M allocations: 39.162 GB, 2.45% gc time)
println(" = Sin/Cos; 10,000 iter")
show_results_summary(pred_cossin_10k, actual_cossin_10k)
 = Sin/Cos; 10,000 iter
mean_error = 2.1036656318235125
std(errors) = 1.4891605067844906
minimum(errors) = 5.8157226292543385e-6
median(errors) = 1.8491085946618226
maximum(errors) = 6.2704853961745375
accurancy = 0.0
accurancy_todefault_epsilon = 0.0
accurancy_to_point001 = 0.128
accurancy_to_point01 = 0.746
accurancy_to_point1 = 0.999
In [ ]:

In [ ]: