#!/usr/bin/env python
# coding: utf-8

# In[42]:


import matplotlib.pyplot as plt
import numpy
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense
from mnist import MNIST
from sklearn.model_selection import GridSearchCV
from keras.layers import Dropout
from keras.wrappers.scikit_learn import KerasClassifier
from keras.constraints import maxnorm
from keras.models import Model   
from keras.layers import * 
from keras import optimizers
get_ipython().run_line_magic('matplotlib', 'inline')


# # Цель вычислительного эксперимента:
# Решить задачу: выбора алгоритма оптимизации (SGD, Nesterov Momentum, Adam)  с использованием нейронных сетей простой структуры, со структурными параметрами: структура сети
# 

# In[18]:


model = Sequential()


# In[19]:


model.add(Dense(units=785, activation='relu', input_dim=100))
model.add(Dense(units=10, activation='softmax'))


# In[20]:


model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])


# In[21]:


train = pd.read_csv("/home/jeny/mnist_train.csv")
test = pd.read_csv("/home/jeny/mnist_test.csv")
train = train.iloc[:40000, :]
test = train.iloc[:40000, :]


# In[22]:


train.shape


# In[23]:


train.head()


# In[24]:


x_train = train.iloc[:,1:]
y_train = train.iloc[:, 0:1]

x_test  = test.iloc[:, 1:]
y_test  = test.iloc[:, 0:1]
x_train.shape


# In[43]:


def create_model(neurons=1):
    model = Sequential()
    model.add(Dense(30, activation='sigmoid', input_shape=(784, )))
    model.add(Dense(1, activation='linear'))
    sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy'])
    return model


# In[44]:


model = KerasClassifier(build_fn=create_model, epochs=15, batch_size=80 , verbose=1)


# # Adam

# In[31]:


neurons  = [5,10,15,20,50,100]
param_grid = dict(neurons=neurons)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
grid_result = grid.fit(x_train, y_train)


# In[33]:


print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))


# # SGD

# In[36]:


neurons  = [5,10,15,20,50,100]
param_grid = dict(neurons=neurons)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
grid_result = grid.fit(x_train, y_train)


# In[37]:


print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))


# # Nesterov

# In[45]:


neurons  = [5,10,15,20,50,100]
param_grid = dict(neurons=neurons)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
grid_result = grid.fit(x_train, y_train)


# In[46]:


print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))


# In[ ]: