#!/usr/bin/env python
# coding: utf-8

# # Activation Functions
# 
# This function introduces activation functions in TensorFlow
# 
# We start by loading the necessary libraries for this script.

# In[1]:


import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
ops.reset_default_graph()


# ### Start a graph session

# In[2]:


sess = tf.Session()


# ### Initialize the X range values for plotting

# In[3]:


x_vals = np.linspace(start=-10., stop=10., num=100)


# ### Activation Functions:
# 
# ReLU activation

# In[4]:


print(sess.run(tf.nn.relu([-3., 3., 10.])))
y_relu = sess.run(tf.nn.relu(x_vals))


# ReLU-6 activation

# In[5]:


print(sess.run(tf.nn.relu6([-3., 3., 10.])))
y_relu6 = sess.run(tf.nn.relu6(x_vals))


# Sigmoid activation

# In[6]:


print(sess.run(tf.nn.sigmoid([-1., 0., 1.])))
y_sigmoid = sess.run(tf.nn.sigmoid(x_vals))


# Hyper Tangent activation

# In[7]:


print(sess.run(tf.nn.tanh([-1., 0., 1.])))
y_tanh = sess.run(tf.nn.tanh(x_vals))


# Softsign activation

# In[8]:


print(sess.run(tf.nn.softsign([-1., 0., 1.])))
y_softsign = sess.run(tf.nn.softsign(x_vals))


# Softplus activation

# In[9]:


print(sess.run(tf.nn.softplus([-1., 0., 1.])))
y_softplus = sess.run(tf.nn.softplus(x_vals))


# Exponential linear activation

# In[10]:


print(sess.run(tf.nn.elu([-1., 0., 1.])))
y_elu = sess.run(tf.nn.elu(x_vals))


# ### Plot the different functions

# In[12]:


plt.plot(x_vals, y_softplus, 'r--', label='Softplus', linewidth=2)
plt.plot(x_vals, y_relu, 'b:', label='ReLU', linewidth=2)
plt.plot(x_vals, y_relu6, 'g-.', label='ReLU6', linewidth=2)
plt.plot(x_vals, y_elu, 'k-', label='ExpLU', linewidth=0.5)
plt.ylim([-1.5,7])
plt.legend(loc='upper left')
plt.show()

plt.plot(x_vals, y_sigmoid, 'r--', label='Sigmoid', linewidth=2)
plt.plot(x_vals, y_tanh, 'b:', label='Tanh', linewidth=2)
plt.plot(x_vals, y_softsign, 'g-.', label='Softsign', linewidth=2)
plt.ylim([-2,2])
plt.legend(loc='upper left')
plt.show()


# ![Acivation_Functions1](https://github.com/nfmcclure/tensorflow_cookbook/raw/jupyter_notebooks/01_Introduction/images/06_activation_funs1.png)
# 
# ![Acivation_Functions2](https://github.com/nfmcclure/tensorflow_cookbook/raw/jupyter_notebooks/01_Introduction/images/06_activation_funs2.png)