import numpy as np
import matplotlib.pyplot as plt
seq_len = 128
d_model = 512
pe = np.zeros((seq_len, d_model))
for pos in range(seq_len):
for i in range(int(d_model / 2)):
pe[pos,2*i] = np.sin(pos/(pow(10000,((2*i)/d_model))))
pe[pos,2*i+1] = np.cos(pos/(pow(10000,((2*i)/d_model))))
fig= plt.figure(figsize=(20,5))
# plt.xlabel('2i (d_model = 512)')
# plt.ylabel('pos')
plt.imshow(pe, cmap = 'Greys')
<matplotlib.image.AxesImage at 0x7f904cad9ac8>
enc = np.zeros((seq_len, d_model))
enc = np.random.rand(seq_len, d_model)
fig= plt.figure(figsize=(20,5))
plt.imshow(enc, cmap = 'Greys')
<matplotlib.image.AxesImage at 0x7f904ca882e8>
out = enc + pe
fig= plt.figure(figsize=(20,5))
plt.imshow(out, cmap = 'Greys')
<matplotlib.image.AxesImage at 0x7f904ca6ce80>