I found this basic Echo State Networks(ESN) source code.
# -*- coding: utf-8 -*-
"""
A minimalistic Echo State Networks demo with Mackey-Glass (delay 17) data
in "plain" scientific Python.
from https://mantas.info/code/simple_esn/
(c) 2012-2020 Mantas Luko?evi?ius
Distributed under MIT license https://opensource.org/licenses/MIT
"""
import sys
import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg
np.set_printoptions(threshold=sys.maxsize)
# numpy.linalg is also an option for even fewer dependencies
# load the data
trainLen = 2000
testLen = 2000
initLen = 100
data = np.loadtxt('MackeyGlass_t17.txt')
# plot some of it
plt.figure(10).clear()
plt.plot(data[:1000])
plt.title('A sample of data')
# generate the ESN reservoir
inSize = outSize = 1
resSize = 1000
a = 0.3 # leaking rate
np.random.seed(42)
Win = (np.random.rand(resSize, 1 + inSize) - 0.5) * 1
W = np.random.rand(resSize, resSize) - 0.5
# normalizing and setting spectral radius (correct, slow):
print('Computing spectral radius...')
rhoW = max(abs(linalg.eig(W)[0]))
print('done.')
W *= 1.25 / rhoW
# allocated memory for the design (collected states) matrix
X = np.zeros((1 + inSize + resSize, trainLen - initLen))
# set the corresponding target matrix directly
Yt = data[None, initLen + 1:trainLen + 1]
# run the reservoir with the data and collect X
x = np.zeros((resSize, 1))
for t in range(trainLen):
u = data[t]
x = (1 - a) * x + a * np.tanh(np.dot(Win, np.vstack((1, u))) + np.dot(W, x))
if t >= initLen:
X[:, t - initLen] = np.vstack((1, u, x))[:, 0]
# train the output by ridge regression
reg = 1e-8 # regularization coefficient
# direct equations from texts:
# X_T = X.T
# Wout = np.dot( np.dot(Yt,X_T), linalg.inv( np.dot(X,X_T) +
# reg*np.eye(1+inSize+resSize) ) )
# using scipy.linalg.solve:
Wout = linalg.solve(np.dot(X, X.T) + reg * np.eye(1 + inSize + resSize),
np.dot(X, Yt.T)).T
# run the trained ESN in a generative mode. no need to initialize here,
# because x is initialized with training data and we continue from there.
Y = np.zeros((outSize, testLen))
u = data[trainLen]
for t in range(testLen):
x = (1 - a) * x + a * np.tanh(np.dot(Win, np.vstack((1, u))) + np.dot(W, x))
y = np.dot(Wout, np.vstack((1, u, x)))
Y[:, t] = y
# generative mode:
u = y
## this would be a predictive mode:
# u = data[trainLen+t+1]
# compute MSE for the first errorLen time steps
errorLen = 500
mse = sum(np.square(data[trainLen + 1:trainLen + errorLen + 1] -
Y[0, 0:errorLen])) / errorLen
print('MSE = ' + str(mse))
# plot some signals
plt.figure(1).clear()
plt.plot(data[trainLen + 1:trainLen + testLen + 1], 'g')
plt.plot(Y.T, 'b')
plt.title('Target and generated signals $y(n)$ starting at $n=0$')
plt.legend(['Target signal', 'Free-running predicted signal'])
plt.figure(2).clear()
plt.plot(X[0:20, 0:200].T)
plt.title(r'Some reservoir activations $mathbf{x}(n)$')
plt.figure(3).clear()
plt.bar(np.arange(1 + inSize + resSize), Wout[0].T)
plt.title(r'Output weights $mathbf{W}^{out}$')
plt.show()
Some of the reservoir activation of this code is
So I'm trying to create a multi layer/deep Echo State Networks using the above source code. I'm using this paper https://www.sciencedirect.com/science/article/abs/pii/S0893608018302223 as reference.
In my code, i added the following for the next layer after the first layer.
# run the reservoir with the data and collect X
x = np.zeros((resSize, 1))
un = np.zeros(trainLen)
for t in range(trainLen):
u = data[t]
x = (1 - a) * x + a * np.tanh(np.dot(Win, np.vstack((1, u))) +
np.dot(W, x))
un[t] = np.dot(Wn, x)
# next layer
for ly in range(layers):
# generate the new ESN reservoir weights for new layer
Win = (np.random.rand(resSize, 1 + inSize) - 0.5) * 1
W = np.random.rand(resSize, resSize) - 0.5
Wn = (np.random.rand(1, resSize) - 0.5) * 1
# normalizing and setting spectral radius (correct, slow):
print('Computing spectral radius...')
rhoW = max(abs(linalg.eig(W)[0]))
print('done.')
W *= 1.25 / rhoW
x = np.zeros((resSize, 1))
# run through reservoir
for t in range(trainLen):
u = un[t]
x = (1 - a) * x + a * np.tanh(np.dot(Win, np.vstack((1, u))) + np.dot(W, x)
un[t] = np.dot(Wn, x)
if t >= initLen:
X[:, t - initLen] = np.vstack((1, u, x))[:, 0] # (3,0)
# train the output by ridge regression
reg = 1e-8 # regularization coefficient
# direct equations from texts:
# X_T = X.T
# Wout = np.dot( np.dot(Yt,X_T), linalg.inv( np.dot(X,X_T) +
# reg*np.eye(1+inSize+resSize) ) )
# using scipy.linalg.solve:
Wout = linalg.solve(np.dot(X, X.T) + reg * np.eye(1 + inSize + resSize),
np.dot(X, Yt.T)).T
Some of the reservoir activation after i added the above code is
As you can see one of the reservoir activation after i added the above code become extremely large. Could anyone explain to me why one of the reservoir activation become extremely large?
question from:https://stackoverflow.com/questions/65559653/creating-multi-layer-echo-state-networks
