From ee159f8fed8b1b7595058c36d432e62b1c85b8da Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Thu, 25 Oct 2018 19:50:37 -0400 Subject: [PATCH 1/9] Change action selection from last spike to sum of spikes --- bindsnet/pipeline/__init__.py | 13 ++++++++----- bindsnet/pipeline/action.py | 3 +-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bindsnet/pipeline/__init__.py b/bindsnet/pipeline/__init__.py index 7d4100f3..b675c579 100644 --- a/bindsnet/pipeline/__init__.py +++ b/bindsnet/pipeline/__init__.py @@ -75,17 +75,20 @@ def __init__(self, network: Network, environment: Environment, encoding: Callabl else: self.history = {} + # monitor spikes for selecting action based on sum of spikes + for l in self.network.layers: + self.network.add_monitor(Monitor(self.network.layers[l], 's', self.time), + name=f'{l}_spikes') + + self.spike_record = {l: torch.Tensor().byte() for l in self.network.layers} + self.set_spike_data() + if self.plot_interval is not None: for l in self.network.layers: - self.network.add_monitor(Monitor(self.network.layers[l], 's', self.plot_interval * self.time), - name=f'{l}_spikes') - if 'v' in self.network.layers[l].__dict__: self.network.add_monitor(Monitor(self.network.layers[l], 'v', self.plot_interval * self.time), name=f'{l}_voltages') - self.spike_record = {l: torch.Tensor().byte() for l in self.network.layers} - self.set_spike_data() self.plot_data() # Set up for multiple layers of input layers. diff --git a/bindsnet/pipeline/action.py b/bindsnet/pipeline/action.py index b14658be..477725a0 100644 --- a/bindsnet/pipeline/action.py +++ b/bindsnet/pipeline/action.py @@ -61,10 +61,9 @@ def select_softmax(pipeline: Pipeline, **kwargs) -> int: 'Output layer size not equal to size of action space.' # Sum of previous iterations' spikes (Not yet implemented) - spikes = pipeline.network.layers[output].s + spikes = torch.sum(pipeline.spike_record[output], dim=1) _sum = torch.sum(torch.exp(spikes.float())) - # Choose action based on readout neuron spiking if _sum == 0: action = np.random.choice(pipeline.env.action_space.n) else: From 73043c84b5857ee0a06cd6ca7f8a853d34e95ff5 Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Thu, 25 Oct 2018 19:52:06 -0400 Subject: [PATCH 2/9] Reduce weights of the breakout example --- examples/breakout/breakout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/breakout/breakout.py b/examples/breakout/breakout.py index 0c1f357d..5daf87e4 100644 --- a/examples/breakout/breakout.py +++ b/examples/breakout/breakout.py @@ -17,8 +17,8 @@ out = LIFNodes(n=4, refrac=0, traces=True) # Connections between layers. -inpt_middle = Connection(source=inpt, target=middle, wmax=1e-2) -middle_out = Connection(source=middle, target=out, wmax=1e-1, nu=2e-2) +inpt_middle = Connection(source=inpt, target=middle, wmin=0, wmax=1e-1) +middle_out = Connection(source=middle, target=out, wmin=0, wmax=1) # Add all layers and connections to the network. network.add_layer(inpt, name='X') From e51a7fdfabbae84ac5e41151b09f3362ce3e4fd2 Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Thu, 25 Oct 2018 21:41:54 -0400 Subject: [PATCH 3/9] Add breakout STDP right and rename layers for breakout --- examples/breakout/breakout.py | 12 ++--- examples/breakout/breakout_stdp_right.py | 56 ++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 examples/breakout/breakout_stdp_right.py diff --git a/examples/breakout/breakout.py b/examples/breakout/breakout.py index 5daf87e4..bf2e814b 100644 --- a/examples/breakout/breakout.py +++ b/examples/breakout/breakout.py @@ -21,11 +21,11 @@ middle_out = Connection(source=middle, target=out, wmin=0, wmax=1) # Add all layers and connections to the network. -network.add_layer(inpt, name='X') -network.add_layer(middle, name='Y') -network.add_layer(out, name='Z') -network.add_connection(inpt_middle, source='X', target='Y') -network.add_connection(middle_out, source='Y', target='Z') +network.add_layer(inpt, name='Input Layer') +network.add_layer(middle, name='Hidden Layer') +network.add_layer(out, name='Output Layer') +network.add_connection(inpt_middle, source='Input Layer', target='Hidden Layer') +network.add_connection(middle_out, source='Hidden Layer', target='Output Layer') # Load SpaceInvaders environment. environment = GymEnvironment('BreakoutDeterministic-v4') @@ -33,7 +33,7 @@ # Build pipeline from specified components. pipeline = Pipeline(network, environment, encoding=bernoulli, - action_function=select_softmax, output='Z', + action_function=select_softmax, output='Output Layer', time=100, history_length=1, delta=1, plot_interval=1, render_interval=1) diff --git a/examples/breakout/breakout_stdp_right.py b/examples/breakout/breakout_stdp_right.py new file mode 100644 index 00000000..39bfd5cf --- /dev/null +++ b/examples/breakout/breakout_stdp_right.py @@ -0,0 +1,56 @@ +import torch + +from bindsnet.network import Network +from bindsnet.pipeline import Pipeline +from bindsnet.learning import PostPre +from bindsnet.encoding import bernoulli +from bindsnet.network.topology import Connection +from bindsnet.environment import GymEnvironment +from bindsnet.network.nodes import Input, LIFNodes +from bindsnet.pipeline.action import select_softmax + +# Build network. +network = Network(dt=1.0) + +# Layers of neurons. +inpt = Input(n=80 * 80, shape=[80, 80], traces=True) +middle = LIFNodes(n=100, traces=True) +out = LIFNodes(n=4, refrac=0, traces=True) + +# Connections between layers. +inpt_middle = Connection(source=inpt, target=middle, wmin=0, wmax=1e-1) +middle_out = Connection(source=middle, target=out, wmin=0, wmax=1, update_rule=PostPre, nu=1e-1) + +# Add all layers and connections to the network. +network.add_layer(inpt, name='Input Layer') +network.add_layer(middle, name='Hidden Layer') +network.add_layer(out, name='Output Layer') +network.add_connection(inpt_middle, source='Input Layer', target='Hidden Layer') +network.add_connection(middle_out, source='Hidden Layer', target='Output Layer') + +# Load SpaceInvaders environment. +environment = GymEnvironment('BreakoutDeterministic-v4') +environment.reset() + +# Build pipeline from specified components. +pipeline = Pipeline(network, environment, encoding=bernoulli, + action_function=select_softmax, output='Output Layer', + time=100, history_length=1, delta=1, + plot_interval=1, render_interval=1) + + +# save the initial weights +initial_weights = network.connections[('Hidden Layer', 'Output Layer')].w.clone() +# Run environment simulation for 100 episodes. +for i in range(100): + # initialize episode reward + reward = 0 + pipeline.reset_() + while True: + pipeline.step() + # reset weights for action 0 = No action, 1 = Fire and 3 = Move left + network.connections[('Hidden Layer', 'Output Layer')].w[:, [0, 1, 3]] = initial_weights[:, [0, 1, 3]] + reward += pipeline.reward + if pipeline.done: + break + print("Episode " + str(i) + " reward:", reward) From 21afaf89742700d522dbfc03ae6c5e89141695eb Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 06:42:32 -0400 Subject: [PATCH 4/9] MSTDP example for breakout --- examples/breakout/breakout_stdp.py | 68 ++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 examples/breakout/breakout_stdp.py diff --git a/examples/breakout/breakout_stdp.py b/examples/breakout/breakout_stdp.py new file mode 100644 index 00000000..7809cfc2 --- /dev/null +++ b/examples/breakout/breakout_stdp.py @@ -0,0 +1,68 @@ +import torch + +from bindsnet.network import Network +from bindsnet.pipeline import Pipeline +from bindsnet.learning import MSTDP +from bindsnet.encoding import bernoulli +from bindsnet.network.topology import Connection +from bindsnet.environment import GymEnvironment +from bindsnet.network.nodes import Input, LIFNodes +from bindsnet.pipeline.action import select_softmax + +# Build network. +network = Network(dt=1.0) + +# Layers of neurons. +inpt = Input(n=80 * 80, shape=[80, 80], traces=True) +middle = LIFNodes(n=100, traces=True) +out = LIFNodes(n=4, refrac=0, traces=True) + +# Connections between layers. +inpt_middle = Connection(source=inpt, target=middle, wmin=0, wmax=1e-1) +middle_out = Connection(source=middle, target=out, wmin=0, wmax=1, update_rule=MSTDP, nu=1e-1, norm=0.5 * middle.n) + +# Add all layers and connections to the network. +network.add_layer(inpt, name='Input Layer') +network.add_layer(middle, name='Hidden Layer') +network.add_layer(out, name='Output Layer') +network.add_connection(inpt_middle, source='Input Layer', target='Hidden Layer') +network.add_connection(middle_out, source='Hidden Layer', target='Output Layer') + +# Load SpaceInvaders environment. +environment = GymEnvironment('BreakoutDeterministic-v4') +environment.reset() + +# Build pipeline from specified components. +pipeline = Pipeline(network, environment, encoding=bernoulli, + action_function=select_softmax, output='Output Layer', + time=100, history_length=1, delta=1, + plot_interval=1, render_interval=1) + + +# Train agent for 100 episodes. +print("Training: ") +for i in range(100): + pipeline.reset_() + # initialize episode reward + reward = 0 + while True: + pipeline.step() + reward += pipeline.reward + if pipeline.done: + break + print("Episode " + str(i) + " reward:", reward) + +# stop MSTDP +pipeline.network.connections[('Hidden Layer', 'Output Layer')].update_rule = None + +print("Testing: ") +for i in range(100): + pipeline.reset_() + # initialize episode reward + reward = 0 + while True: + pipeline.step() + reward += pipeline.reward + if pipeline.done: + break + print("Episode " + str(i) + " reward:", reward) \ No newline at end of file From f425497d8cf1f3dfa2967f8267f9b5bb1f09e73b Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 08:46:02 -0400 Subject: [PATCH 5/9] Bug fix --- examples/breakout/breakout_stdp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/breakout/breakout_stdp.py b/examples/breakout/breakout_stdp.py index 7809cfc2..9014cd2b 100644 --- a/examples/breakout/breakout_stdp.py +++ b/examples/breakout/breakout_stdp.py @@ -53,7 +53,7 @@ print("Episode " + str(i) + " reward:", reward) # stop MSTDP -pipeline.network.connections[('Hidden Layer', 'Output Layer')].update_rule = None +pipeline.network.learning = False print("Testing: ") for i in range(100): From 70f8920033c494c9a543a1ac5681e7cfca570b19 Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 14:05:52 -0400 Subject: [PATCH 6/9] remove default addition of monitor in pipeline --- bindsnet/pipeline/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bindsnet/pipeline/__init__.py b/bindsnet/pipeline/__init__.py index b675c579..ba5adf0f 100644 --- a/bindsnet/pipeline/__init__.py +++ b/bindsnet/pipeline/__init__.py @@ -75,20 +75,16 @@ def __init__(self, network: Network, environment: Environment, encoding: Callabl else: self.history = {} - # monitor spikes for selecting action based on sum of spikes - for l in self.network.layers: - self.network.add_monitor(Monitor(self.network.layers[l], 's', self.time), - name=f'{l}_spikes') - - self.spike_record = {l: torch.Tensor().byte() for l in self.network.layers} - self.set_spike_data() - if self.plot_interval is not None: for l in self.network.layers: + self.network.add_monitor(Monitor(self.network.layers[l], 's', self.time), + name=f'{l}_spikes') if 'v' in self.network.layers[l].__dict__: self.network.add_monitor(Monitor(self.network.layers[l], 'v', self.plot_interval * self.time), name=f'{l}_voltages') + self.spike_record = {l: torch.Tensor().byte() for l in self.network.layers} + self.set_spike_data() self.plot_data() # Set up for multiple layers of input layers. From 2a77d5e81621c9fc3a42cf1f0c77803cad2ea33e Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 14:15:49 -0400 Subject: [PATCH 7/9] remove breakout_stdp_right.py --- examples/breakout/breakout_stdp_right.py | 56 ------------------------ 1 file changed, 56 deletions(-) delete mode 100644 examples/breakout/breakout_stdp_right.py diff --git a/examples/breakout/breakout_stdp_right.py b/examples/breakout/breakout_stdp_right.py deleted file mode 100644 index 39bfd5cf..00000000 --- a/examples/breakout/breakout_stdp_right.py +++ /dev/null @@ -1,56 +0,0 @@ -import torch - -from bindsnet.network import Network -from bindsnet.pipeline import Pipeline -from bindsnet.learning import PostPre -from bindsnet.encoding import bernoulli -from bindsnet.network.topology import Connection -from bindsnet.environment import GymEnvironment -from bindsnet.network.nodes import Input, LIFNodes -from bindsnet.pipeline.action import select_softmax - -# Build network. -network = Network(dt=1.0) - -# Layers of neurons. -inpt = Input(n=80 * 80, shape=[80, 80], traces=True) -middle = LIFNodes(n=100, traces=True) -out = LIFNodes(n=4, refrac=0, traces=True) - -# Connections between layers. -inpt_middle = Connection(source=inpt, target=middle, wmin=0, wmax=1e-1) -middle_out = Connection(source=middle, target=out, wmin=0, wmax=1, update_rule=PostPre, nu=1e-1) - -# Add all layers and connections to the network. -network.add_layer(inpt, name='Input Layer') -network.add_layer(middle, name='Hidden Layer') -network.add_layer(out, name='Output Layer') -network.add_connection(inpt_middle, source='Input Layer', target='Hidden Layer') -network.add_connection(middle_out, source='Hidden Layer', target='Output Layer') - -# Load SpaceInvaders environment. -environment = GymEnvironment('BreakoutDeterministic-v4') -environment.reset() - -# Build pipeline from specified components. -pipeline = Pipeline(network, environment, encoding=bernoulli, - action_function=select_softmax, output='Output Layer', - time=100, history_length=1, delta=1, - plot_interval=1, render_interval=1) - - -# save the initial weights -initial_weights = network.connections[('Hidden Layer', 'Output Layer')].w.clone() -# Run environment simulation for 100 episodes. -for i in range(100): - # initialize episode reward - reward = 0 - pipeline.reset_() - while True: - pipeline.step() - # reset weights for action 0 = No action, 1 = Fire and 3 = Move left - network.connections[('Hidden Layer', 'Output Layer')].w[:, [0, 1, 3]] = initial_weights[:, [0, 1, 3]] - reward += pipeline.reward - if pipeline.done: - break - print("Episode " + str(i) + " reward:", reward) From 68bd2edca660a98a57f7f47d36c74f017d2138e1 Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 14:17:29 -0400 Subject: [PATCH 8/9] Error handle in select_softmax function --- bindsnet/pipeline/action.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bindsnet/pipeline/action.py b/bindsnet/pipeline/action.py index 477725a0..d38e8b3f 100644 --- a/bindsnet/pipeline/action.py +++ b/bindsnet/pipeline/action.py @@ -60,6 +60,8 @@ def select_softmax(pipeline: Pipeline, **kwargs) -> int: assert pipeline.network.layers[output].n == pipeline.env.action_space.n, \ 'Output layer size not equal to size of action space.' + assert hasattr(pipeline, 'spike_record'), 'Pipeline has not attribute named: spike_record.' + # Sum of previous iterations' spikes (Not yet implemented) spikes = torch.sum(pipeline.spike_record[output], dim=1) _sum = torch.sum(torch.exp(spikes.float())) From a3942d83b243e464f444071a114f21af5520420c Mon Sep 17 00:00:00 2001 From: Devdhar Patel Date: Fri, 26 Oct 2018 14:19:39 -0400 Subject: [PATCH 9/9] minor bug fix --- bindsnet/pipeline/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindsnet/pipeline/__init__.py b/bindsnet/pipeline/__init__.py index ba5adf0f..47f0824f 100644 --- a/bindsnet/pipeline/__init__.py +++ b/bindsnet/pipeline/__init__.py @@ -77,7 +77,7 @@ def __init__(self, network: Network, environment: Environment, encoding: Callabl if self.plot_interval is not None: for l in self.network.layers: - self.network.add_monitor(Monitor(self.network.layers[l], 's', self.time), + self.network.add_monitor(Monitor(self.network.layers[l], 's', self.plot_interval * self.time), name=f'{l}_spikes') if 'v' in self.network.layers[l].__dict__: self.network.add_monitor(Monitor(self.network.layers[l], 'v', self.plot_interval * self.time),