From ef753a208829d6f75dbdd0e74eaba93c48abfbaf Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sat, 6 Aug 2016 18:05:28 +0300
Subject: [PATCH 01/20] BP: added extension for neuron process

---
 .../back_prop/processes/neuron.ex             | 19 +++++++++++++++++++
 lib/neuron_system/processes/neuron.ex         |  1 +
 2 files changed, 20 insertions(+)
 create mode 100644 lib/neuron_system/back_prop/processes/neuron.ex

diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
new file mode 100644
index 0000000..3f12d47
--- /dev/null
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -0,0 +1,19 @@
+defmodule NeuronSystem.BackProp.Processes.Neuron do
+  defmacro __using__(opts \\ :empty) do
+    quote do
+      @spec back_prop(pid, {:output, float} | {:hidden}) :: :ok
+      def back_prop(pid, {:output, valid_output}) do
+        GenServer.cast(pid, {:back_prop, :output, valid_output})
+      end
+      def back_prop(pid, {:hidden}) do
+        GenServer.cast(pid, {:back_prop, :hidden})
+      end
+
+      def handle_cast({:back_prop, :output, valid_output}, state) do
+      end
+
+      def handle_cast({:back_prop, :hidden}, state) do
+      end
+    end
+  end
+end
diff --git a/lib/neuron_system/processes/neuron.ex b/lib/neuron_system/processes/neuron.ex
index 03969bb..1ed8e7c 100644
--- a/lib/neuron_system/processes/neuron.ex
+++ b/lib/neuron_system/processes/neuron.ex
@@ -6,6 +6,7 @@ defmodule NeuronSystem.Processes.Neuron do
   """
 
   use GenServer
+  # use NeuronSystem.BackProp.Processes.Neuron
 
   alias NeuronSystem.{Models, Processes, Utils}
 

From fdc9a9b2e4def02f464328f0639227545da3608a Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sat, 6 Aug 2016 18:54:44 +0300
Subject: [PATCH 02/20] Refactored forward way

* moved `root_pid` to the Net model. So we do not need to pass it via
the numeruos calls
* refactored income payloads processor
* extended neuron process state by additional options map
---
 lib/neuron_system/models/net.ex               |  2 +-
 lib/neuron_system/net.ex                      |  4 +-
 lib/neuron_system/net/activator.ex            | 14 +++---
 lib/neuron_system/processes/neuron.ex         | 14 +++---
 .../utils/neuron_income_payload_processor.ex  | 43 ++++++++++++++-----
 .../utils/send_to_neuron_proxy.ex             |  8 ++--
 6 files changed, 54 insertions(+), 31 deletions(-)

diff --git a/lib/neuron_system/models/net.ex b/lib/neuron_system/models/net.ex
index 8b639b7..9b4cc12 100644
--- a/lib/neuron_system/models/net.ex
+++ b/lib/neuron_system/models/net.ex
@@ -5,5 +5,5 @@ defmodule NeuronSystem.Models.Net do
   Contains the only one value - the PID of the supervisor process of the Net.
   """
 
-  defstruct [:pid]
+  defstruct [:pid, :root_pid]
 end
diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index c9c0db8..09d7b89 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -50,7 +50,7 @@ defmodule NeuronSystem.Net do
       "net_process"
     )
     {:ok, pid} = Supervisor.start_child(NeuronSystem.Supervisor, supervisor_spec)
-    %Models.Net{pid: pid}
+    %Models.Net{pid: pid, root_pid: self()}
   end
 
   @doc """
@@ -152,7 +152,7 @@ defmodule NeuronSystem.Net do
   """
   @spec activate!(Models.Net.t, map) :: list
   def activate!(net, income) do
-    NeuronSystem.Net.Activator.call(net, income, self())
+    NeuronSystem.Net.Activator.call(net, income)
   end
 
   @doc """
diff --git a/lib/neuron_system/net/activator.ex b/lib/neuron_system/net/activator.ex
index 4475e84..be20fdf 100644
--- a/lib/neuron_system/net/activator.ex
+++ b/lib/neuron_system/net/activator.ex
@@ -13,16 +13,16 @@ defmodule NeuronSystem.Net.Activator do
   @doc """
   The main point for an activator.
   """
-  @spec call(Models.Net.t, map, pid) :: list
-  def call(net, income, root_pid) do
-    net |> send_in_events(income, root_pid)
+  @spec call(Models.Net.t, map) :: list
+  def call(%Models.Net{} = net, income) do
+    net |> send_in_events(income)
     net |> collect_net_results
   end
 
-  defp send_in_events(net, income, root_pid) do
+  defp send_in_events(net, income) do
     net
     |> net_in_connections
-    |> Enum.each(&send_event(&1, net, income, root_pid))
+    |> Enum.each(&send_event(&1, net, income))
   end
 
   defp net_in_connections(%Models.Net{} = net) do
@@ -30,9 +30,9 @@ defmodule NeuronSystem.Net.Activator do
     |> Processes.ConnectionManager.get_net_in_connections
   end
 
-  defp send_event(connection, net, income, root_pid) do
+  defp send_event(connection, net, income) do
     value = income[connection.key]
-    NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, value, root_pid)
+    NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, value)
   end
 
   defp collect_net_results(net) do
diff --git a/lib/neuron_system/processes/neuron.ex b/lib/neuron_system/processes/neuron.ex
index 1ed8e7c..2b85f9b 100644
--- a/lib/neuron_system/processes/neuron.ex
+++ b/lib/neuron_system/processes/neuron.ex
@@ -12,7 +12,7 @@ defmodule NeuronSystem.Processes.Neuron do
 
   @spec start_link(NeuronSystem.Models.Neuron.t) :: any
   def start_link(neuron_model) do
-    GenServer.start_link(__MODULE__, {neuron_model, %{}})
+    GenServer.start_link(__MODULE__, {neuron_model, %{income_payloads: %{}}})
   end
 
   # Public API
@@ -34,15 +34,17 @@ defmodule NeuronSystem.Processes.Neuron do
   This event means that some neuron from the previous layer was activated and sent its value to
   a next layer.
   """
-  @spec income_payload(pid, {binary, float, Models.Net.t, pid}) :: :ok
-  def income_payload(pid, {source_neuron_id, value, net, root_pid}) do
-    GenServer.cast(pid, {:income_payload, source_neuron_id, value, net, root_pid})
+  @spec income_payload(pid, {binary, float, Models.Net.t}) :: :ok
+  def income_payload(pid, {source_neuron_id, value, net}) do
+    GenServer.cast(pid, {:income_payload, source_neuron_id, value, net})
   end
 
   # Callbacks
 
-  def handle_cast({:income_payload, source_neuron_id, value, net, root_pid}, state) do
-    new_state = NeuronSystem.Utils.NeuronIncomePayloadProcessor.call({source_neuron_id, value, net, root_pid}, state)
+  def handle_cast({:income_payload, source_neuron_id, value, net}, state) do
+    new_state = NeuronSystem.Utils.NeuronIncomePayloadProcessor.call(
+      source_neuron_id, value, net, state
+    )
     {:noreply, new_state}
   end
 
diff --git a/lib/neuron_system/utils/neuron_income_payload_processor.ex b/lib/neuron_system/utils/neuron_income_payload_processor.ex
index db461e9..95cdea3 100644
--- a/lib/neuron_system/utils/neuron_income_payload_processor.ex
+++ b/lib/neuron_system/utils/neuron_income_payload_processor.ex
@@ -1,19 +1,40 @@
 defmodule NeuronSystem.Utils.NeuronIncomePayloadProcessor do
+  @moduledoc """
+  Encapsulates the whole functional for processing income payload to a neuron.
+  """
+
   alias NeuronSystem.{Models, Processes, Utils}
 
-  def call({source_neuron_id, value, net, root_pid}, {%Models.Neuron{id: neuron_id} = neuron_model, income_payloads}) do
-    new_income_payloads = Map.put(income_payloads, source_neuron_id, value)
+  @type neuron_process_state :: {Models.Neuron.t, map}
+
+  @doc """
+  Calls income payload processor
+  """
+  @spec call(atom | bitstring, float, Models.Net.t, neuron_process_state) :: neuron_process_state
+  def call(source_neuron_id, value, net, {%Models.Neuron{id: neuron_id} = neuron_model, %{income_payloads: income_payloads} = options}) do
+    neuron_connections = get_neuron_connections(net, neuron_id)
+    new_income_payloads = income_payloads |> Map.put(source_neuron_id, value)
     payloads_count = new_income_payloads |> Map.keys |> Enum.count
+    in_connections_count = neuron_connections[:in] |> Enum.count
+    if in_connections_count == payloads_count do
+      send_out_messages(net, neuron_model, new_income_payloads, neuron_connections[:out])
+    end
+    new_options = options |> Map.put(:income_payloads, new_income_payloads)
+    {neuron_model, new_options}
+  end
+
+  defp get_neuron_connections(net, neuron_id) do
     manager_pid = NeuronSystem.Net.connection_manager(net)
     neuron_in_connections = Processes.ConnectionManager.get_neuron_in_connections(manager_pid, neuron_id)
-    if Enum.count(neuron_in_connections) == payloads_count do
-      payloads_sum_value = new_income_payloads |> Map.values |> Enum.sum
-      neuron_final_value = NeuronSystem.Neuron.activate(neuron_model, payloads_sum_value)
-      Processes.ConnectionManager.get_neuron_out_connections(manager_pid, neuron_id)
-      |> Enum.each(fn(out_connection) ->
-        Utils.SendToNeuronProxy.call(net, out_connection, neuron_final_value, root_pid)
-      end)
-    end
-    {neuron_model, new_income_payloads}
+    neuron_out_connections = Processes.ConnectionManager.get_neuron_out_connections(manager_pid, neuron_id)
+    %{in: neuron_in_connections, out: neuron_out_connections}
+  end
+
+  defp send_out_messages(net, neuron_model, income_payloads, out_connections) do
+    payloads_sum_value = income_payloads |> Map.values |> Enum.sum
+    neuron_final_value = NeuronSystem.Neuron.activate(neuron_model, payloads_sum_value)
+    out_connections |> Enum.each(fn(out_connection) ->
+      Utils.SendToNeuronProxy.call(net, out_connection, neuron_final_value)
+    end)
   end
 end
diff --git a/lib/neuron_system/utils/send_to_neuron_proxy.ex b/lib/neuron_system/utils/send_to_neuron_proxy.ex
index 8d449db..7e87b01 100644
--- a/lib/neuron_system/utils/send_to_neuron_proxy.ex
+++ b/lib/neuron_system/utils/send_to_neuron_proxy.ex
@@ -8,15 +8,15 @@ defmodule NeuronSystem.Utils.SendToNeuronProxy do
   @doc """
   Calls sending event processing
   """
-  @spec call(Models.Net.t, Models.InConnection.t | Models.OutConnection.t | Models.Connection.t, float, pid) :: any
-  def call(_net, %Models.OutConnection{} = connection, value, root_pid) do
+  @spec call(Models.Net.t, Models.InConnection.t | Models.OutConnection.t | Models.Connection.t, float) :: any
+  def call(%Models.Net{root_pid: root_pid} = _net, %Models.OutConnection{} = connection, value) do
     send root_pid, {:out_result, connection.key, value}
   end
-  def call(net, connection, value, root_pid) do
+  def call(%Models.Net{root_pid: root_pid} = net, connection, value) do
     neuron_source_key = NeuronSystem.Connection.source(connection)
     neuron_id = connection.target_neuron
     payload_value = value * connection.weight
     neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, neuron_id)
-    Processes.Neuron.income_payload(neuron_process_pid, {neuron_source_key, payload_value, net, root_pid})
+    Processes.Neuron.income_payload(neuron_process_pid, {neuron_source_key, payload_value, net})
   end
 end

From 425c3e474aab40a4102a78a4919fe8e288fef2d9 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sat, 6 Aug 2016 22:44:15 +0300
Subject: [PATCH 03/20] BP: added basic impl of Net extension

---
 lib/neuron_system/back_prop/net.ex                 | 14 ++++++++++++++
 .../back_prop/utils/back_prop_runner.ex            |  6 ++++++
 lib/neuron_system/net.ex                           |  2 ++
 lib/neuron_system/processes/neuron.ex              |  2 +-
 4 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 lib/neuron_system/back_prop/net.ex
 create mode 100644 lib/neuron_system/back_prop/utils/back_prop_runner.ex

diff --git a/lib/neuron_system/back_prop/net.ex b/lib/neuron_system/back_prop/net.ex
new file mode 100644
index 0000000..3a82e15
--- /dev/null
+++ b/lib/neuron_system/back_prop/net.ex
@@ -0,0 +1,14 @@
+defmodule NeuronSystem.BackProp.Net do
+  @moduledoc """
+  Represents an extension of the Net to support back propagation learning algorithm.
+  """
+
+  defmacro __using__(_opts \\ :empty) do
+    quote do
+      @spec back_prop!(NeuronSystem.Models.Net.t, map) :: :ok
+      def back_prop!(net, valid_output) do
+        NeuronSystem.BackProp.Utils.BackPropRunner.call(net, valid_output)       
+      end
+    end
+  end
+end
diff --git a/lib/neuron_system/back_prop/utils/back_prop_runner.ex b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
new file mode 100644
index 0000000..e28b59d
--- /dev/null
+++ b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
@@ -0,0 +1,6 @@
+defmodule NeuronSystem.BackProp.Utils.BackPropRunner do
+  @spec call(NeuronSystem.Models.Net.t, map) :: :ok
+  def call(net, valid_output) do
+    :ok   
+  end
+end
diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index 09d7b89..790e972 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -21,6 +21,8 @@ defmodule NeuronSystem.Net do
    ```
   """
 
+  use NeuronSystem.BackProp.Net
+
   import NeuronSystem.Utils.SpecHelper
 
   alias NeuronSystem.{Models, Processes}
diff --git a/lib/neuron_system/processes/neuron.ex b/lib/neuron_system/processes/neuron.ex
index 2b85f9b..6cc2f20 100644
--- a/lib/neuron_system/processes/neuron.ex
+++ b/lib/neuron_system/processes/neuron.ex
@@ -6,7 +6,7 @@ defmodule NeuronSystem.Processes.Neuron do
   """
 
   use GenServer
-  # use NeuronSystem.BackProp.Processes.Neuron
+  use NeuronSystem.BackProp.Processes.Neuron
 
   alias NeuronSystem.{Models, Processes, Utils}
 

From 8a815cb173741b501abb84a7756d64cf259a4ec7 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 12:13:32 +0300
Subject: [PATCH 04/20] Basic BP runner; added new options to the state

* added calculated OUT and dOUT values to the state
---
 .../back_prop/processes/neuron.ex             |  2 +-
 .../back_prop/utils/back_prop_runner.ex       | 41 ++++++++++++++++++-
 lib/neuron_system/net.ex                      | 18 ++++++++
 lib/neuron_system/net/activator.ex            | 14 +------
 lib/neuron_system/processes/neuron.ex         |  7 +++-
 .../utils/neuron_income_payload_processor.ex  |  9 ++--
 6 files changed, 72 insertions(+), 19 deletions(-)

diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
index 3f12d47..89cfb33 100644
--- a/lib/neuron_system/back_prop/processes/neuron.ex
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -9,7 +9,7 @@ defmodule NeuronSystem.BackProp.Processes.Neuron do
         GenServer.cast(pid, {:back_prop, :hidden})
       end
 
-      def handle_cast({:back_prop, :output, valid_output}, state) do
+      def handle_cast({:back_prop, :output, valid_output}, {neuron_model, options}) do
       end
 
       def handle_cast({:back_prop, :hidden}, state) do
diff --git a/lib/neuron_system/back_prop/utils/back_prop_runner.ex b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
index e28b59d..b8a2767 100644
--- a/lib/neuron_system/back_prop/utils/back_prop_runner.ex
+++ b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
@@ -1,6 +1,43 @@
 defmodule NeuronSystem.BackProp.Utils.BackPropRunner do
+  alias NeuronSystem.{Models, Processes}
+
   @spec call(NeuronSystem.Models.Net.t, map) :: :ok
-  def call(net, valid_output) do
-    :ok   
+  def call(%Models.Net{} = net, valid_output) do
+    net
+    |> send_propagation(valid_output)
+    |> collect_results
+  end
+
+  defp send_propagation(net, valid_output) do
+    net
+    |> NeuronSystem.Net.out_connections
+    |> Enum.each(&send_event_to_source_neuron(&1, net, valid_output))
+    net
+  end
+
+  def send_event_to_source_neuron(connection, net, valid_output) do
+    neuron_id = connection.source_neuron
+    neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, neuron_id)
+    needed_output = valid_output[connection.key]
+    Processes.Neuron.back_prop(neuron_process_pid, {:output, needed_output})
+  end
+
+  defp collect_results(net) do
+    net
+    |> detect_input_neurons
+    |> Enum.map(&collect_result_for_connection(&1))
+  end
+
+  def detect_input_neurons(net) do
+    net
+    |> NeuronSystem.Net.in_connections
+    |> Enum.map(&(&1.target_neuron))
+    |> Enum.uniq
+  end
+
+  defp collect_result_for_connection(neuron_id) do
+    receive do
+      {:back_prop_completed, ^neuron_id} -> neuron_id
+    end
   end
 end
diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index 790e972..64aefc0 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -170,6 +170,24 @@ defmodule NeuronSystem.Net do
     end)
   end
 
+  @doc """
+  Detects and returns net's output connections list
+  """
+  @spec out_connections(Models.Net.t) :: list
+  def out_connections(%Models.Net{pid: net_pid} = net) do
+    NeuronSystem.Net.connection_manager(net)
+    |> Processes.ConnectionManager.get_net_out_connections
+  end
+
+  @doc """
+  Detects and returns net's input connections list
+  """
+  @spec in_connections(Models.Net.t) :: list
+  def in_connections(%Models.Net{pid: net_pid} = net) do
+    NeuronSystem.Net.connection_manager(net)
+    |> Processes.ConnectionManager.get_net_in_connections
+  end
+
   defp detect_child_pid(net_pid, child_module) do
     worker_spec = Supervisor.which_children(net_pid)
                   |> Enum.find(fn({module, _pid, _type, _opts}) ->
diff --git a/lib/neuron_system/net/activator.ex b/lib/neuron_system/net/activator.ex
index be20fdf..bcac957 100644
--- a/lib/neuron_system/net/activator.ex
+++ b/lib/neuron_system/net/activator.ex
@@ -21,15 +21,10 @@ defmodule NeuronSystem.Net.Activator do
 
   defp send_in_events(net, income) do
     net
-    |> net_in_connections
+    |> NeuronSystem.Net.in_connections
     |> Enum.each(&send_event(&1, net, income))
   end
 
-  defp net_in_connections(%Models.Net{} = net) do
-    NeuronSystem.Net.connection_manager(net)
-    |> Processes.ConnectionManager.get_net_in_connections
-  end
-
   defp send_event(connection, net, income) do
     value = income[connection.key]
     NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, value)
@@ -37,15 +32,10 @@ defmodule NeuronSystem.Net.Activator do
 
   defp collect_net_results(net) do
     net
-    |> net_out_connections
+    |> NeuronSystem.Net.out_connections
     |> collect_results
   end
 
-  defp net_out_connections(%Models.Net{} = net) do
-    NeuronSystem.Net.connection_manager(net)
-    |> Processes.ConnectionManager.get_net_out_connections
-  end
-
   defp collect_results(connections) do
     connections |> Enum.map(&collect_result_for_connection(&1))
   end
diff --git a/lib/neuron_system/processes/neuron.ex b/lib/neuron_system/processes/neuron.ex
index 6cc2f20..d27e888 100644
--- a/lib/neuron_system/processes/neuron.ex
+++ b/lib/neuron_system/processes/neuron.ex
@@ -12,7 +12,12 @@ defmodule NeuronSystem.Processes.Neuron do
 
   @spec start_link(NeuronSystem.Models.Neuron.t) :: any
   def start_link(neuron_model) do
-    GenServer.start_link(__MODULE__, {neuron_model, %{income_payloads: %{}}})
+    options = %{
+      income_payloads: %{},
+      out_value: nil,
+      d_out_value: nil
+    }
+    GenServer.start_link(__MODULE__, {neuron_model, options})
   end
 
   # Public API
diff --git a/lib/neuron_system/utils/neuron_income_payload_processor.ex b/lib/neuron_system/utils/neuron_income_payload_processor.ex
index 95cdea3..7410bb3 100644
--- a/lib/neuron_system/utils/neuron_income_payload_processor.ex
+++ b/lib/neuron_system/utils/neuron_income_payload_processor.ex
@@ -17,7 +17,8 @@ defmodule NeuronSystem.Utils.NeuronIncomePayloadProcessor do
     payloads_count = new_income_payloads |> Map.keys |> Enum.count
     in_connections_count = neuron_connections[:in] |> Enum.count
     if in_connections_count == payloads_count do
-      send_out_messages(net, neuron_model, new_income_payloads, neuron_connections[:out])
+      {out_value, d_out_value} = send_out_messages(net, neuron_model, new_income_payloads, neuron_connections[:out])
+      options = options |> Map.put(:out_value, out_value) |> Map.put(:d_out_value, d_out_value)
     end
     new_options = options |> Map.put(:income_payloads, new_income_payloads)
     {neuron_model, new_options}
@@ -32,9 +33,11 @@ defmodule NeuronSystem.Utils.NeuronIncomePayloadProcessor do
 
   defp send_out_messages(net, neuron_model, income_payloads, out_connections) do
     payloads_sum_value = income_payloads |> Map.values |> Enum.sum
-    neuron_final_value = NeuronSystem.Neuron.activate(neuron_model, payloads_sum_value)
+    out_value = NeuronSystem.Neuron.activate(neuron_model, payloads_sum_value)
     out_connections |> Enum.each(fn(out_connection) ->
-      Utils.SendToNeuronProxy.call(net, out_connection, neuron_final_value)
+      Utils.SendToNeuronProxy.call(net, out_connection, out_value)
     end)
+    d_out_value = out_value * (1 - out_value)
+    {out_value, d_out_value}
   end
 end

From b80f421ddba6263c118ef4f6631b63f1aa012fb3 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 12:20:21 +0300
Subject: [PATCH 05/20] Added ID to the all connection types

---
 lib/neuron_system/models/connection.ex     | 4 +++-
 lib/neuron_system/models/in_connection.ex  | 4 +++-
 lib/neuron_system/models/out_connection.ex | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/neuron_system/models/connection.ex b/lib/neuron_system/models/connection.ex
index a6e82af..9688355 100644
--- a/lib/neuron_system/models/connection.ex
+++ b/lib/neuron_system/models/connection.ex
@@ -6,9 +6,10 @@ defmodule NeuronSystem.Models.Connection do
   its weight.
   """
 
-  defstruct [:source_neuron, :target_neuron, :weight]
+  defstruct [:id, :source_neuron, :target_neuron, :weight]
 
   alias NeuronSystem.Models
+  alias NeuronSystem.Utils.CommonHelper
 
   @type t :: %__MODULE__{}
 
@@ -18,6 +19,7 @@ defmodule NeuronSystem.Models.Connection do
   @spec build(Models.Neuron.t, Models.Neuron.t, float) :: Models.Connection.t
   def build(%Models.Neuron{id: source_id}, %Models.Neuron{id: target_id}, weight) do
     %__MODULE__{
+      id: CommonHelper.gen_process_id("connection"),
       source_neuron: source_id,
       target_neuron: target_id,
       weight: weight
diff --git a/lib/neuron_system/models/in_connection.ex b/lib/neuron_system/models/in_connection.ex
index 2a07db5..f816cd6 100644
--- a/lib/neuron_system/models/in_connection.ex
+++ b/lib/neuron_system/models/in_connection.ex
@@ -3,9 +3,10 @@ defmodule NeuronSystem.Models.InConnection do
   Represents an input connection for a Net.
   """
 
-  defstruct [:target_neuron, :weight, :key]
+  defstruct [:id, :target_neuron, :weight, :key]
 
   alias NeuronSystem.Models
+  alias NeuronSystem.Utils.CommonHelper
 
   @type t :: %__MODULE__{}
 
@@ -15,6 +16,7 @@ defmodule NeuronSystem.Models.InConnection do
   @spec build(Models.Neuron.t, float, atom) :: __MODULE__.t
   def build(%Models.Neuron{id: neuron_id}, weight, key) do
     %__MODULE__{
+      id: CommonHelper.gen_process_id("connection"),
       target_neuron: neuron_id,
       weight: weight,
       key: key
diff --git a/lib/neuron_system/models/out_connection.ex b/lib/neuron_system/models/out_connection.ex
index 2aef76f..c29da98 100644
--- a/lib/neuron_system/models/out_connection.ex
+++ b/lib/neuron_system/models/out_connection.ex
@@ -3,9 +3,10 @@ defmodule NeuronSystem.Models.OutConnection do
   Represents an output connection for a Net.
   """
 
-  defstruct [:source_neuron, :weight, :key]
+  defstruct [:id, :source_neuron, :weight, :key]
 
   alias NeuronSystem.Models
+  alias NeuronSystem.Utils.CommonHelper
 
   @type t :: %__MODULE__{}
 
@@ -15,6 +16,7 @@ defmodule NeuronSystem.Models.OutConnection do
   @spec build(Models.Neuron.t, float, atom) :: __MODULE__.t
   def build(%Models.Neuron{id: neuron_id}, weight, key) do
     %__MODULE__{
+      id: CommonHelper.gen_process_id("connection"),
       source_neuron: neuron_id,
       weight: weight,
       key: key

From 36faf4076f79dd14f2978f852923013559cb61c0 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 12:47:40 +0300
Subject: [PATCH 06/20] BP: basic impl of output neuron processor

---
 lib/neuron_system/back_prop/neuron/output_processor.ex |  4 ++++
 lib/neuron_system/back_prop/processes/neuron.ex        | 10 +++++++---
 lib/neuron_system/back_prop/utils/back_prop_runner.ex  |  2 +-
 3 files changed, 12 insertions(+), 4 deletions(-)
 create mode 100644 lib/neuron_system/back_prop/neuron/output_processor.ex

diff --git a/lib/neuron_system/back_prop/neuron/output_processor.ex b/lib/neuron_system/back_prop/neuron/output_processor.ex
new file mode 100644
index 0000000..25029a5
--- /dev/null
+++ b/lib/neuron_system/back_prop/neuron/output_processor.ex
@@ -0,0 +1,4 @@
+defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
+  def call(net, valid_output, {neuron_model, options}) do
+  end
+end
diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
index 89cfb33..01e5c42 100644
--- a/lib/neuron_system/back_prop/processes/neuron.ex
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -2,14 +2,18 @@ defmodule NeuronSystem.BackProp.Processes.Neuron do
   defmacro __using__(opts \\ :empty) do
     quote do
       @spec back_prop(pid, {:output, float} | {:hidden}) :: :ok
-      def back_prop(pid, {:output, valid_output}) do
-        GenServer.cast(pid, {:back_prop, :output, valid_output})
+      def back_prop(pid, {:output, net, valid_output}) do
+        GenServer.cast(pid, {:back_prop, :output, net, valid_output})
       end
       def back_prop(pid, {:hidden}) do
         GenServer.cast(pid, {:back_prop, :hidden})
       end
 
-      def handle_cast({:back_prop, :output, valid_output}, {neuron_model, options}) do
+      def handle_cast({:back_prop, :output, net, valid_output}, {_neuron_model, _options} = state) do
+        new_state = NeuronSystem.BackProp.Neuron.OutputProcessor.call(
+          net, valid_output, state
+        )
+        {:noreply, new_state}
       end
 
       def handle_cast({:back_prop, :hidden}, state) do
diff --git a/lib/neuron_system/back_prop/utils/back_prop_runner.ex b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
index b8a2767..dda183f 100644
--- a/lib/neuron_system/back_prop/utils/back_prop_runner.ex
+++ b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
@@ -19,7 +19,7 @@ defmodule NeuronSystem.BackProp.Utils.BackPropRunner do
     neuron_id = connection.source_neuron
     neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, neuron_id)
     needed_output = valid_output[connection.key]
-    Processes.Neuron.back_prop(neuron_process_pid, {:output, needed_output})
+    Processes.Neuron.back_prop(neuron_process_pid, {:output, net, needed_output})
   end
 
   defp collect_results(net) do

From 62142139060ad068090c50d3baf37a5946f84ab1 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 12:51:05 +0300
Subject: [PATCH 07/20] Fixed income payloads clear procedure

---
 lib/neuron_system/processes/neuron.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/neuron_system/processes/neuron.ex b/lib/neuron_system/processes/neuron.ex
index d27e888..003e55a 100644
--- a/lib/neuron_system/processes/neuron.ex
+++ b/lib/neuron_system/processes/neuron.ex
@@ -53,7 +53,8 @@ defmodule NeuronSystem.Processes.Neuron do
     {:noreply, new_state}
   end
 
-  def handle_cast(:clear_income_payloads, {neuron_model, _income_payloads}) do
-    {:noreply, {neuron_model, %{}}}
+  def handle_cast(:clear_income_payloads, {neuron_model, options}) do
+    new_options = options |> Map.put(:income_payloads, %{})
+    {:noreply, {neuron_model, new_options}}
   end
 end

From ae26a9a4d8ffe51ddfe4c27768c2db7b6e67f9e6 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 13:14:54 +0300
Subject: [PATCH 08/20] Added ability to change weight of the connection

---
 lib/neuron_system/processes/connection_manager.ex | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/neuron_system/processes/connection_manager.ex b/lib/neuron_system/processes/connection_manager.ex
index 2407ae6..bc3b060 100644
--- a/lib/neuron_system/processes/connection_manager.ex
+++ b/lib/neuron_system/processes/connection_manager.ex
@@ -53,6 +53,14 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     GenServer.call(manager_pid, {:get_neuron_in_out, :out, neuron_id})
   end
 
+  @doc """
+  Changes weight of the specific connection
+  """
+  @spec change_weight(pid, bitstring, float) :: :ok
+  def change_weight(manager_pid, connection_id, new_weight) do
+    GenServer.cast(manager_pid, {:change_weight, connection_id, new_weight})
+  end
+
   def handle_cast({:add, connection}, state) do
     new_state = [connection | state]
     {:noreply, new_state}
@@ -76,6 +84,13 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     {:reply, connections, state} 
   end
 
+  def handle_cast({:change_weight, connection_id, new_weight}, state) do
+    connection = state |> Enum.filter(&(&1.id == connection_id)) |> Enum.first
+    other_connections = state |> Enum.filter(&(&1.id != connection_id))
+    new_connection = %{connection | weight: new_weight}
+    {:noreply, [new_connection | other_connections]}
+  end
+
   defp filter_by_type(collection, module) do
     collection
     |> Enum.filter(fn(x) -> x.__struct__ == module end)

From 5d684c9bafa5f4eacc58ac85d71625984e31f175 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 7 Aug 2016 22:07:48 +0300
Subject: [PATCH 09/20] BP: mostly completed output neuron processor

---
 .../back_prop/neuron/output_processor.ex      | 23 ++++++++++++++++++-
 lib/neuron_system/net.ex                      | 19 +++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/lib/neuron_system/back_prop/neuron/output_processor.ex b/lib/neuron_system/back_prop/neuron/output_processor.ex
index 25029a5..6ef5c3f 100644
--- a/lib/neuron_system/back_prop/neuron/output_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/output_processor.ex
@@ -1,4 +1,25 @@
 defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
-  def call(net, valid_output, {neuron_model, options}) do
+  alias NeuronSystem.{Processes, Models}
+
+  @learn_speed 0.4
+
+  def call(net, valid_output, {%Models.Neuron{id: neuron_id} = neuron_model, options}) do
+    lapse = calc_lapse(valid_output, options)
+    income_payloads = options |> Map.get(:income_payloads)
+    NeuronSystem.Net.neuron_out_connections(net, neuron_id)
+    |> Enum.each(fn(connection) ->
+      connection_source = NeuronSystem.Connection.source(connection)
+      connection_income = income_payloads[connection_source]
+      delta_w = @learn_speed * lapse * connection_income
+      new_w = connection.weight + delta_w
+      connection_lapse = lapse * connection.weight
+      # TODO: Send messages to the hidden layer
+    end)
+  end
+
+  defp calc_lapse(valid_output, options) do
+    out_value = options |> Map.get(:out_value)
+    d_out_value = options |> Map.get(:d_out_value)
+    (valid_output - out_value) * d_out_value
   end
 end
diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index 64aefc0..9bb9a07 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -188,6 +188,25 @@ defmodule NeuronSystem.Net do
     |> Processes.ConnectionManager.get_net_in_connections
   end
 
+  @doc """
+  Fetches out connections for the given neuron in the Net
+  """
+  @spec neuron_out_connections(Models.Net.t, bitstring) :: list
+  def neuron_out_connections(%Models.Net{pid: net_pid} = net, neuron_id) do
+    NeuronSystem.Net.connection_manager(net)
+    |> Processes.ConnectionManager.get_neuron_out_connections(neuron_id)
+  end
+
+  @doc """
+  Changes a weight of a given connection and saves it in a connection manager.
+  """
+  @spec set_connection_weight(Models.Net.t, Models.InConnection.t | Models.OutConnection.t | Models.Connection.t, float) :: :ok
+  def set_connection_weight(net, connection, new_weight) do
+    net
+    |> connection_manager
+    |> Processes.ConnectionManager.change_weight(connection.id, new_weight)
+  end
+
   defp detect_child_pid(net_pid, child_module) do
     worker_spec = Supervisor.which_children(net_pid)
                   |> Enum.find(fn({module, _pid, _type, _opts}) ->

From ecb63f078f3b9c3d521e51779cff114b2c31af34 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Mon, 8 Aug 2016 20:24:58 +0300
Subject: [PATCH 10/20] Completed output neuron processor

---
 lib/neuron_system/back_prop/net.ex            | 19 +++++++
 .../back_prop/neuron/output_processor.ex      | 49 ++++++++++++++-----
 .../back_prop/utils/back_prop_runner.ex       |  7 +++
 .../back_prop/utils/calculations.ex           | 39 +++++++++++++++
 4 files changed, 102 insertions(+), 12 deletions(-)
 create mode 100644 lib/neuron_system/back_prop/utils/calculations.ex

diff --git a/lib/neuron_system/back_prop/net.ex b/lib/neuron_system/back_prop/net.ex
index 3a82e15..29fd371 100644
--- a/lib/neuron_system/back_prop/net.ex
+++ b/lib/neuron_system/back_prop/net.ex
@@ -1,6 +1,25 @@
 defmodule NeuronSystem.BackProp.Net do
   @moduledoc """
   Represents an extension of the Net to support back propagation learning algorithm.
+
+  ## Usage
+
+  Use this module in the origin Net module:
+
+  ```elixir
+  defmodule NeuronSystem.Net do
+    ...
+    use NeuronSystem.BackProp.Net
+    ...
+  end
+  ```
+
+  After that you can learn your net by calling:
+  ```elixir
+  net = ... # Net creation flow
+  # Add neurons, connections...
+  NeuronSystem.Net.back_prop!(net, %{y: 1})
+  ```
   """
 
   defmacro __using__(_opts \\ :empty) do
diff --git a/lib/neuron_system/back_prop/neuron/output_processor.ex b/lib/neuron_system/back_prop/neuron/output_processor.ex
index 6ef5c3f..11a4a68 100644
--- a/lib/neuron_system/back_prop/neuron/output_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/output_processor.ex
@@ -1,25 +1,50 @@
 defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
-  alias NeuronSystem.{Processes, Models}
+  @moduledoc """
+  Processor of Back Propagation algorithm messages for an output neuron.
+
+  The algorithm for the output neurons is a little bit different then for the input and hidden ones
+  and contains the following steps:
+  
+  1. Calculate a lapse value using an needed output value and value after a forward pass.
+  2. For each neuron's input connection calculate a delta value.
+  3. Calculate a new weight for each ouput connection
+  4. Change weights of all input connections in a Connection Manager
+  5. Send messages to hidden neurons with needed calculated values: in this particular case it is `lapse`.
+  """
 
-  @learn_speed 0.4
+  alias NeuronSystem.{Processes, Models}
+  alias NeuronSystem.BackProp.Utils
 
-  def call(net, valid_output, {%Models.Neuron{id: neuron_id} = neuron_model, options}) do
+  @doc """
+  Calls processor for an output neuron.
+  """
+  def call(net, valid_output, {%Models.Neuron{id: neuron_id} = neuron_model, %{income_payloads: income_payloads} = options}) do
     lapse = calc_lapse(valid_output, options)
-    income_payloads = options |> Map.get(:income_payloads)
-    NeuronSystem.Net.neuron_out_connections(net, neuron_id)
+    NeuronSystem.Net.neuron_in_connections(net, neuron_id)
     |> Enum.each(fn(connection) ->
-      connection_source = NeuronSystem.Connection.source(connection)
-      connection_income = income_payloads[connection_source]
-      delta_w = @learn_speed * lapse * connection_income
-      new_w = connection.weight + delta_w
-      connection_lapse = lapse * connection.weight
-      # TODO: Send messages to the hidden layer
+      connection_income = fetch_income_for_connection(connection, income_payloads)
+      delta_w = Utils.Calculations.delta_weight(lapse, connection_income)
+      new_w = Utils.Calculations.new_weight(connection.weight, delta_w)
+      connection_lapse = Utils.Calculations.connection_lapse(lapse, connection.weight)
+      NeuronSystem.Net.set_connection_weight(net, connection, new_w)
+      send_back_prop_inside(net, connection, lapse)
     end)
   end
 
   defp calc_lapse(valid_output, options) do
     out_value = options |> Map.get(:out_value)
     d_out_value = options |> Map.get(:d_out_value)
-    (valid_output - out_value) * d_out_value
+    Utils.Calculations.out_lapse(out_value, d_out_value, valid_output)
+  end
+
+  defp fetch_income_for_connection(connection, income_payloads) do
+    connection_source = NeuronSystem.Connection.source(connection)
+    income_payloads[connection_source]
+  end
+
+  defp send_back_prop_inside(net, connection, lapse) do
+    connection_source = NeuronSystem.Connection.source(connection)
+    neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, connection_source)
+    Processes.Neuron.back_prop(neuron_process_pid, {:hidden, net, lapse})
   end
 end
diff --git a/lib/neuron_system/back_prop/utils/back_prop_runner.ex b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
index dda183f..a9505f6 100644
--- a/lib/neuron_system/back_prop/utils/back_prop_runner.ex
+++ b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
@@ -1,6 +1,13 @@
 defmodule NeuronSystem.BackProp.Utils.BackPropRunner do
+  @moduledoc """
+
+  """
+
   alias NeuronSystem.{Models, Processes}
 
+  @doc """
+  Starts back propagation learning procedure for a specific Net using a some pair from a learning set.
+  """
   @spec call(NeuronSystem.Models.Net.t, map) :: :ok
   def call(%Models.Net{} = net, valid_output) do
     net
diff --git a/lib/neuron_system/back_prop/utils/calculations.ex b/lib/neuron_system/back_prop/utils/calculations.ex
new file mode 100644
index 0000000..4f70f99
--- /dev/null
+++ b/lib/neuron_system/back_prop/utils/calculations.ex
@@ -0,0 +1,39 @@
+defmodule NeuronSystem.BackProp.Utils.Calculations do
+  @moduledoc """
+  In this module are encapsulated all calculations for Back Prop algorithm.
+  """
+
+  @learning_speed 0.4
+
+  @doc """
+  Calculates a lapse for an output neuron.
+  """
+  @spec out_lapse(float, float, float) :: float
+  def out_lapse(calculated_output, d_calculated_output, needed_output) do
+    (needed_output - calculated_output) * d_calculated_output
+  end
+
+  @doc """
+  Calculates a delta value for a weight of a connection.
+  """
+  @spec delta_weight(float, float) :: float
+  def delta_weight(lapse, income) do
+    @learning_speed * lapse * income
+  end
+
+  @doc """
+  Calculates new weight of a connection by old one and delta value.
+  """
+  @spec new_weight(float, float) :: float
+  def new_weight(weight, delta_w) do
+    weight + delta_w
+  end
+
+  @doc """
+  Calculates connection's lapse by output lapse and connection's weight.
+  """
+  @spec connection_lapse(float, float) :: float
+  def connection_lapse(lapse, weight) do
+    lapse * weight
+  end
+end

From 0ce324469c9795734cc64f769acf123703544407 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Mon, 8 Aug 2016 20:55:07 +0300
Subject: [PATCH 11/20] Got rid off state changing when output neuron

---
 lib/neuron_system/back_prop/neuron/output_processor.ex | 1 +
 lib/neuron_system/back_prop/processes/neuron.ex        | 6 ++----
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/neuron_system/back_prop/neuron/output_processor.ex b/lib/neuron_system/back_prop/neuron/output_processor.ex
index 11a4a68..72e6563 100644
--- a/lib/neuron_system/back_prop/neuron/output_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/output_processor.ex
@@ -29,6 +29,7 @@ defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
       NeuronSystem.Net.set_connection_weight(net, connection, new_w)
       send_back_prop_inside(net, connection, lapse)
     end)
+
   end
 
   defp calc_lapse(valid_output, options) do
diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
index 01e5c42..21706b1 100644
--- a/lib/neuron_system/back_prop/processes/neuron.ex
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -10,10 +10,8 @@ defmodule NeuronSystem.BackProp.Processes.Neuron do
       end
 
       def handle_cast({:back_prop, :output, net, valid_output}, {_neuron_model, _options} = state) do
-        new_state = NeuronSystem.BackProp.Neuron.OutputProcessor.call(
-          net, valid_output, state
-        )
-        {:noreply, new_state}
+        NeuronSystem.BackProp.Neuron.OutputProcessor.call(net, valid_output, state)
+        {:noreply, state}
       end
 
       def handle_cast({:back_prop, :hidden}, state) do

From 97d0f908273e0418f05f1b895cd51db59d8342c6 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Mon, 8 Aug 2016 21:01:33 +0300
Subject: [PATCH 12/20] Added some docs for back prop extnsions

---
 lib/neuron_system/back_prop/processes/neuron.ex       | 10 ++++++++--
 lib/neuron_system/back_prop/utils/back_prop_runner.ex |  6 ++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
index 21706b1..845481c 100644
--- a/lib/neuron_system/back_prop/processes/neuron.ex
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -1,12 +1,18 @@
 defmodule NeuronSystem.BackProp.Processes.Neuron do
+  @moduledoc """
+  Acts as an extension for an original Neuron process.
+
+  This extension provides additional API specifically for Back Propagation feature.
+  """
+
   defmacro __using__(opts \\ :empty) do
     quote do
       @spec back_prop(pid, {:output, float} | {:hidden}) :: :ok
       def back_prop(pid, {:output, net, valid_output}) do
         GenServer.cast(pid, {:back_prop, :output, net, valid_output})
       end
-      def back_prop(pid, {:hidden}) do
-        GenServer.cast(pid, {:back_prop, :hidden})
+      def back_prop(pid, {:hidden, net, lapse}) do
+        GenServer.cast(pid, {:back_prop, :hidden, net, lapse})
       end
 
       def handle_cast({:back_prop, :output, net, valid_output}, {_neuron_model, _options} = state) do
diff --git a/lib/neuron_system/back_prop/utils/back_prop_runner.ex b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
index a9505f6..f428dbf 100644
--- a/lib/neuron_system/back_prop/utils/back_prop_runner.ex
+++ b/lib/neuron_system/back_prop/utils/back_prop_runner.ex
@@ -1,6 +1,12 @@
 defmodule NeuronSystem.BackProp.Utils.BackPropRunner do
   @moduledoc """
+  Contains logic to start Back Propagation learning algorithm.
 
+  From very superficial sight an algorithm steps are:
+
+  1. Detect output connection of a Net
+  2. Send to each output neuron a message with a needed value inside
+  3. Put a receive to wait for learning end
   """
 
   alias NeuronSystem.{Models, Processes}

From 01b9b39a575e81d33cce4b270a3f3ecd0b09d5de Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 14 Aug 2016 16:06:55 +0300
Subject: [PATCH 13/20] Mostly completed Back Prop learning algorithm

---
 .../back_prop/neuron/hidden_processor.ex      | 61 +++++++++++++++++++
 .../back_prop/neuron/output_processor.ex      |  7 +--
 .../back_prop/processes/neuron.ex             |  8 ++-
 3 files changed, 69 insertions(+), 7 deletions(-)
 create mode 100644 lib/neuron_system/back_prop/neuron/hidden_processor.ex

diff --git a/lib/neuron_system/back_prop/neuron/hidden_processor.ex b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
new file mode 100644
index 0000000..dc876c5
--- /dev/null
+++ b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
@@ -0,0 +1,61 @@
+defmodule NeuronSystem.BackProp.Neuron.HiddenProcessor do
+  alias NeuronSystem.{Models, Processes}
+  alias NeuronSystem.BackProp.Utils
+
+  def call(net, from_neuron_id, lapse, {%Models.Neuron{id: neuron_id} = neuron_model, options}) do
+    new_options = options |> push_new_income_lapse(from_neuron_id, lapse)
+    neuron_out_connections = NeuronSystem.Net.neuron_out_connections(net, neuron_id) 
+    if received_all?(new_options, neuron_out_connections) do
+      lapse = calc_overall_lapse(new_options)
+      neuron_in_connections = NeuronSystem.Net.neuron_in_connections(net, neuron_id)
+      neuron_in_connections |> Enum.each(fn(connection) ->
+        connection_income = fetch_income_for_connection(connection, income_payloads)
+        delta_w = Utils.Calculations.delta_weight(lapse, connection_income)
+        new_w = Utils.Calculations.new_weight(connection.weight, delta_w)
+        connection_lapse = Utils.Calculations.connection_lapse(lapse, connection.weight)
+        NeuronSystem.Net.set_connection_weight(net, connection, new_w)
+        if %Models.Connection{} = connection do
+          send_back_prop_inside(net, neuron_id, connection, connection_lapse)
+        end
+      end)
+      if at_least_one_is_in_connection?(neuron_in_connections) do
+        send_back_prop_completed(net, neuron_id)
+      end
+    end
+    {neuron_model, new_options}
+  end
+
+  @spec push_new_income_lapse(map, bitstring, float) :: map
+  defp push_new_income_lapse(options, from_neuron_id, lapse) do
+    income_lapse = options |> Map.get(:income_lapse, %{})
+    new_income_lapse = income_lapse |> Map.put(from_neuron_id, lapse)
+    Map.put(options, :income_lapse, new_income_lapse)
+  end
+
+  defp received_all?(%{income_lapse: income_lapse} = _options, out_connections) do
+    income_count = income_lapse |> Map.keys |> Enum.count
+    connections_count = out_connections |> Enum.count
+    income_count == connections_count
+  end
+
+  defp calc_overall_lapse(%{income_lapse: income_lapse} = _options) do
+    income_lapse |> Map.values |> Enum.sum
+  end
+
+  defp send_back_prop_inside(net, neuron_id, connection, connection_lapse) do
+    connection_source = NeuronSystem.Connection.source(connection)
+    neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, connection_source)
+    Processes.Neuron.back_prop(neuron_process_pid, {:hidden, net, neuron_id, lapse})
+  end
+
+  defp at_least_one_is_in_connection?(connections) do
+    Enum.any?(neuron_in_connections, fn
+      (%Models.InConnection{} = conn) -> true
+      _ -> false
+    end)
+  end
+
+  defp send_back_prop_completed(%Models.Net{root_pid: root_pid} = _net, neuron_id) do
+    send root_pid, {:back_prop_completed, neuron_id}
+  end
+end
diff --git a/lib/neuron_system/back_prop/neuron/output_processor.ex b/lib/neuron_system/back_prop/neuron/output_processor.ex
index 72e6563..5e1ee44 100644
--- a/lib/neuron_system/back_prop/neuron/output_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/output_processor.ex
@@ -27,9 +27,8 @@ defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
       new_w = Utils.Calculations.new_weight(connection.weight, delta_w)
       connection_lapse = Utils.Calculations.connection_lapse(lapse, connection.weight)
       NeuronSystem.Net.set_connection_weight(net, connection, new_w)
-      send_back_prop_inside(net, connection, lapse)
+      send_back_prop_inside(net, neuron_id, connection, connection_lapse)
     end)
-
   end
 
   defp calc_lapse(valid_output, options) do
@@ -43,9 +42,9 @@ defmodule NeuronSystem.BackProp.Neuron.OutputProcessor do
     income_payloads[connection_source]
   end
 
-  defp send_back_prop_inside(net, connection, lapse) do
+  defp send_back_prop_inside(net, neuron_id, connection, lapse) do
     connection_source = NeuronSystem.Connection.source(connection)
     neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, connection_source)
-    Processes.Neuron.back_prop(neuron_process_pid, {:hidden, net, lapse})
+    Processes.Neuron.back_prop(neuron_process_pid, {:hidden, net, neuron_id, lapse})
   end
 end
diff --git a/lib/neuron_system/back_prop/processes/neuron.ex b/lib/neuron_system/back_prop/processes/neuron.ex
index 845481c..fd06d97 100644
--- a/lib/neuron_system/back_prop/processes/neuron.ex
+++ b/lib/neuron_system/back_prop/processes/neuron.ex
@@ -7,7 +7,7 @@ defmodule NeuronSystem.BackProp.Processes.Neuron do
 
   defmacro __using__(opts \\ :empty) do
     quote do
-      @spec back_prop(pid, {:output, float} | {:hidden}) :: :ok
+      @spec back_prop(pid, {:output | :hidden, NeuronSystem.Models.Net.t, float}) :: :ok
       def back_prop(pid, {:output, net, valid_output}) do
         GenServer.cast(pid, {:back_prop, :output, net, valid_output})
       end
@@ -15,12 +15,14 @@ defmodule NeuronSystem.BackProp.Processes.Neuron do
         GenServer.cast(pid, {:back_prop, :hidden, net, lapse})
       end
 
-      def handle_cast({:back_prop, :output, net, valid_output}, {_neuron_model, _options} = state) do
+      def handle_cast({:back_prop, :output, net, valid_output}, state) do
         NeuronSystem.BackProp.Neuron.OutputProcessor.call(net, valid_output, state)
         {:noreply, state}
       end
 
-      def handle_cast({:back_prop, :hidden}, state) do
+      def handle_cast({:back_prop, :hidden, net, from_neuron_id, lapse}, state) do
+        new_state = NeuronSystem.BackProp.Neuron.HiddenProcessor.call(net, from_neuron_id, lapse, state)
+        {:noreply, state}
       end
     end
   end

From 8e2345e56caddaa34e6cfb2b182c06f7e4eaba5b Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 14 Aug 2016 16:16:31 +0300
Subject: [PATCH 14/20] Fixed all inconsistences and warnings

---
 .../back_prop/neuron/hidden_processor.ex          | 10 ++++++++--
 lib/neuron_system/net.ex                          | 15 ++++++++++++---
 lib/neuron_system/processes/connection_manager.ex | 11 +++++------
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/lib/neuron_system/back_prop/neuron/hidden_processor.ex b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
index dc876c5..0dc62e5 100644
--- a/lib/neuron_system/back_prop/neuron/hidden_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
@@ -7,6 +7,7 @@ defmodule NeuronSystem.BackProp.Neuron.HiddenProcessor do
     neuron_out_connections = NeuronSystem.Net.neuron_out_connections(net, neuron_id) 
     if received_all?(new_options, neuron_out_connections) do
       lapse = calc_overall_lapse(new_options)
+      income_payloads = options |> Map.get(:income_payloads)
       neuron_in_connections = NeuronSystem.Net.neuron_in_connections(net, neuron_id)
       neuron_in_connections |> Enum.each(fn(connection) ->
         connection_income = fetch_income_for_connection(connection, income_payloads)
@@ -38,18 +39,23 @@ defmodule NeuronSystem.BackProp.Neuron.HiddenProcessor do
     income_count == connections_count
   end
 
+  defp fetch_income_for_connection(connection, income_payloads) do
+    connection_source = NeuronSystem.Connection.source(connection)
+    income_payloads[connection_source]
+  end
+
   defp calc_overall_lapse(%{income_lapse: income_lapse} = _options) do
     income_lapse |> Map.values |> Enum.sum
   end
 
-  defp send_back_prop_inside(net, neuron_id, connection, connection_lapse) do
+  defp send_back_prop_inside(net, neuron_id, connection, lapse) do
     connection_source = NeuronSystem.Connection.source(connection)
     neuron_process_pid = NeuronSystem.Net.neuron_process_pid(net, connection_source)
     Processes.Neuron.back_prop(neuron_process_pid, {:hidden, net, neuron_id, lapse})
   end
 
   defp at_least_one_is_in_connection?(connections) do
-    Enum.any?(neuron_in_connections, fn
+    Enum.any?(connections, fn
       (%Models.InConnection{} = conn) -> true
       _ -> false
     end)
diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index 9bb9a07..2306dff 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -174,7 +174,7 @@ defmodule NeuronSystem.Net do
   Detects and returns net's output connections list
   """
   @spec out_connections(Models.Net.t) :: list
-  def out_connections(%Models.Net{pid: net_pid} = net) do
+  def out_connections(%Models.Net{} = net) do
     NeuronSystem.Net.connection_manager(net)
     |> Processes.ConnectionManager.get_net_out_connections
   end
@@ -183,7 +183,7 @@ defmodule NeuronSystem.Net do
   Detects and returns net's input connections list
   """
   @spec in_connections(Models.Net.t) :: list
-  def in_connections(%Models.Net{pid: net_pid} = net) do
+  def in_connections(%Models.Net{} = net) do
     NeuronSystem.Net.connection_manager(net)
     |> Processes.ConnectionManager.get_net_in_connections
   end
@@ -192,11 +192,20 @@ defmodule NeuronSystem.Net do
   Fetches out connections for the given neuron in the Net
   """
   @spec neuron_out_connections(Models.Net.t, bitstring) :: list
-  def neuron_out_connections(%Models.Net{pid: net_pid} = net, neuron_id) do
+  def neuron_out_connections(%Models.Net{} = net, neuron_id) do
     NeuronSystem.Net.connection_manager(net)
     |> Processes.ConnectionManager.get_neuron_out_connections(neuron_id)
   end
 
+  @doc """
+  Fetches in connections for the given neuron in the Net
+  """
+  @spec neuron_in_connections(Models.Net.t, bitstring) :: list
+  def neuron_in_connections(%Models.Net{} = net, neuron_id) do
+    NeuronSystem.Net.connection_manager(net)
+    |> Processes.ConnectionManager.get_neuron_in_connections(neuron_id)
+  end
+
   @doc """
   Changes a weight of a given connection and saves it in a connection manager.
   """
diff --git a/lib/neuron_system/processes/connection_manager.ex b/lib/neuron_system/processes/connection_manager.ex
index bc3b060..60ed304 100644
--- a/lib/neuron_system/processes/connection_manager.ex
+++ b/lib/neuron_system/processes/connection_manager.ex
@@ -61,11 +61,6 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     GenServer.cast(manager_pid, {:change_weight, connection_id, new_weight})
   end
 
-  def handle_cast({:add, connection}, state) do
-    new_state = [connection | state]
-    {:noreply, new_state}
-  end
-
   def handle_call({:get_net_in_out, :in = _type}, _from, state) do
     connections = state |> filter_by_type(Models.InConnection)
     {:reply, connections, state}
@@ -84,8 +79,12 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     {:reply, connections, state} 
   end
 
+  def handle_cast({:add, connection}, state) do
+    new_state = [connection | state]
+    {:noreply, new_state}
+  end
   def handle_cast({:change_weight, connection_id, new_weight}, state) do
-    connection = state |> Enum.filter(&(&1.id == connection_id)) |> Enum.first
+    connection = state |> Enum.filter(&(&1.id == connection_id)) |> Enum.at(0)
     other_connections = state |> Enum.filter(&(&1.id != connection_id))
     new_connection = %{connection | weight: new_weight}
     {:noreply, [new_connection | other_connections]}

From f949fc57f500af86e9d580b0370443590c9a99f8 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Sun, 14 Aug 2016 17:17:52 +0300
Subject: [PATCH 15/20] Added basic unit tests

---
 .../back_prop/neuron/hidden_processor.ex      |  2 +-
 test/lib/neuron_system/connection_test.exs    | 25 +++++++++++++++++++
 test/lib/neuron_system/net_test.exs           | 24 ++++++++++++++++++
 test/lib/neuron_system/neuron_test.exs        | 12 +++++++++
 test/neuron_system_test.exs                   |  8 ------
 5 files changed, 62 insertions(+), 9 deletions(-)
 create mode 100644 test/lib/neuron_system/connection_test.exs
 create mode 100644 test/lib/neuron_system/net_test.exs
 create mode 100644 test/lib/neuron_system/neuron_test.exs
 delete mode 100644 test/neuron_system_test.exs

diff --git a/lib/neuron_system/back_prop/neuron/hidden_processor.ex b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
index 0dc62e5..44f937a 100644
--- a/lib/neuron_system/back_prop/neuron/hidden_processor.ex
+++ b/lib/neuron_system/back_prop/neuron/hidden_processor.ex
@@ -56,7 +56,7 @@ defmodule NeuronSystem.BackProp.Neuron.HiddenProcessor do
 
   defp at_least_one_is_in_connection?(connections) do
     Enum.any?(connections, fn
-      (%Models.InConnection{} = conn) -> true
+      (%Models.InConnection{} = _conn) -> true
       _ -> false
     end)
   end
diff --git a/test/lib/neuron_system/connection_test.exs b/test/lib/neuron_system/connection_test.exs
new file mode 100644
index 0000000..df5723f
--- /dev/null
+++ b/test/lib/neuron_system/connection_test.exs
@@ -0,0 +1,25 @@
+defmodule NeuronSystem.ConnectionTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "source when connection is input" do
+    connection = %Models.InConnection{key: :x1}
+    source_value = NeuronSystem.Connection.source(connection)
+    assert source_value == :x1
+  end
+
+  test "source when connection is between neurons" do
+    source_neuron = "neuron:sadsdasdas"
+    connection = %Models.Connection{source_neuron: source_neuron}
+    source_value = NeuronSystem.Connection.source(connection)
+    assert source_value == source_neuron
+  end
+
+  test "source when connection is output" do
+    source_neuron = "neuron:sadsdasdas"
+    connection = %Models.OutConnection{source_neuron: source_neuron}
+    source_value = NeuronSystem.Connection.source(connection)
+    assert source_value == source_neuron
+  end
+end
diff --git a/test/lib/neuron_system/net_test.exs b/test/lib/neuron_system/net_test.exs
new file mode 100644
index 0000000..cc174d8
--- /dev/null
+++ b/test/lib/neuron_system/net_test.exs
@@ -0,0 +1,24 @@
+defmodule NeuronSystem.NetTest do
+  use ExUnit.Case
+
+  alias NeuronSystem.{Models}
+
+  test "create adds new supervisor for a Net" do
+    net_model = NeuronSystem.Net.create
+    assert %Models.Net{} = net_model
+    process_info = Process.info(net_model.pid)
+    assert process_info != nil
+  end
+
+  test "create saves info to the struct and returns it" do
+    %Models.Net{pid: net_pid, root_pid: root_pid} = NeuronSystem.Net.create
+    assert net_pid != nil
+    assert root_pid == self()
+  end
+
+  test "create adds default list of childrens for a Net" do
+    net_model = NeuronSystem.Net.create
+    childrens = Supervisor.which_children(net_model.pid)
+    assert Enum.count(childrens) == 1
+  end
+end
diff --git a/test/lib/neuron_system/neuron_test.exs b/test/lib/neuron_system/neuron_test.exs
new file mode 100644
index 0000000..029c719
--- /dev/null
+++ b/test/lib/neuron_system/neuron_test.exs
@@ -0,0 +1,12 @@
+defmodule NeuronSystem.NeuronTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "activate calls activation function with a given net value" do
+    neuron_model = %Models.Neuron{activation_function: fn(x) -> x * 2 end}
+    net_value = 3
+    activation_result = NeuronSystem.Neuron.activate(neuron_model, net_value)
+    assert activation_result == 6
+  end
+end
diff --git a/test/neuron_system_test.exs b/test/neuron_system_test.exs
deleted file mode 100644
index 2fe4d4d..0000000
--- a/test/neuron_system_test.exs
+++ /dev/null
@@ -1,8 +0,0 @@
-defmodule NeuronSystemTest do
-  use ExUnit.Case
-  doctest NeuronSystem
-
-  test "the truth" do
-    assert 1 + 1 == 2
-  end
-end

From 6dac76aadc64805e75cf6ffd1c2019bd89c03c57 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Wed, 17 Aug 2016 16:34:21 +0300
Subject: [PATCH 16/20] Added almost all tests for Net

---
 lib/neuron_system/net.ex                      |   9 ++
 .../processes/connection_manager.ex           |  11 ++
 mix.exs                                       |   2 +-
 mix.lock                                      |   2 +
 test/lib/neuron_system/net_test.exs           | 128 ++++++++++++++++++
 5 files changed, 151 insertions(+), 1 deletion(-)

diff --git a/lib/neuron_system/net.ex b/lib/neuron_system/net.ex
index 2306dff..d5498dd 100644
--- a/lib/neuron_system/net.ex
+++ b/lib/neuron_system/net.ex
@@ -170,6 +170,15 @@ defmodule NeuronSystem.Net do
     end)
   end
 
+  @doc """
+  Detects and returns the whole connections in a Net
+  """
+  @spec get_all_connections(Models.Net.t) :: list
+  def get_all_connections(%Models.Net{} = net) do
+    NeuronSystem.Net.connection_manager(net)
+    |> Processes.ConnectionManager.get_all
+  end
+
   @doc """
   Detects and returns net's output connections list
   """
diff --git a/lib/neuron_system/processes/connection_manager.ex b/lib/neuron_system/processes/connection_manager.ex
index 60ed304..cfc2f48 100644
--- a/lib/neuron_system/processes/connection_manager.ex
+++ b/lib/neuron_system/processes/connection_manager.ex
@@ -21,6 +21,14 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     GenServer.cast(manager_pid, {:add, connection_model})
   end
 
+  @doc """
+  Gets the list of all connections and returns it
+  """
+  @spec get_all(pid) :: list
+  def get_all(manager_pid) do
+    GenServer.call(manager_pid, :get_all)
+  end
+
   @doc """
   Gets a list of all input connection for a whole Net
   """
@@ -61,6 +69,9 @@ defmodule NeuronSystem.Processes.ConnectionManager do
     GenServer.cast(manager_pid, {:change_weight, connection_id, new_weight})
   end
 
+  def handle_call(:get_all, _from, state) do
+    {:reply, state, state}
+  end
   def handle_call({:get_net_in_out, :in = _type}, _from, state) do
     connections = state |> filter_by_type(Models.InConnection)
     {:reply, connections, state}
diff --git a/mix.exs b/mix.exs
index 0ab24f5..2e19039 100644
--- a/mix.exs
+++ b/mix.exs
@@ -29,7 +29,7 @@ defmodule NeuronSystem.Mixfile do
   # Type "mix help deps" for more examples and options
   defp deps do
     [
-      {:ex_doc, "~> 0.13", only: :dev},
+      { :ex_doc, "~> 0.13", only: :dev },
       { :uuid, "~> 1.1.4" }
     ]
   end
diff --git a/mix.lock b/mix.lock
index 16ce6c5..fca4bd5 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,3 +1,5 @@
 %{"earmark": {:hex, :earmark, "1.0.1", "2c2cd903bfdc3de3f189bd9a8d4569a075b88a8981ded9a0d95672f6e2b63141", [:mix], []},
   "ex_doc": {:hex, :ex_doc, "0.13.0", "aa2f8fe4c6136a2f7cfc0a7e06805f82530e91df00e2bff4b4362002b43ada65", [:mix], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]},
+  "meck": {:hex, :meck, "0.8.4", "59ca1cd971372aa223138efcf9b29475bde299e1953046a0c727184790ab1520", [:rebar, :make], []},
+  "mock": {:hex, :mock, "0.1.1", "e21469ca27ba32aa7b18b61699db26f7a778171b21c0e5deb6f1218a53278574", [:mix], [{:meck, "~> 0.8.2", [hex: :meck, optional: false]}]},
   "uuid": {:hex, :uuid, "1.1.4", "36c7734e4c8e357f2f67ba57fb61799d60c20a7f817b104896cca64b857e3686", [:mix], []}}
diff --git a/test/lib/neuron_system/net_test.exs b/test/lib/neuron_system/net_test.exs
index cc174d8..914848e 100644
--- a/test/lib/neuron_system/net_test.exs
+++ b/test/lib/neuron_system/net_test.exs
@@ -21,4 +21,132 @@ defmodule NeuronSystem.NetTest do
     childrens = Supervisor.which_children(net_model.pid)
     assert Enum.count(childrens) == 1
   end
+
+  test "add_neuron adds new neuron with a custom activation function" do
+    %Models.Net{pid: net_pid} = net = NeuronSystem.Net.create
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 1
+    {:ok, %Models.Neuron{}} = NeuronSystem.Net.add_neuron(net, fn(x) -> x end)
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 2
+  end
+
+  test "add_neuron adds perceptron neuron" do
+    %Models.Net{pid: net_pid} = net = NeuronSystem.Net.create
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 1
+    {:ok, %Models.Neuron{}} = NeuronSystem.Net.add_neuron(net, :perceptron, [threshold: 0.5])
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 2
+  end
+
+  test "add_neuron adds sigmoid neuron" do
+    %Models.Net{pid: net_pid} = net = NeuronSystem.Net.create
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 1
+    {:ok, %Models.Neuron{}} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.5])
+    childrens = Supervisor.which_children(net_pid)
+    assert Enum.count(childrens) == 2
+  end
+
+  test "neuron_process_pid returns PID of a neuron's process if such one exists" do
+    net = NeuronSystem.Net.create
+    {:ok, %Models.Neuron{id: neuron_id}} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.5])
+    neuron_pid = NeuronSystem.Net.neuron_process_pid(net, neuron_id)
+    assert neuron_pid != nil
+  end
+
+  test "neuron_process_pid returns nil if such one does not exist" do
+    net = NeuronSystem.Net.create
+    neuron_pid = NeuronSystem.Net.neuron_process_pid(net, "neuron:ahahahaa")
+    assert neuron_pid == nil
+  end
+
+  test "connection_manager returns PID of a connection manager of a Net" do
+    net = NeuronSystem.Net.create
+    connection_manager_pid = NeuronSystem.Net.connection_manager(net)
+    assert connection_manager_pid != nil
+  end
+
+  test "add_connection adds link between neurons" do
+    net = NeuronSystem.Net.create
+    source_neuron_model = %Models.Neuron{}
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, source_neuron_model, target_neuron_model, 0.3)
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 1
+  end
+
+  test "add_connection adds income connection of a Net" do
+    net = NeuronSystem.Net.create
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, :in, target_neuron_model, 0.3, :x1)
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 1
+  end
+
+  test "add_connection adds output connection of a Net" do
+    net = NeuronSystem.Net.create
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, :out, target_neuron_model, 0.3, :y1)
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 1
+  end
+
+  test "in_connections returns input connections for a Net" do
+    net = NeuronSystem.Net.create
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.in_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, :in, target_neuron_model, 0.3, :x1)
+    connections = NeuronSystem.Net.in_connections(net)
+    assert Enum.count(connections) == 1
+  end
+
+  test "out_connection returns output connections of a Net" do
+    net = NeuronSystem.Net.create
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.out_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, :out, target_neuron_model, 0.3, :y1)
+    connections = NeuronSystem.Net.out_connections(net)
+    assert Enum.count(connections) == 1
+  end
+
+  test "get_all_connections returns the list of all connections in a Net" do
+    net = NeuronSystem.Net.create
+    source_neuron_model = %Models.Neuron{}
+    target_neuron_model = %Models.Neuron{}
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 0
+    NeuronSystem.Net.add_connection(net, source_neuron_model, target_neuron_model, 0.3)
+    NeuronSystem.Net.add_connection(net, :in, target_neuron_model, 0.3, :x1)
+    NeuronSystem.Net.add_connection(net, :out, source_neuron_model, 0.3, :y1)
+    connections = NeuronSystem.Net.get_all_connections(net)
+    assert Enum.count(connections) == 3
+  end
+
+  test "neuron_out_connections returns a list of out connection of a neuron" do
+    net = NeuronSystem.Net.create
+    source_neuron_model = %Models.Neuron{id: "source_neuron"}
+    target_neuron_model = %Models.Neuron{id: "target_neuron"}
+    NeuronSystem.Net.add_connection(net, source_neuron_model, target_neuron_model, 0.3)
+    connections = NeuronSystem.Net.neuron_out_connections(net, source_neuron_model.id)
+    assert Enum.count(connections) == 1
+  end
+
+  test "neuron_in_connections returns a list of out connection of a neuron" do
+    net = NeuronSystem.Net.create
+    source_neuron_model = %Models.Neuron{id: "source_neuron"}
+    target_neuron_model = %Models.Neuron{id: "target_neuron"}
+    NeuronSystem.Net.add_connection(net, source_neuron_model, target_neuron_model, 0.3)
+    connections = NeuronSystem.Net.neuron_in_connections(net, target_neuron_model.id)
+    assert Enum.count(connections) == 1
+  end
 end

From 4c0af245a88871dc6419e5d977b0946064a3ffef Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Fri, 19 Aug 2016 15:34:51 +0300
Subject: [PATCH 17/20] Added test for specification builder helper

---
 mix.exs                                       |  3 +-
 mix.lock                                      |  2 +-
 .../utils/common_helper_test.exs              |  8 +++
 .../utils/send_to_neuron_proxy_test.exs       | 31 ++++++++++
 .../neuron_system/utils/spec_helper_test.exs  | 59 +++++++++++++++++++
 5 files changed, 101 insertions(+), 2 deletions(-)
 create mode 100644 test/lib/neuron_system/utils/common_helper_test.exs
 create mode 100644 test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
 create mode 100644 test/lib/neuron_system/utils/spec_helper_test.exs

diff --git a/mix.exs b/mix.exs
index 2e19039..38d8872 100644
--- a/mix.exs
+++ b/mix.exs
@@ -30,7 +30,8 @@ defmodule NeuronSystem.Mixfile do
   defp deps do
     [
       { :ex_doc, "~> 0.13", only: :dev },
-      { :uuid, "~> 1.1.4" }
+      { :uuid, "~> 1.1.4" },
+      { :mock, github: "jjh42/mock", ref: "7f2251f781f646a08bb65c85c215f107c9627435" }
     ]
   end
 end
diff --git a/mix.lock b/mix.lock
index fca4bd5..5e9e229 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,5 +1,5 @@
 %{"earmark": {:hex, :earmark, "1.0.1", "2c2cd903bfdc3de3f189bd9a8d4569a075b88a8981ded9a0d95672f6e2b63141", [:mix], []},
   "ex_doc": {:hex, :ex_doc, "0.13.0", "aa2f8fe4c6136a2f7cfc0a7e06805f82530e91df00e2bff4b4362002b43ada65", [:mix], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]},
   "meck": {:hex, :meck, "0.8.4", "59ca1cd971372aa223138efcf9b29475bde299e1953046a0c727184790ab1520", [:rebar, :make], []},
-  "mock": {:hex, :mock, "0.1.1", "e21469ca27ba32aa7b18b61699db26f7a778171b21c0e5deb6f1218a53278574", [:mix], [{:meck, "~> 0.8.2", [hex: :meck, optional: false]}]},
+  "mock": {:git, "https://github.com/jjh42/mock.git", "7f2251f781f646a08bb65c85c215f107c9627435", [ref: "7f2251f781f646a08bb65c85c215f107c9627435"]},
   "uuid": {:hex, :uuid, "1.1.4", "36c7734e4c8e357f2f67ba57fb61799d60c20a7f817b104896cca64b857e3686", [:mix], []}}
diff --git a/test/lib/neuron_system/utils/common_helper_test.exs b/test/lib/neuron_system/utils/common_helper_test.exs
new file mode 100644
index 0000000..a8c3992
--- /dev/null
+++ b/test/lib/neuron_system/utils/common_helper_test.exs
@@ -0,0 +1,8 @@
+defmodule NeuronSystem.Utils.CommonHelperTest do
+  use ExUnit.Case, async: true
+
+  test "gen_process_id generates string with a given prefix and random suffix" do
+    process_id = NeuronSystem.Utils.CommonHelper.gen_process_id("neuron")
+    assert String.contains?(process_id, "neuron")
+  end
+end
diff --git a/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs b/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
new file mode 100644
index 0000000..abeede3
--- /dev/null
+++ b/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
@@ -0,0 +1,31 @@
+defmodule NeuronSystem.Utils.SendToNeuronProxyTest do
+  use ExUnit.Case, async: false
+
+  alias NeuronSystem.{Models, Processes}
+
+  import Mock
+
+  test "call sends message to the root if connection is output" do
+    net = %Models.Net{root_pid: self()}
+    connection = %Models.OutConnection{key: :y1}
+    value = 10
+    NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, value)
+    receive do
+      {:out_result, :y1, ^value} -> assert true
+    after
+      1_000 -> assert false
+    end
+  end
+
+  test "call sends income payload to a neurons in a next layer" do
+    with_mocks([
+      {Processes.Neuron, [], [income_payload: fn(_pid, {_source_key, _payload_value, _net}) -> :ok end]},
+      {NeuronSystem.Net, [], [neuron_process_pid: fn(_net, _neuron_id) -> "pid2" end]}
+      ]) do
+      net = %Models.Net{root_pid: self()}
+      connection = %Models.Connection{weight: 0.5, source_neuron: "source_neuron", target_neuron: "target_neuron"}
+      NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, 5)
+      assert called Processes.Neuron.income_payload("pid2", {"source_neuron", 2.5, net})
+    end
+  end
+end
diff --git a/test/lib/neuron_system/utils/spec_helper_test.exs b/test/lib/neuron_system/utils/spec_helper_test.exs
new file mode 100644
index 0000000..1224af0
--- /dev/null
+++ b/test/lib/neuron_system/utils/spec_helper_test.exs
@@ -0,0 +1,59 @@
+defmodule NeuronSystem.Utils.SpecHelperTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  defmodule TestProcess do
+  end
+
+  test "build_supervisor_spec builds specification for a supervisor process" do
+    spec = NeuronSystem.Utils.SpecHelper.build_supervisor_spec(TestProcess, [], "test_process") 
+    assert {:ok,
+      {"test_process:" <> _random_uuid, {
+          NeuronSystem.Utils.SpecHelperTest.TestProcess,
+          :start_link, []
+        },
+        :permanent,
+        :infinity,
+        :supervisor,
+        [NeuronSystem.Utils.SpecHelperTest.TestProcess]
+      }
+    } = spec
+  end
+
+  test "build_worker_spec builds specification for a worker process" do
+    worker_id = "test_process:a1212"
+    spec = NeuronSystem.Utils.SpecHelper.build_worker_spec(TestProcess, [], worker_id)
+    assert {
+      :ok,
+      {"test_process:a1212", {
+          NeuronSystem.Utils.SpecHelperTest.TestProcess,
+          :start_link,
+          []
+        },
+        :permanent,
+        5000,
+        :worker,
+        [NeuronSystem.Utils.SpecHelperTest.TestProcess]
+      }
+    } = spec
+  end
+
+  test "build_neuron_worker_spec builds specification for a neuron process" do
+    neuron_model = %Models.Neuron{id: "neuron:1234"}
+    spec = NeuronSystem.Utils.SpecHelper.build_neuron_worker_spec(neuron_model)
+    assert {
+      :ok, {
+        "neuron:1234", {
+          NeuronSystem.Processes.Neuron,
+          :start_link,
+          [%NeuronSystem.Models.Neuron{activation_function: nil, id: "neuron:1234"}]
+        },
+        :permanent,
+        5000,
+        :worker,
+        [NeuronSystem.Processes.Neuron]
+      }
+    } = spec
+  end
+end

From 62b9225dbf3b05f6c48afe6471ae8078fee73ac8 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Fri, 19 Aug 2016 15:41:26 +0300
Subject: [PATCH 18/20] Added 3rd layers net to test

---
 test/lib/neuron_system_test.exs | 44 +++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 test/lib/neuron_system_test.exs

diff --git a/test/lib/neuron_system_test.exs b/test/lib/neuron_system_test.exs
new file mode 100644
index 0000000..8cc57f2
--- /dev/null
+++ b/test/lib/neuron_system_test.exs
@@ -0,0 +1,44 @@
+defmodule NeuronSystemTest do
+  use ExUnit.Case
+
+  test "forward way of a simple Net with 3 layers" do
+    income = %{x1: 1.0, x2: 0.0}
+
+    # Create network
+    net = NeuronSystem.Net.create
+
+    # Create needed neurons 
+    {:ok, neuronA1} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+    {:ok, neuronA2} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+    {:ok, neuronA3} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+    {:ok, neuronB1} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+    {:ok, neuronB2} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+    {:ok, neuronC1} = NeuronSystem.Net.add_neuron(net, :sigmoid, [threshold: 0.0])
+
+    # Add in connection
+    NeuronSystem.Net.add_connection(net, :in, neuronA1, 0.19592, :x1)
+    NeuronSystem.Net.add_connection(net, :in, neuronA2, 0.32244, :x1)
+    NeuronSystem.Net.add_connection(net, :in, neuronA3, 0.16243, :x1)
+    NeuronSystem.Net.add_connection(net, :in, neuronA1, 0.6, :x2)
+    NeuronSystem.Net.add_connection(net, :in, neuronA2, -0.5, :x2)
+    NeuronSystem.Net.add_connection(net, :in, neuronA3, 0.4, :x2)
+
+    # Add out connection
+    NeuronSystem.Net.add_connection(net, :out, neuronC1, 1.0, :y)
+
+    # Add connection in hidden layers
+    NeuronSystem.Net.add_connection(net, neuronA1, neuronB1, -0.0855)
+    NeuronSystem.Net.add_connection(net, neuronA1, neuronB2, -0.398)
+    NeuronSystem.Net.add_connection(net, neuronA2, neuronB1, 0.715)
+    NeuronSystem.Net.add_connection(net, neuronA2, neuronB2, 1.10207)
+    NeuronSystem.Net.add_connection(net, neuronA3, neuronB1, 0.374)
+    NeuronSystem.Net.add_connection(net, neuronA3, neuronB2, 0.79194)
+    NeuronSystem.Net.add_connection(net, neuronB1, neuronC1, 0.831)
+    NeuronSystem.Net.add_connection(net, neuronB2, neuronC1, 0.133)
+
+    # Activate it!
+    results = NeuronSystem.Net.activate!(net, income)
+    
+    assert [y: 0.6511113215704272] = results
+  end
+end

From b211c46cfab343062db71ced56678cd3d3100e10 Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Tue, 30 Aug 2016 15:40:45 +0300
Subject: [PATCH 19/20] More models tests

---
 .../neuron_system/models/connection_test.exs    | 17 +++++++++++++++++
 .../neuron_system/models/in_connection_test.exs | 16 ++++++++++++++++
 test/lib/neuron_system/models/neuron_test.exs   | 11 +++++++++++
 .../models/out_connection_test.exs              | 17 +++++++++++++++++
 4 files changed, 61 insertions(+)
 create mode 100644 test/lib/neuron_system/models/connection_test.exs
 create mode 100644 test/lib/neuron_system/models/in_connection_test.exs
 create mode 100644 test/lib/neuron_system/models/neuron_test.exs
 create mode 100644 test/lib/neuron_system/models/out_connection_test.exs

diff --git a/test/lib/neuron_system/models/connection_test.exs b/test/lib/neuron_system/models/connection_test.exs
new file mode 100644
index 0000000..7e23a3e
--- /dev/null
+++ b/test/lib/neuron_system/models/connection_test.exs
@@ -0,0 +1,17 @@
+defmodule NeuronSystem.Models.ConnectionTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "build builds new connection model" do
+    source_neuron = %Models.Neuron{id: "source_neuron"}
+    target_neuron = %Models.Neuron{id: "target_neuron"}
+    connection = Models.Connection.build(source_neuron, target_neuron, 0.5)
+    assert %Models.Connection{
+      id: "connection" <> _random_uuid,
+      source_neuron: "source_neuron",
+      target_neuron: "target_neuron",
+      weight: 0.5
+    } = connection
+  end
+end
diff --git a/test/lib/neuron_system/models/in_connection_test.exs b/test/lib/neuron_system/models/in_connection_test.exs
new file mode 100644
index 0000000..c04887d
--- /dev/null
+++ b/test/lib/neuron_system/models/in_connection_test.exs
@@ -0,0 +1,16 @@
+defmodule NeuronSystem.Models.InConnectionTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "build builds new model for input connection" do
+    neuron_model = %Models.Neuron{id: "target_neuron"}
+    connection = Models.InConnection.build(neuron_model, 0.5, :x1)
+    assert %Models.InConnection{
+      id: "connection" <> _random_uuid,
+      target_neuron: "target_neuron",
+      weight: 0.5,
+      key: :x1
+    } = connection
+  end
+end
diff --git a/test/lib/neuron_system/models/neuron_test.exs b/test/lib/neuron_system/models/neuron_test.exs
new file mode 100644
index 0000000..e4dedf2
--- /dev/null
+++ b/test/lib/neuron_system/models/neuron_test.exs
@@ -0,0 +1,11 @@
+defmodule NeuronSystem.Models.NeuronTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "build build new model for neuron" do
+    activation_func = func(x) -> x * 3 end
+    neuron = Models.Neuron.build(activation_func)
+    assert %Models.Neuron{activation_function: activation_func} = neuron
+  end
+end
diff --git a/test/lib/neuron_system/models/out_connection_test.exs b/test/lib/neuron_system/models/out_connection_test.exs
new file mode 100644
index 0000000..b34d78c
--- /dev/null
+++ b/test/lib/neuron_system/models/out_connection_test.exs
@@ -0,0 +1,17 @@
+defmodule NeuronSystem.Models.OutConnectionTest do
+  use ExUnit.Case, async: true
+
+  alias NeuronSystem.{Models}
+
+  test "build builds new model for output connection" do
+    neuron_model = %Models.Neuron{id: "source_neuron"}
+    connection = Models.OutConnection.build(neuron_model, 0.5, :y)
+    assert %Models.OutConnection{
+      id: "connection" <> _random_uuid,
+      source_neuron: "source_neuron",
+      weight: 0.5,
+      key: :y
+    } = connection
+  end
+end
+

From 9e7e5d4a76c526bccdfe5daae1cccea3b3a3262a Mon Sep 17 00:00:00 2001
From: Sergey Gernyak <sergeg1990@gmail.com>
Date: Tue, 30 Aug 2016 16:47:18 +0300
Subject: [PATCH 20/20] Added test for net activator; fixed all warnings

---
 lib/neuron_system/net/activator.ex            |  4 ++--
 test/lib/neuron_system/models/neuron_test.exs |  4 ++--
 test/lib/neuron_system/net/activator_test.exs | 23 +++++++++++++++++++
 .../utils/send_to_neuron_proxy_test.exs       |  2 +-
 4 files changed, 28 insertions(+), 5 deletions(-)
 create mode 100644 test/lib/neuron_system/net/activator_test.exs

diff --git a/lib/neuron_system/net/activator.ex b/lib/neuron_system/net/activator.ex
index bcac957..7426ef5 100644
--- a/lib/neuron_system/net/activator.ex
+++ b/lib/neuron_system/net/activator.ex
@@ -8,7 +8,7 @@ defmodule NeuronSystem.Net.Activator do
   2. Puts a bunch of `receive` blocks to collect outputs
   """
 
-  alias NeuronSystem.{Models, Processes}
+  alias NeuronSystem.{Models}
 
   @doc """
   The main point for an activator.
@@ -43,7 +43,7 @@ defmodule NeuronSystem.Net.Activator do
   defp collect_result_for_connection(connection) do
     connection_key = connection.key
     receive do
-      {:out_result, connection_key, value} -> {connection.key, value}
+      {:out_result, ^connection_key, value} -> {connection.key, value}
     end
   end
 end
diff --git a/test/lib/neuron_system/models/neuron_test.exs b/test/lib/neuron_system/models/neuron_test.exs
index e4dedf2..051644e 100644
--- a/test/lib/neuron_system/models/neuron_test.exs
+++ b/test/lib/neuron_system/models/neuron_test.exs
@@ -4,8 +4,8 @@ defmodule NeuronSystem.Models.NeuronTest do
   alias NeuronSystem.{Models}
 
   test "build build new model for neuron" do
-    activation_func = func(x) -> x * 3 end
+    activation_func = fn(x) -> x * 3 end
     neuron = Models.Neuron.build(activation_func)
-    assert %Models.Neuron{activation_function: activation_func} = neuron
+    assert %Models.Neuron{activation_function: _activation_func} = neuron
   end
 end
diff --git a/test/lib/neuron_system/net/activator_test.exs b/test/lib/neuron_system/net/activator_test.exs
new file mode 100644
index 0000000..4225e6d
--- /dev/null
+++ b/test/lib/neuron_system/net/activator_test.exs
@@ -0,0 +1,23 @@
+defmodule NeuronSystem.Net.ActivatorTest do
+  use ExUnit.Case, async: true
+
+  import Mock
+
+  test "call sends events to neurons and waits for responses" do
+    net = NeuronSystem.Net.create
+    {:ok, neuronA} = NeuronSystem.Net.add_neuron(net, :perceptron, [threshold: 0.5])
+    {:ok, inConnection} = NeuronSystem.Net.add_connection(net, :in, neuronA, 0.5, :x1)
+    NeuronSystem.Net.add_connection(net, :out, neuronA, 0.5, :y1)
+
+
+    with_mock NeuronSystem.Utils.SendToNeuronProxy, [call: fn(_net, _connection, _value) -> :ok end] do
+      send self(), {:out_result, :y1, 2.3}
+
+      results = NeuronSystem.Net.Activator.call(net, %{x1: 2.0})
+
+      assert [y1: 2.3] = results
+
+      assert called NeuronSystem.Utils.SendToNeuronProxy.call(net, inConnection, 2.0)
+    end
+  end
+end
diff --git a/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs b/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
index abeede3..8f68aae 100644
--- a/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
+++ b/test/lib/neuron_system/utils/send_to_neuron_proxy_test.exs
@@ -11,7 +11,7 @@ defmodule NeuronSystem.Utils.SendToNeuronProxyTest do
     value = 10
     NeuronSystem.Utils.SendToNeuronProxy.call(net, connection, value)
     receive do
-      {:out_result, :y1, ^value} -> assert true
+      {:out_result, :y1, out_value} -> assert value == out_value
     after
       1_000 -> assert false
     end