Commit 339fb1d0 authored by Tim Molderez's avatar Tim Molderez
Browse files

Refactored Marlon.LearningAlgorithm to use Classy structs

parent 76ba138a

lib/marlon/marl/agent.ex

+10 −11
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
# You should have received a copy of the GNU General Public License

# along with Marlon.  If not, see http://www.gnu.org/licenses



use Class



defmodule Marlon.Agent do

  @moduledoc """
@@ -32,7 +33,7 @@ defmodule Marlon.Agent do 


  def init([goal: p, actor: a, actor_state: state]) do

    instance = p.instantiate()

    initialized_instance = Marlon.LearningAlgorithm.init_agent(instance, state)

    initialized_instance = Marlon.LearningAlgorithm~>init_agent(instance, state)

    {:ok, %{goal: initialized_instance, actor: a}}

  end
@@ -64,13 +65,12 @@ defmodule Marlon.Agent do 


    # IO.puts Kernel.inspect(Marlon.LearningAlgorithm.action_probabilities(p)) <> " " <> Kernel.inspect(Node.self())



    if (Marlon.LearningAlgorithm.is_learning(p)) do

      action_index = Marlon.LearningAlgorithm.sample_action(p)

      msg_tuple = Marlon.LearningAlgorithm.action_index_to_message(p, action_index)

    if (Marlon.LearningAlgorithm~>is_learning(p)) do

      action_index = Marlon.LearningAlgorithm~>sample_action(p)

      msg_tuple = Marlon.LearningAlgorithm~>action_index_to_message(p, action_index)

      GenServer.cast(a, msg_tuple) # Execute the selected action



      new_base = %{p.base | last_action: action_index}

      new_p = %{p | base: new_base}

      new_p = %{p | last_action: action_index}

      {:noreply, %{state | goal: new_p}}

    else

      # Execute the best action we've learnt
@@ -84,13 +84,12 @@ defmodule Marlon.Agent do 
  def handle_cast({:update_reward, actor_state}, state) do

    %{goal: p, actor: _a} = state

    

    if (Marlon.LearningAlgorithm.is_learning(p)) do

    if (Marlon.LearningAlgorithm~>is_learning(p)) do

      abstr_actor_state = Marlon.LearningAlgorithm.abstract_state(p, actor_state)

      base = %{p.base | last_state: abstr_actor_state}

      # r = Marlon.LearningAlgorithm.reward(p, abstr_actor_state) # TODO Maybe useful to pass in both abstr. and the full state?

      r = Marlon.LearningAlgorithm.reward(p, actor_state)

      p = Marlon.LearningAlgorithm.update(p, r)

      {:noreply, %{state | goal: %{p | base: base}}}

      r = Marlon.LearningAlgorithm~>reward(p, actor_state)

      p = Marlon.LearningAlgorithm~>update(p, r)

      {:noreply, %{state | goal: %{p | last_state: abstr_actor_state}}}

    else

      {:noreply, state}

    end

lib/marlon/marl/algorithms/esrl.ex

+30 −50
Original line number Diff line number Diff line
@@ -16,75 +16,55 @@ 
# You should have received a copy of the GNU General Public License

# along with Marlon.  If not, see http://www.gnu.org/licenses



use Class



defmodule Marlon.ESRL do

  @moduledoc """

    Implementation of the Exploring Selfish Reinforcement Learning algorithm

    (see the paper "Exploring Selfish Reinforcement Learning in Repeated Games with Stochastic Rewards")

  """

defclass Marlon.ESRL do

  extends Marlon.LearningAlgorithm



  defstruct(

    python: nil,  # Process identifier of a running Python instance

    module: :marlonEsrl, # Name of the Python file we're running

  var python: nil  # Process identifier of a running Python instance

  var module: :marlonEsrl # Name of the Python file we're running



    a: 0.05,

    steps: 2000,

    explorations: 7,

    window: 100,

    base: %Marlon.LearningAlgorithmBase{}

  )

end



defimpl Marlon.LearningAlgorithm, for: Marlon.ESRL do



  # (These functions are all inherited from Marlon.LearningAlgorithmDefaults)

  defdelegate init_algorithm(goal, data), to: Marlon.LearningAlgorithmDefaults

  defdelegate number_of_actions(goal), to: Marlon.LearningAlgorithmDefaults

  defdelegate action_index_to_message(goal, action_index), to: Marlon.LearningAlgorithmDefaults

  defdelegate reward(goal, actor_state), to: Marlon.LearningAlgorithmDefaults

  defdelegate best_action(goal), to: Marlon.LearningAlgorithmDefaults

  var a: 0.05

  var steps: 2000

  var explorations: 7

  var window: 100



  def init_agent(goal, actor_state) do

  def init_agent(this, actor_state) do

    # Start a new Python instance

    python_path = Application.fetch_env!(:marlon, :python_path) # Path to the folder with our Python code

    {:ok, instance} = :python.start([{:python_path, python_path}])



    # Create a new instance of an ESLR goal (Note that the Python script stores this instance as a global variable, so we can access it later.)

    :python.call(instance, goal.module, :createGoal,

      [Marlon.LearningAlgorithmDefaults.number_of_actions(goal),

      goal.a,

      goal.steps,

      goal.explorations,

      goal.window])

    :python.call(instance, this.module, :createGoal,

      [Marlon.LearningAlgorithm.number_of_actions(this),

      this.a,

      this.steps,

      this.explorations,

      this.window])



    new_b = %{goal.base | last_state: actor_state}

    %{goal | python: instance, base: new_b}

    %{this | python: instance, last_state: actor_state}

  end



  def inc_timestep(goal) do

    :python.call(goal.python, goal.module, :incTimeStep, [])

  def inc_timestep(this) do

    :python.call(this.python, this.module, :incTimeStep, [])

  end



  def is_learning(goal) do

    :python.call(goal.python, goal.module, :isLearning, [])

  def is_learning(this) do

    :python.call(this.python, this.module, :isLearning, [])

  end



  def sample_action(goal) do

    :python.call(goal.python, goal.module, :sampleAction, [])

  def sample_action(this) do

    :python.call(this.python, this.module, :sampleAction, [])

  end



  def update(goal, reward) do

    :python.call(goal.python, goal.module, :update, [goal.base.last_action, reward])

    goal

  def update(this, reward) do

    :python.call(this.python, this.module, :update, [this.last_action, reward])

    this

  end



  def action_probabilities(goal) do

    Enum.map(0..number_of_actions(goal)-1, fn(action_index) ->

        :python.call(goal.python, goal.module, :action_probability, [action_index])

  def action_probabilities(this) do

    Enum.map(0..Marlon.LearningAlgorithm.number_of_actions(this)-1, fn(action_index) ->

        :python.call(this.python, this.module, :action_probability, [action_index])

      end)

  end



  def abstract_state(_goal, _actor_state) do

    #TODO

  end

end
 
 No newline at end of file

lib/marlon/marl/algorithms/jointactionlearning.ex

0 → 100644
+26 −0
Original line number Diff line number Diff line 
# Marlon - Multi-Agent Reinforcement Learning On Networks

# Copyright (C) 2018 Smile-IT project

#

# This file is part of Marlon.

#

# Marlon is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# Marlon is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with Marlon.  If not, see http://www.gnu.org/licenses



use Class 



# defclass Marlon.JointActionLearning do

#   extends Marlon.LearningAlgorithm



#   var discount_factor: 1.0 # gamma

#   var learning_rate: 0.1 # alpha

# end
 
 No newline at end of file

lib/marlon/marl/algorithms/qlearning.ex

+46 −62
Original line number Diff line number Diff line
@@ -16,63 +16,47 @@ 
# You should have received a copy of the GNU General Public License

# along with Marlon.  If not, see http://www.gnu.org/licenses



defmodule Marlon.Qlearning do

  @moduledoc """

    Implementation of the basic Q-learning algorithm

  """

use Class



  defstruct(

    discount_factor: 0.0, # gamma

    learning_rate: 1.0, # alpha

    initial_q: 0,

defclass Marlon.Qlearning do

  extends Marlon.LearningAlgorithm



    epsilon: 0.8,

    # temperature: 1.0, # Used by SoftMax strategy

  var discount_factor: 0.0 # gamma

  var learning_rate: 1.0 # alpha

  var initial_q: 0



    q_table: %{},

    prev_state: 0,

  var epsilon: 0.8

  #var temperature: 1.0, # Used by SoftMax strategy



    base: %Marlon.LearningAlgorithmBase{}

  )

end



defimpl Marlon.LearningAlgorithm, for: Marlon.Qlearning do



  # (These functions are all inherited from Marlon.LearningAlgorithmDefaults)

  defdelegate init_algorithm(goal, data), to: Marlon.LearningAlgorithmDefaults

  defdelegate number_of_actions(goal), to: Marlon.LearningAlgorithmDefaults

  defdelegate action_index_to_message(goal, action_index), to: Marlon.LearningAlgorithmDefaults

  defdelegate reward(goal, actor_state), to: Marlon.LearningAlgorithmDefaults

  defdelegate abstract_state(goal, actor_state), to: Marlon.LearningAlgorithmDefaults

  defdelegate best_action(goal), to: Marlon.LearningAlgorithmDefaults

  var q_table: %{}

  var prev_state: 0



  def init_agent(goal, actor_state) do

    if goal.learning_rate < 0 or goal.learning_rate > 1 do

  def init_agent(this, actor_state) do

    if this.learning_rate < 0 or this.learning_rate > 1 do

      raise "Q-learning configuration error: Learning rate must be in [0.0, 1.0]"

    end

    if goal.discount_factor < 0 or goal.discount_factor > 1 do

    if this.discount_factor < 0 or this.discount_factor > 1 do

      raise "Q-learning configuration error: Discount factor must be in [0.0, 1.0]"

    end



    s = abstract_state(goal, actor_state)

    new_b = %{goal.base | last_state: abstract_state(goal, actor_state)}

    %{goal | base: new_b, prev_state: s}

    s = Marlon.LearningAlgorithm.abstract_state(this, actor_state)

    %{this | last_state: s, prev_state: s}

  end



  def inc_timestep(_goal) do

  def inc_timestep(_this) do

    nil

  end



  def is_learning(_goal) do

  def is_learning(_this) do

    true

  end



  #Epsilon-greedy strategy: the greediest action is taken epsilon % of the time; otherwise a random action is taken

  def sample_action(goal) do

    if :rand.uniform > goal.epsilon do

      Marlon.Utils.random_int(0, number_of_actions(goal) - 1)

  def sample_action(this) do

    if :rand.uniform > this.epsilon do

      Marlon.Utils.random_int(0, number_of_actions(this) - 1)

    else

      greedy_sample_action(goal)

      greedy_sample_action(this)

    end

  end
@@ -115,15 +99,15 @@ defimpl Marlon.LearningAlgorithm, for: Marlon.Qlearning do 
  # end



  # Greedy strategy

  def greedy_sample_action(goal) do

  def greedy_sample_action(this) do



    num_actions = number_of_actions(goal)

    state = goal.base.last_state

    num_actions = number_of_actions(this)

    state = this.last_state

    [best_actions, _best_q_value, best_action_count] = Enum.reduce(

      1..num_actions-1,

      [%{0 => 0}, q(goal, state, 0), 1],

      [%{0 => 0}, q(this, state, 0), 1],

      fn(action, [best_actions, best_q_value, best_action_count]) ->

        q_val = q(goal, state, action)

        q_val = q(this, state, action)

        cond do

          (q_val > best_q_value) ->

            [Map.put(best_actions, 0, action), q_val, 1]
@@ -138,33 +122,33 @@ defimpl Marlon.LearningAlgorithm, for: Marlon.Qlearning do 
      Map.get(best_actions, selection)

  end



  def update(goal, reward) do

  def update(this, reward) do



    prev_state = goal.prev_state

    action = goal.base.last_action

    state = goal.base.last_state

    prev_state = this.prev_state

    action = this.last_action

    state = this.last_state



    max_q = max_coeff(goal, state)

    q_value = (1 - goal.learning_rate) * q(goal, prev_state, action) + goal.learning_rate * (reward + goal.discount_factor * max_q)

    q_table = Map.put(goal.q_table, [prev_state, action], q_value)

    max_q = max_coeff(this, state)

    q_value = (1 - this.learning_rate) * q(this, prev_state, action) + this.learning_rate * (reward + this.discount_factor * max_q)

    q_table = Map.put(this.q_table, [prev_state, action], q_value)



    %{goal | q_table: q_table, prev_state: state}

    %{this | q_table: q_table, prev_state: state}

  end



  # Estimate of the optimal future q-value

  def max_coeff(goal, state) do

    values = Enum.map(0..number_of_actions(goal),

      fn(action) -> q(goal, state, action) end)

  def max_coeff(this, state) do

    values = Enum.map(0..number_of_actions(this),

      fn(action) -> q(this, state, action) end)

    values = Enum.filter(values, &(&1 != nil))

    Enum.max(values)

  end



  # Get a value from the Q-table (given a certain state and action)

  def q(goal, state, action) do

    Map.get(goal.q_table, [state, action], goal.initial_q)

  def q(this, state, action) do

    Map.get(this.q_table, [state, action], this.initial_q)

  end



  def action_probabilities(_goal) do

  def action_probabilities(_this) do

    #TODO

  end

lib/marlon/marl/algorithms/sparsecooperativeqlearning.ex

0 → 100644
+26 −0
Original line number Diff line number Diff line 
# Marlon - Multi-Agent Reinforcement Learning On Networks

# Copyright (C) 2018 Smile-IT project

#

# This file is part of Marlon.

#

# Marlon is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# Marlon is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with Marlon.  If not, see http://www.gnu.org/licenses



use Class 



# defclass Marlon.SparseCooperativeQlearning do

#   extends Marlon.LearningAlgorithm

  

#   var discount_factor: 0.9 # gamma

#   var learning_rate: 0.3 # alpha

# end
 
 No newline at end of file