qlearning.h

/*
DESCRIPTION
C++ implementation of table-based Q-learning with Local (L), Global (G), Difference (D) and Abstract (A) rewards.

PAPER
Please cite our work as follows:
K. Malialis, S. Devlin and D. Kudenko. Resource Abstraction for Reinforcement Learning in Multiagent Congestion Problems. In Proceedings of the 15th International Conference on Autonomous Agents and Multiagent Systems (AAMAS), 2016.
*/

#ifndef QLEARNER_H
#define	QLEARNER_H

#ifndef VECTOR_INCLUDE
#define VECTOR_INCLUDE
#include <vector.h>
#include <math.h>
#endif

class QLearner {

    double learning_signal; /// The value used to update the Q table. G, L, or D are assigned to this value.

    double global_reward;
    double local_reward;
    double difference_reward;
    double coordinated_reward;		// Resource abstraction

    void console_1vector(vector<double>);

    int rand_action();
    int greedy_action();
    void initial_Qtable();
    void set_initial_state();
    void show_action();


    double Initial_Q_Value;

public:
    void console_2vector(vector<vector<double> >);
    vector< vector<double> > Qtable;
    int id;
    int previousState;
    int state;
    int action;
    double alpha;
    double gamma;
    double epsilon;
    void start();       //Used before first run of a statistical run / repeat
    void restart();     //Used at start of episode (i.e. does not change Qtable or learning parameters)
    void Qupdate();
    void final_Qupdate();
    void choose_egreedy_action();
    void choose_greedy_action();

    void set_local(double);
    void set_global(double);
    void set_difference(double);
    void set_coordinated(double);	// Resource abstraction

    void learn_with_global();
    void learn_with_difference();
    void learn_with_local();
    void learn_with_coordinated();	// Resource abstraction

    void decay_epsilon();
    void decay_alpha();
};

void QLearner::decay_epsilon()
{
    epsilon*=0.9999;
}
void QLearner::decay_alpha()
{
    alpha*=0.9999;
}

void QLearner::set_initial_state()
{
    state = id % LANES; //uniform initial distribution

    // Initialise states - NOTE: This version is specific to 5 lanes
/*
    if (id < NUM_AGENTS / 2) {
        state = 1;
    } else {
        state = 3;
    }
*/
}

void QLearner::start() {
    previousState = 0;
    set_initial_state();

    action = 0;
    alpha = 0.1;
    epsilon = 0.05;
    gamma = 1.0;
    Initial_Q_Value = -1;
    learning_signal = 0;

    global_reward = 0;
    local_reward = 0;
    difference_reward = 0;
    coordinated_reward = 0;		// Resource abstraction

    //Initialise Q-table, LANES(STATES)x3(ACTIONS)
    for (int i = 0; i < LANES; i++) {
        vector<double> vec;
        vec.resize(ACTIONS,0); /// create inner vector
        Qtable.push_back(vec); /// push inner vector into outer vector
    }
    /// Q = Qtable.at(state).at(action);
    initial_Qtable();
}

void QLearner::restart() {
    previousState = 0;
    set_initial_state();

    action = 0;

    learning_signal = 0;
    global_reward = 0;
    local_reward = 0;
    difference_reward = 0;
    coordinated_reward = 0;	// Resource abstraction
}

void QLearner::initial_Qtable() {
   for (int i = 0; i < Qtable.size(); i++) {
        for (int j = 0; j < Qtable.at(i).size(); j++) {
                Qtable.at(i).at(j) = Initial_Q_Value + LYRAND*SMALL - LYRAND*SMALL;
        }
   }
}

void QLearner::console_2vector(vector< vector<double> > a) {
    for (int i = 0; i < a.size(); i++) {
        console_1vector(a.at(i));
        cout << endl;
    }
}

void QLearner::console_1vector(vector<double> a) {
    for (int i = 0; i < a.size(); i++) {
        cout << a.at(i);
        cout << "\t";
    }
}

void QLearner::choose_egreedy_action() {
    double a = (double) rand() / RAND_MAX;
    if (a < epsilon) {
        action = rand_action();
    } else {
        action = greedy_action();
    }
}

void QLearner::choose_greedy_action() {
    action = greedy_action();
}

int QLearner::rand_action() {
    int a;
    a = rand() % ACTIONS;
    return a;
}

int QLearner::greedy_action() {
    int LL = ACTIONS;
    double best = -9999999999;
    int bestdex = -1;
    for (int i = 0; i < LL; i++) {
        if (Qtable.at(state).at(i) > best) {
            best = Qtable.at(state).at(i);
            bestdex = i;
        }
    }
    return bestdex;
}

void QLearner::show_action() {
   // cout << "Agent " << index << ": " << action << endl;
}

void QLearner::learn_with_global() {
    learning_signal = global_reward;
}

void QLearner::learn_with_difference() {
    learning_signal = difference_reward;
}

void QLearner::learn_with_local() {
    learning_signal = local_reward;
}

// Resource abstraction
void QLearner::learn_with_coordinated() {
	learning_signal = coordinated_reward;
}

void QLearner::Qupdate() {
    double Q = Qtable.at(previousState).at(action);

    double Qmax = -9999999999;
    for (int i = 0; i < ACTIONS; i++) {
        if (Qtable.at(state).at(i) > Qmax) {
            Qmax = Qtable.at(state).at(i);
        }
    }

    //cout << "Q before: " << Q << endl;
    //cout << "In state: " << state << endl;
    //cout << "For action: " << action << endl;
    Q = Q + alpha * (learning_signal + gamma * Qmax - Q);
    //cout << "Q after: "  << Q << endl;
    Qtable.at(previousState).at(action) = Q;
}

void QLearner::final_Qupdate() {
    double Q = Qtable.at(state).at(action);
    //cout << "Q before: " << Q << endl;
    //cout << "In state: " << state << endl;
    //cout << "For action: " << action << endl;
    Q = Q + alpha * (- Q);
    //cout << "Q after: "  << Q << endl;
    Qtable.at(state).at(action) = Q;
}


void QLearner::set_local(double L) {
    local_reward = L;
}

void QLearner::set_global(double G) {
    global_reward = G;
}

void QLearner::set_difference(double D) {
    difference_reward = D;
}

// Resource abstraction
void QLearner::set_coordinated(double C) {
	coordinated_reward = C;
}

#endif	/* QLEARNER_H */