main.py

# -*- coding: utf-8 -*-
"""contextualBandits.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1j49NAkMl1stMplDx0wwO2MRUTNZlk30V
"""

import numpy as np
import sys
import os
import Context
import scipy
import matplotlib.pyplot as plt

py_file_location = "/content/drive/MyDrive/Colab Notebooks"
sys.path.append(os.path.abspath(py_file_location))

flist = ["/content/drive/MyDrive/Colab Notebooks/data/set1.train.txt",
         "/content/drive/MyDrive/Colab Notebooks/data/set1.valid.txt",
         "/content/drive/MyDrive/Colab Notebooks/data/set1.test.txt",
         "/content/drive/MyDrive/Colab Notebooks/data/set2.train.txt",
         "/content/drive/MyDrive/Colab Notebooks/data/set2.valid.txt",
         "/content/drive/MyDrive/Colab Notebooks/data/set2.test.txt"]

def preprocess():
    fidx = 0
    fname = flist[fidx]

    queryIds = {}
    queryDocs = {}
    order = []

    set1_features = set([])
    set2_features = set([])

    maxDocsPerQuery = 0
    print("Opening new file: %s" % (fname))
    f = open(fname,'r')
    num_lines = 0
    while True:
        curr_line = None
        done = False
        while curr_line == None:
            try:
                curr_line = f.__next__()
            except Exception:
                fidx += 1
                curr_line = None
                if fidx == len(flist):
                    done = True
                    break
                fname = flist[fidx]
                print("Opening new file: %s" % (fname))
                f = open(fname, 'r')
        if done:
            break
        num_lines += 1
        data = curr_line.split()
        curr_q = int(data[1].split(":")[-1])
        rel = int(data[0])
        features = dict([[int(x.split(":")[0]), float(x.split(":")[-1])] for x in data[2:]])

        if fname.find("set1") != -1:
            set1_features.update(set(features.keys()))
        if fname.find("set2") != -1:
            set2_features.update(set(features.keys()))

        if curr_q not in queryIds:
            queryIds[curr_q] = len(queryIds)
            queryDocs[queryIds[curr_q]] = 0

        queryid = queryIds[curr_q]
        order.append(queryid)
        doc_id = queryDocs[queryid]
        queryDocs[queryid] += 1
        if queryDocs[queryid] > maxDocsPerQuery:
            maxDocsPerQuery = queryDocs[queryid]

    numQueries = len(queryIds)
    numDocs = np.minimum(maxDocsPerQuery, 6)

    ## TODO: update
    feature_set = set1_features & set2_features
    numFeatures = len(feature_set)
    feature_map = dict(zip(list(feature_set), range(len(feature_set))))

    print("Yahoo info compiled statistics",
          " NumQueries, NumUnique, MaxNumDocs, MaxNumFeatures: ", numQueries, len(order), numDocs, numFeatures, flush = True)

    docsPerQuery = np.zeros(numQueries, dtype = np.int32)
    processed = {}
    for qid in queryIds.keys():
        docsPerQuery[queryIds[qid]] = queryDocs[queryIds[qid]]
        processed[queryIds[qid]] = 0
    relevances = -np.ones((numQueries, numDocs), dtype = np.int32)
    features = np.nan * np.ones((numQueries, numDocs, numFeatures), dtype = np.float32)

    fidx = 0
    fname = flist[fidx]
    print("Opening new file: %s" % (fname))
    f = open(fname,'r')
    num_lines = 0
    while True:
        curr_line = None
        done = False
        while curr_line == None:
            try:
                curr_line = f.__next__()
            except Exception:
                fidx += 1
                curr_line = None
                if fidx == len(flist):
                    done = True
                    break
                fname = flist[fidx]
                print("Opening new file: %s" % (fname))
                f = open(fname, 'r')
        if done:
            break

        num_lines += 1
        # if num_lines % 10000 == 0:
        #     print("Processed %d lines" % (num_lines))

        data = curr_line.split()
        relevance = int(data[0])
        qid = int(data[1].split(":")[-1])
        queryid = queryIds[qid]
        fvec = dict([[int(x.split(":")[0]), float(x.split(":")[-1])] for x in data[2:]])
        arr = np.array(np.zeros(numFeatures))
        for (k,v) in fvec.items():
            if k in feature_map.keys():
                arr[feature_map[k]] = v

        docIndex = processed[queryid]
        if docIndex >= numDocs:
            docsPerQuery[queryid] = numDocs
            continue
        processed[queryid] += 1
        relevances[queryid,docIndex] = relevance
        features[queryid,docIndex,:] = arr

    np.savez_compressed('/content/drive/MyDrive/Colab Notebooks/data/yahoo_big.npz', relevances=relevances,
                        features = features, docsPerQuery = docsPerQuery, queryOrder = order)
    print("Yahoo info"
          " [Min,Max] number of Docs: ", np.min(docsPerQuery), np.max(docsPerQuery), flush = True)

    #for i in range(orders):
    #    o = np.random.permutation(len(docsPerQuery))
    #    np.savez_compressed('/content/drive/MyDrive/Colab Notebooks/data/yahoo_train_%d.npz' % (i), order=o)
    #print("PreloadYahoo:preprocess [INFO] Generate %d shuffles" % (orders))

    return (docsPerQuery, relevances, features)


class YahooContextIterator(object):
    def __init__(self, K=6, L=2, train=True, loop=False):
        ## Ignore train flag we don't have test data
        self.d = 415

        self.loop = loop
        npFile = np.load('/content/drive/MyDrive/Colab Notebooks/data/yahoo_big.npz')
        self.L = L
        self.K = K
        self.relevances = npFile['relevances']
        self.features = npFile['features']
        self.docsPerQuery = npFile['docsPerQuery']
        self.features = np.nan_to_num(self.features)

        toretain = np.where(self.docsPerQuery >= self.K)[0]
        self.relevances = self.relevances[toretain,:self.K]
        self.features = self.features[toretain,:self.K,:]
        self.docsPerQuery = self.K*np.ones(len(toretain),dtype=np.int)
        self.X = len(self.docsPerQuery)

        self.order = np.random.permutation(self.X)
        self.curr_idx = 0
        self.has_ldf = True

    def next(self):
        if self.loop:
            self.curr_idx = np.random.randint(0, len(self.docsPerQuery))
        if self.curr_idx >= len(self.docsPerQuery):
            return None
        q_idx = self.order[self.curr_idx]
        curr_x = Context.Context(q_idx, self.features[q_idx,:,:], self.docsPerQuery[q_idx], self.L)
        curr_r = self.relevances[q_idx,:]
        self.curr_idx += 1
        return (curr_x, curr_r)

    def get_all_relevances(self):
        return self.relevances

#preprocess()
yahoo = YahooContextIterator()

x, r = yahoo.next()
print(x.get_ld_features())
print(x.get_name())
print(x.get_K())
print(r)

class OFUL(object):
    """
    Implementation of OFUL
    """
    def __init__(self, T = 50000, d = 415, reg_param = 1):
        """
        Initialize the regression target and the 
        feature covariance. 
        """
        self.T = T
        self.d = d
        self.reg_param = reg_param
        self.b_vec = np.matrix(np.zeros((self.d,1)))
        self.cov = np.matrix(self.reg_param*np.eye(self.d))
        self.Cinv = scipy.linalg.inv(self.cov)
        self.weights = self.Cinv*self.b_vec
        self.t = 1
        self.reward = []
        self.opt_reward = []

    def update(self, x, act, est_r, r):
        """
        Update the regression target and feature cov. 
        """
        features = np.matrix(x.get_ld_features())
        obs_r = r[act] + np.random.normal(0,1)
        self.cov += features[act,:].T*features[act,:]
        self.b_vec += obs_r*features[act,:].T

        self.t += 1
        
        self.Cinv = scipy.linalg.inv(self.cov)
        self.weights = self.Cinv*self.b_vec
        #self.opt_reward.append(max(r))

    def get_action(self, x):
        """
        Find the predicted reward for each base action
        and play the action that maximizes the reward. 
        """
        features = np.matrix(x.get_ld_features())
        K = x.get_K()

        ucbs = [features[k,:]*self.weights for k in range(K)]

        ucbs = [a[0,0] for a in ucbs]
        #ranks = np.argsort(ucbs)
        est_r = np.max(ucbs)
        idx = np.where(ucbs == est_r)
        id = np.random.choice(idx[0], size=1, replace=False)
        #self.reward.append(est_r)
        return id[0], est_r
    def play(self, context_object): 
      for t in range(self.T):
            #if t % 100 == 0: print('Iteration: ', t)
            x, r = context_object.next()
            if x == None:
                break
            act, est_r = self.get_action(x)
            self.update(x, act, est_r, r)
            self.reward.append(est_r)
            self.opt_reward.append(max(r))
      l1 = np.cumsum(self.reward)
      l2 = np.cumsum(self.opt_reward)            
      self.cum_regret = abs(l2-l1)

oful = OFUL()
oful.play(yahoo)
plt.plot(oful.cum_regret, linewidth = 3)
plt.xlabel('Iterations',fontsize = 14)
plt.ylabel('Cumulative regret', fontsize = 14)
plt.grid()

plt.plot(oful.cum_regret, linewidth = 3)
plt.xlabel('Iterations',fontsize = 14)
plt.ylabel('Cumulative regret', fontsize = 14)
plt.grid()