Improvements: Tests

estimators/test/test_bandits.py

@@ -10,39 +10,41 @@
 from estimators.bandits import clopper_pearson
 from estimators.test.utils import Helper
 
-def test_bandits_unit_test():
-    listofestimators = [(ips.Estimator(), 2.0), (snips.Estimator(), 1.0), (mle.Estimator(), 1.0), (cressieread.Estimator(), 1.0)]
-
+random.seed(0)
+
+def test_single_example():
+    estimators = [(ips.Estimator(), 2.0), (snips.Estimator(), 1.0), (mle.Estimator(), 1.0), (cressieread.Estimator(), 1.0)]
+
     p_log = 0.3
     p_pred = 0.6
     reward = 1
 
-    for Estimator in listofestimators:
+    for Estimator in estimators:
         Estimator[0].add_example(p_log, reward, p_pred)
         assert Estimator[0].get() == Estimator[1]
 
 
-def test_bandits():
+def test_multiple_examples():
     ''' To test correctness of estimators: Compare the expected value with value returned by Estimator.get()'''
 
-    # The tuple (Estimator, expected value) for each estimator is stored in listofestimators
-    listofestimators = [(ips.Estimator(), 1), (snips.Estimator(), 1), (mle.Estimator(), 1), (cressieread.Estimator(), 1)]
+    # The tuple (Estimator, expected value) for each estimator is stored in estimators
+    estimators = [(ips.Estimator(), 1), (snips.Estimator(), 1), (mle.Estimator(), 1), (cressieread.Estimator(), 1)]
 
     def datagen():
         return  {'p_log': 1,
                 'r': 1,
                 'p_pred': 1}
 
-    estimates = Helper.get_estimate(datagen, listofestimators=[l[0] for l in listofestimators], num_examples=4)
+    estimates = Helper.get_estimate(datagen, estimators=[l[0] for l in estimators], num_examples=4)
 
-    for Estimator, estimate in zip(listofestimators, estimates):
+    for Estimator, estimate in zip(estimators, estimates):
         Helper.assert_is_close(Estimator[1], estimate)
 
 
 def test_narrowing_intervals():
     ''' To test for narrowing intervals; Number of examples increase => narrowing CI '''
 
-    listofintervals = [cressieread.Interval(), gaussian.Interval(), clopper_pearson.Interval()]
+    intervals = [cressieread.Interval(), gaussian.Interval(), clopper_pearson.Interval()]
 
     def datagen(epsilon, delta=0.5):
         # Logged Policy
@@ -59,15 +61,15 @@ def datagen(epsilon, delta=0.5):
                 'r': int(random.random() < 1-delta) if chosen == 1 else int(random.random() < delta),
                 'p_pred': int(chosen==1)}
 
-    intervals_n1 = Helper.get_estimate(lambda: datagen(epsilon=0.5), listofintervals, num_examples=100)
-    intervals_n2 = Helper.get_estimate(lambda: datagen(epsilon=0.5), listofintervals, num_examples=10000)
+    intervals_less_data = Helper.get_estimate(lambda: datagen(epsilon=0.5), intervals, num_examples=100)
+    intervals_more_data = Helper.get_estimate(lambda: datagen(epsilon=0.5), intervals, num_examples=10000)
 
-    for interval_n1, interval_n2 in zip(intervals_n1, intervals_n2):
-        width_n1 = interval_n1[1] - interval_n1[0]
-        width_n2 = interval_n2[1] - interval_n2[0]
-        assert width_n1 > 0
-        assert width_n2 > 0
-        assert width_n2 < width_n1
+    for interval_less_data, interval_more_data in zip(intervals_less_data, intervals_more_data):
+        width_wider = interval_less_data[1] - interval_less_data[0]
+        width_narrower = interval_more_data[1] - interval_more_data[0]
+        assert width_wider > 0
+        assert width_narrower > 0
+        assert width_narrower < width_wider
 
 
 def test_cats_ips():
@@ -115,7 +117,7 @@ def test_cats_transformer_on_edges():
         data['a'] = logged_action
         data['cost'] = r
         data['p'] = logged_prob
-    
+
         pred_action = logged_action
         data = cats_transformer.transform(data, logged_action) # same action, so pred_p should be 1
         assert data['pred_p'] == 1.0 / (2 * bandwidth)

estimators/test/test_ccb.py

@@ -10,25 +10,27 @@
 from estimators.ccb import first_slot
 from estimators.test.utils import Helper
 
-def test_ccb_unit_test():
-    listofestimators = [(first_slot.Estimator(ips.Estimator()), 2.0),
+random.seed(0)
+
+def test_single_example():
+    estimators = [(first_slot.Estimator(ips.Estimator()), 2.0),
                         (first_slot.Estimator(snips.Estimator()), 1.0),
                         (first_slot.Estimator(mle.Estimator()), 1.0),
                         (first_slot.Estimator(cressieread.Estimator()), 1.0)]
-    
+
     p_log = [0.3]
     p_pred = [0.6]
     reward = [1]
 
-    for Estimator in listofestimators:
+    for Estimator in estimators:
         Estimator[0].add_example(p_log, reward, p_pred)
         assert Estimator[0].get() == Estimator[1]
 
-def test_ccb():
+def test_multiple_examples():
     ''' To test correctness of estimators: Compare the expected value with value returned by Estimator.get()'''
 
-    # The tuple (Estimator, expected value) for each estimator is stored in listofestimators
-    listofestimators = [(first_slot.Estimator(ips.Estimator()), 1.0),
+    # The tuple (Estimator, expected value) for each estimator is stored in estimators
+    estimators = [(first_slot.Estimator(ips.Estimator()), 1.0),
                         (first_slot.Estimator(snips.Estimator()), 1.0),
                         (first_slot.Estimator(mle.Estimator()), 1.0),
                         (first_slot.Estimator(cressieread.Estimator()), 1.0)]
@@ -43,18 +45,18 @@ def datagen_single_slot_value():
                 'r': [1],
                 'p_pred': [1]}
 
-    estimates_multiple = Helper.get_estimate(datagen_multiple_slot_values, listofestimators=[l[0] for l in listofestimators], num_examples=4)
-    estimates_single = Helper.get_estimate(datagen_single_slot_value, listofestimators=[l[0] for l in listofestimators], num_examples=4)
+    estimates_multiple = Helper.get_estimate(datagen_multiple_slot_values, estimators=[l[0] for l in estimators], num_examples=4)
+    estimates_single = Helper.get_estimate(datagen_single_slot_value, estimators=[l[0] for l in estimators], num_examples=4)
 
-    for Estimator, estimate_multiple, estimate_single in zip(listofestimators, estimates_multiple, estimates_single):
+    for Estimator, estimate_multiple, estimate_single in zip(estimators, estimates_multiple, estimates_single):
         Helper.assert_is_close(Estimator[1], estimate_multiple)
         Helper.assert_is_close(Estimator[1], estimate_single)
         assert estimate_single == estimate_multiple
 
 def test_narrowing_intervals():
     ''' To test for narrowing intervals; Number of examples increase => narrowing CI '''
 
-    listofintervals = [first_slot.Interval(cressieread.Interval()), first_slot.Interval(gaussian.Interval()), first_slot.Interval(clopper_pearson.Interval())]
+    intervals = [first_slot.Interval(cressieread.Interval()), first_slot.Interval(gaussian.Interval()), first_slot.Interval(clopper_pearson.Interval())]
 
     def datagen(epsilon, delta=0.5):
         # Logged Policy
@@ -71,12 +73,12 @@ def datagen(epsilon, delta=0.5):
                 'r': [int(random.random() < 1-delta) if chosen == 1 else int(random.random() < delta)],
                 'p_pred': [int(chosen==1)]}
 
-    intervals_n1 = Helper.get_estimate(lambda: datagen(epsilon=0.5), listofintervals, num_examples=100)
-    intervals_n2 = Helper.get_estimate(lambda: datagen(epsilon=0.5), listofintervals, num_examples=10000)
+    intervals_less_data = Helper.get_estimate(lambda: datagen(epsilon=0.5), intervals, num_examples=100)
+    intervals_more_data = Helper.get_estimate(lambda: datagen(epsilon=0.5), intervals, num_examples=10000)
 
-    for interval_n1, interval_n2 in zip(intervals_n1, intervals_n2):
-        width_n1 = interval_n1[1] - interval_n1[0]
-        width_n2 = interval_n2[1] - interval_n2[0]
-        assert width_n1 > 0
-        assert width_n2 > 0
-        assert width_n2 < width_n1
+    for interval_less_data, interval_more_data in zip(intervals_less_data, intervals_more_data):
+        width_wider = interval_less_data[1] - interval_less_data[0]
+        width_narrower = interval_more_data[1] - interval_more_data[0]
+        assert width_wider > 0
+        assert width_narrower > 0
+        assert width_narrower < width_wider

estimators/test/test_slates.py

@@ -6,6 +6,8 @@
 from estimators.bandits import ips
 from estimators.test.utils import Helper
 
+random.seed(0)
+
 def test_single_slot_pi_equivalent_to_ips():
     ''' PI should be equivalent to IPS when there is only a single slot '''
 
@@ -21,11 +23,11 @@ def test_single_slot_pi_equivalent_to_ips():
         ips_estimator.add_example(p_log, r, p_pred)
         Helper.assert_is_close(pi_estimator.get() , ips_estimator.get())
 
-def test_slates():
+def test_multiple_slots():
     ''' To test correctness of estimators: Compare the expected value with value returned by Estimator.get()'''
 
-    # The tuple (Estimator, expected value) for each estimator is stored in listofestimators
-    listofestimators = [(pseudo_inverse.Estimator(), 1)]
+    # The tuple (Estimator, expected value) for each estimator is stored in estimators
+    estimators = [(pseudo_inverse.Estimator(), 1)]
 
     def datagen(num_slots):
         # num_slots represents the len(p_logs) or len(p_pred) for each example
@@ -40,15 +42,15 @@ def datagen(num_slots):
     # p_logs = [1,1,1,1]
     # p_pred = [1,1,1,1]
     # reward = 1
-    estimates = Helper.get_estimate(lambda: datagen(num_slots=4), listofestimators=[l[0] for l in listofestimators], num_examples=4)
+    estimates = Helper.get_estimate(lambda: datagen(num_slots=4), estimators=[l[0] for l in estimators], num_examples=4)
 
-    for Estimator, estimate in zip(listofestimators, estimates):
+    for Estimator, estimate in zip(estimators, estimates):
         Helper.assert_is_close(Estimator[1], estimate)
 
 def test_narrowing_intervals():
     ''' To test for narrowing intervals; Number of examples increase => narrowing CI '''
 
-    listofintervals = [gaussian.Interval()]
+    intervals = [gaussian.Interval()]
 
     def datagen(num_slots, epsilon, delta=0.5):
 
@@ -71,12 +73,12 @@ def datagen(num_slots, epsilon, delta=0.5):
 
         return data
 
-    intervals_n1 = Helper.get_estimate(lambda: datagen(num_slots=4, epsilon=0.5), listofintervals, num_examples=100)
-    intervals_n2 = Helper.get_estimate(lambda: datagen(num_slots=4, epsilon=0.5), listofintervals, num_examples=10000)
+    intervals_less_data = Helper.get_estimate(lambda: datagen(num_slots=4, epsilon=0.5), intervals, num_examples=100)
+    intervals_more_data = Helper.get_estimate(lambda: datagen(num_slots=4, epsilon=0.5), intervals, num_examples=10000)
 
-    for interval_n1, interval_n2 in zip(intervals_n1, intervals_n2):
-        width_n1 = interval_n1[1] - interval_n1[0]
-        width_n2 = interval_n2[1] - interval_n2[0]
-        assert width_n1 > 0
-        assert width_n2 > 0
-        assert width_n2 < width_n1
+    for interval_less_data, interval_more_data in zip(intervals_less_data, intervals_more_data):
+        width_wider = interval_less_data[1] - interval_less_data[0]
+        width_narrower = interval_more_data[1] - interval_more_data[0]
+        assert width_wider > 0
+        assert width_narrower > 0
+        assert width_narrower < width_wider

estimators/test/utils.py

@@ -21,9 +21,9 @@ def run_add_example(datagen, estimator, num_examples):
         return Estimator
 
     @staticmethod
-    def get_estimate(datagen, listofestimators, num_examples):
+    def get_estimate(datagen, estimators, num_examples):
         estimates = []
-        for Estimator in listofestimators:
+        for Estimator in estimators:
 
             estimator = Helper.run_add_example(datagen, Estimator, num_examples)
             estimates.append(estimator.get())