Modify the tools to support an extra constant, D4, for discounting parts of counts larger than 3.

Author: danpovey

egs/swbd/run.sh

@@ -47,9 +47,7 @@ get_initial_metaparameters.py \
 optimize_metaparameters.py --gradient-tolerance=0.005 \
   data/counts_20k_3 data/optimize_20k_3
 
-# optimize_metaparameters.py: log-prob on dev data increased from -4.42783439035
-# to -4.41743853837 over 9 passes of derivative estimation (perplexity:
-# 83.7498508954->82.8837097801
+# log-prob on dev data increased from -4.42278966972 to -4.41127767165 over 6 passes of derivative estimation (perplexity: 83.3284201887->82.3746440442)
 
 
 get_counts.sh data/int_20k 4 data/counts_20k_4
@@ -63,6 +61,6 @@ get_initial_metaparameters.py \
 optimize_metaparameters.py --gradient-tolerance=0.005 \
   data/counts_20k_4 data/optimize_20k_4
 
-# optimize_metaparameters.py: log-prob on dev data increased from -4.42864701686
-# to -4.38964142483 over 13 passes of derivative estimation (perplexity:
-# 83.8179359045->80.6115085121
+# optimize_metaparameters.py: log-prob on dev data increased from -4.4224930661
+# to -4.38089709795 over 13 passes of derivative estimation (perplexity:
+# 83.3037083426->79.909688077

scripts/get_initial_metaparameters.py

@@ -145,4 +145,5 @@ def ReadWeights(weights_file):
     print("order{0}_D1 0.8".format(o))
     print("order{0}_D2 0.4".format(o))
     print("order{0}_D3 0.2".format(o))
+    print("order{0}_D4 0.1".format(o))
 

scripts/get_objf_and_derivs.py

@@ -61,15 +61,17 @@
 train_set_scale = {}
 for n in range(1, num_train_sets + 1):
     train_set_scale[n] = float(f.readline().split()[1])
-# the discounting constants will be stored as maps d1,d2,d3 from integer order
+# the discounting constants will be stored as maps d1,d2,d3,d4 from integer order
 # to discounting constant.
 d1 = {}
 d2 = {}
 d3 = {}
+d4 = {}
 for o in range(2, ngram_order + 1):
     d1[o] = float(f.readline().split()[1])
     d2[o] = float(f.readline().split()[1])
     d3[o] = float(f.readline().split()[1])
+    d4[o] = float(f.readline().split()[1])
 f.close()
 
 
@@ -149,27 +151,28 @@ def MergeCountsBackward(order):
 def DiscountCounts(order):
     # discount counts of the specified order > 1.
     assert order > 1
-    command = "discount-counts {d1} {d2} {d3} {work}/merged.{order} {work}/float.{order} {work}/discounted.{orderm1} ".format(
-        d1 = d1[order], d2 = d2[order], d3 = d3[order], work = args.work_dir,
-        order = order, orderm1 = order - 1)
+    command = "discount-counts {d1} {d2} {d3} {d4} {work}/merged.{order} {work}/float.{order} {work}/discounted.{orderm1} ".format(
+        d1 = d1[order], d2 = d2[order], d3 = d3[order], d4 = d4[order],
+        work = args.work_dir, order = order, orderm1 = order - 1)
     RunCommand(command)
 
 def DiscountCountsBackward(order):
     # discount counts of the specified order > 1; backprop version.
     assert order > 1
-    command = ("discount-counts-backward {d1} {d2} {d3} {work}/merged.{order} {work}/float.{order} "
+    command = ("discount-counts-backward {d1} {d2} {d3} {d4} {work}/merged.{order} {work}/float.{order} "
                "{work}/float_derivs.{order} {work}/discounted.{orderm1} {work}/discounted_derivs.{orderm1} "
                "{work}/merged_derivs.{order}".format(
-            d1 = d1[order], d2 = d2[order], d3 = d3[order], work = args.work_dir,
-            order = order, orderm1 = order - 1))
+            d1 = d1[order], d2 = d2[order], d3 = d3[order], d4 = d4[order],
+            work = args.work_dir, order = order, orderm1 = order - 1))
     output = GetCommandStdout(command);
     try:
-        [ deriv1, deriv2, deriv3 ] = output.split()
+        [ deriv1, deriv2, deriv3, deriv4 ] = output.split()
     except:
         sys.exit("get_objf_and_derivs.py: could not parse output of command: " + output)
     d1_deriv[order] = float(deriv1) / num_dev_set_words
     d2_deriv[order] = float(deriv2) / num_dev_set_words
     d3_deriv[order] = float(deriv3) / num_dev_set_words
+    d4_deriv[order] = float(deriv4) / num_dev_set_words
 
 
 def DiscountCountsOrder1():
@@ -226,6 +229,7 @@ def WriteDerivs():
         print("order{0}_D1 {1}".format(o, d1_deriv[o]), file=f)
         print("order{0}_D2 {1}".format(o, d2_deriv[o]), file=f)
         print("order{0}_D3 {1}".format(o, d3_deriv[o]), file=f)
+        print("order{0}_D4 {1}".format(o, d4_deriv[o]), file=f)
     f.close()
 
 # for n-gram orders down to 2, do the merging and discounting.
@@ -247,6 +251,7 @@ def WriteDerivs():
 d1_deriv = {}
 d2_deriv = {}
 d3_deriv = {}
+d4_deriv = {}
 
 # Now comes the backprop code.
 

scripts/optimize_metaparameters.py

@@ -99,16 +99,17 @@ def WriteMetaparameters(file, array):
 # d3 > 0.  Otherwise it returns false.
 def MetaparametersAreAllowed(x):
     global num_train_sets, ngram_order
-    assert len(x) == num_train_sets + 3 * (ngram_order - 1)
+    assert len(x) == num_train_sets + 4 * (ngram_order - 1)
     for i in range(num_train_sets):
         if x[i] <= 0.0 or x[i] >= 1.0:
             return False
     for o in range(2, ngram_order + 1):
-        dim_offset = num_train_sets + 3 * (o-2)
+        dim_offset = num_train_sets + 4 * (o-2)
         d1 = x[dim_offset]
         d2 = x[dim_offset + 1]
         d3 = x[dim_offset + 2]
-        if not (1.0 > d1 and d1 > d2 and d2 > d3 and d3 > 0.0):
+        d4 = x[dim_offset + 3]
+        if not (1.0 > d1 and d1 > d2 and d2 > d3 and d3 > d4 and d4 > 0.0):
             return False
     return True
 
@@ -124,30 +125,35 @@ def ModifyWithBarrierFunction(x, objf, derivs):
     epsilon = args.barrier_epsilon
     derivs = derivs.copy() # don't overwrite the object.
     global num_train_sets, ngram_order
-    assert len(x) == num_train_sets + 3 * (ngram_order - 1)
+    assert len(x) == num_train_sets + 4 * (ngram_order - 1)
     for i in range(num_train_sets):
         xi = x[i]
         # the constraints are: xi > 0.0, and 1.0 - xi > 0.0
         objf += epsilon * (log(xi - 0.0) + log(1.0 - xi))
         derivs[i] += epsilon * ((1.0 / xi) + (-1.0 / (1.0 - xi)))
 
     for o in range(2, ngram_order + 1):
-        dim_offset = num_train_sets + 3 * (o-2)
+        dim_offset = num_train_sets + 4 * (o-2)
         d1 = x[dim_offset]
         d2 = x[dim_offset + 1]
         d3 = x[dim_offset + 2]
+        d4 = x[dim_offset + 3]
         # the constraints are:
         # 1.0 - d1 > 0.0
         # d1 - d2 > 0.0
         # d2 - d3 > 0.0
-        #      d3 > 0.0
-        objf += epsilon * (log(1.0 - d1) + log(d1 - d2) + log(d2 - d3) + log(d3))
+        # d3 - d4 > 0.0
+        #      d4 > 0.0
+        objf += epsilon * (log(1.0 - d1) + log(d1 - d2) + log(d2 - d3) +
+                           log(d3 - d4) + log(d4))
         # deriv for d1
         derivs[dim_offset] += epsilon * (-1.0 / (1.0 - d1) + 1.0 / (d1 - d2))
         # deriv for d2
         derivs[dim_offset + 1] += epsilon * (-1.0 / (d1 - d2) + 1.0 / (d2 - d3))
         # deriv for d3
-        derivs[dim_offset + 2] += epsilon * (-1.0 / (d2 - d3) + 1.0 / d3)
+        derivs[dim_offset + 2] += epsilon * (-1.0 / (d2 - d3) + 1.0 / (d3 - d4))
+        # deriv for d4
+        derivs[dim_offset + 3] += epsilon * (-1.0 / (d3 - d4) + 1.0 / d4)
     return (objf, derivs)
 
 

scripts/validate_metaparameter_derivs.py

@@ -63,7 +63,7 @@
                                                             deriv_line[0:-1]))
 
 for o in range(2, args.ngram_order + 1):
-    for n in range(3):
+    for n in range(4):
         line = f.readline()
         deriv_line = deriv_f.readline()
         try:

scripts/validate_metaparameters.py

@@ -51,27 +51,22 @@
                                                      args.metaparameter_file))
 
 for o in range(2, args.ngram_order + 1):
-    line1 = f.readline()
-    line2 = f.readline()
-    line3 = f.readline()
+    lines = []
+    values = []
+    for n in range(4):
+        lines.append(f.readline())
     try:
-        [ name1, value1 ] = line1.split()
-        [ name2, value2 ] = line2.split()
-        [ name3, value3 ] = line3.split()
-        value1 = float(value1)
-        value2 = float(value2)
-        value3 = float(value3)
-        assert name1 == "order{0}_D1".format(o)
-        assert name2 == "order{0}_D2".format(o)
-        assert name3 == "order{0}_D3".format(o)
-        assert 1.0 > value1 and value1 > value2 and value2 > value3 and value3 > 0.0
-    except:
+        for n in range(4):
+            [ name, value ] = lines[n].split()
+            assert name == "order{0}_D{1}".format(o, n + 1)
+            value = float(value)
+            values.append(value)
+            assert 1.0 > value and value > 0.0 and (n == 0 or value < values[n-1])
+    except Exception as e:
         sys.exit("validate_metaparameters.py: bad values for {0}'th order "
-                 "n-gram discounting parameters: '{1}', '{2}', '{3}',"
-                 " in file {4}".format(o, line1[0:-1], line2[0:-1], line3[0:-1],
-                                       args.metaparameter_file))
+                 "n-gram discounting parameters: in file {1}: {2}".format(
+                o, args.metaparameter_file, str(e)))
 
 if f.readline() != '':
     sys.exit("validate_metaparameters.py: junk at end of "
              "metaparameters file {0}".format(args.metaparameter_file))
-

src/count.h

@@ -28,13 +28,11 @@ namespace pocolm {
 
 /**
    This class is used to store a special type of count that we use in estimating
-   these language models.  You can think of it as a type of 'extended' float that
-   stores the sum of a bunch of individual small counts or parts of counts.
+   these language models.  You can think of it as a type of 'extended' float
+   that stores the sum of a bunch of individual small counts or parts of counts.
    In addition to storing the total count, it also stores the top-1 "part"
-   (i.e. the largest of the component parts), and also the runners up, which
+   (i.e. the largest of the component parts), and also the two runners up, which
    we call top-2 and top-3.
-
-
  */
 class Count {
  public: