new features, like saving the stats files

Author: Masoud

.gitignore

@@ -3,6 +3,7 @@
 *.pyo
 test*
 note.txt
+quartiles
 # Except this file
 !.gitignore
 

config.json

@@ -1,36 +1,43 @@
 {
 	"options": {
-		"input file":				"sample_en_it.csv",
+		"input file1":				"sample_en_it.csv",
+		"input file":				"en_it_1m_with_bgold",
 
-		"align file":				"sample_align",
-		"token file":				"sample_token",
+		"align file1":				"sample_align",
+		"token file1":				"sample_token",
+		"align file":				"en_it_1m_with_bgold.align",
+		"token file":				"en_it_1m_with_bgold.tok",
 
 		"output folder":			"output",
-		"source language":			"English",
-		"target language":			"Italian",
+		"source language":			"en",
+		"target language":			"it",
 
-		"no out files":				1
+		"normalize scores":			"true",
+		"emit scores":				"true",
+		"no out files":				"true",
+
+		"max decision":				"0"
 	},
 
 	"policies": [
-		["OneNo",					"off"],
+		["OneNo",					"on"],
 		["TwentyNo",				"off"],
 		["MajorityVoting",			"off"],
 		["SingleFilterPolicy",		"off"]
 	],
 
 	"filters": [
-		["SampleFilter",					"on"],
+		["SampleFilter",					"off"],
 		["LengthStats",						"off"],
 
-		["LengthRatio",						"off"],
-		["ReverseLengthRatio",				"off"],
-		["WordRatio",						"off"],
-		["ReverseWordRatio",				"off"],
-		["WordLength",						"off"],
-		["TagFinder",						"off"],
-		["RepeatedChars",					"off"],
-		["RepeatedWords",					"off"],
+		["LengthRatio",						"on"],
+		["ReverseLengthRatio",				"of"],
+		["WordRatio",						"of"],
+		["ReverseWordRatio",				"of"],
+		["WordLength",						"of"],
+		["TagFinder",						"of"],
+		["RepeatedChars",					"of"],
+		["RepeatedWords",					"of"],
 
 		["Lang_Identifier",					"off"],
 

data/sample_token

@@ -0,0 +1,5 @@
+Having regard to the Opinion of the European Parliament ;	VISTO IL PARERE DEL PARLAMENTO EUROPEO ,
+Measurement of fuel consumption	MISURA DEL CONSUMO DI CARBURANTE
+in the case of carbon monoxide , d = 1 7 250 ;	PER L'OSSIDO DI CARBONIO , D = 1,250 ;
+Common compensation procedures	Metodi comuni di compensazione
+The provisions of Articles 10 to 13 shall apply .	Sono applicabili le disposizioni degli articolo da 10 a 13 .

filters/AlignedProportion/AlignedProportion.py

@@ -1,10 +1,15 @@
 # sys.path.append(os.getcwd() + '/..') # Uncomment for standalone running
 from abstract_filter import *
+import os.path
 import math
+import numpy as np
 
 
 class AlignedProportion(AbstractFilter):
 	def __init__(self):
+		self.var_mult = 2
+		# self.var_mult = 100 - self.var_mult
+
 		self.num_of_scans = 1
 		self.src_language = ""
 		self.trg_language = ""
@@ -21,15 +26,42 @@ def __init__(self):
 		self.trg_mean = 0.0
 		self.trg_var = 0.0
 
+		self.src_scores = []
+		self.trg_scores = []
+		self.s_thresh = 0.0
+		self.t_thresh = 0.0
+
 	#
-	def initialize(self, source_language, target_language):
+	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
-		self.src_language = source_language
-		self.trg_language = target_language
+		self.src_language = extra_args['source language']
+		self.trg_language = extra_args['target language']
+		self.normalize = extra_args['normalize scores']
+		self.model_filename = "models/" + extra_args['input filename'] + "__AlignedProportion.stats"
+		if self.normalize:
+			self.model_filename += "_n"
+
+		if os.path.isfile(self.model_filename):
+			self.num_of_scans = 0
+
+			f = open(self.model_filename, 'r')
+			l = f.readline().strip().split("\t")
+			self.src_mean = float(l[1])
+			self.src_var = float(l[2])
+
+			l = f.readline().strip().split("\t")
+			self.trg_mean = float(l[1])
+			self.trg_var = float(l[2])
+
+			f.close()
+			print "Loaded stats from the model file."
 
 		return
 
 	def finalize(self):
+		if self.num_of_scans == 0:
+			return
+
 		if self.n <= 1:
 			self.n = 2.0
 		self.src_mean = self.src_sum / self.n
@@ -44,6 +76,30 @@ def finalize(self):
 		print "source mean & deviation:", self.src_mean, "\t", self.src_var
 		print "target mean & deviation:", self.trg_mean, "\t", self.trg_var
 
+		f = open(self.model_filename, 'w')
+
+		f.write("source\t" + str(self.src_mean) + "\t" + str(self.src_var) + "\n")
+		f.write("target\t" + str(self.trg_mean) + "\t" + str(self.trg_var) + "\n")
+
+		f.close()
+
+		self.s_thresh = np.percentile(self.src_scores, self.var_mult)
+		self.t_thresh = np.percentile(self.trg_scores, self.var_mult)
+
+		f = open("models/quartiles", "a")
+
+		f.write("Aligned Proportion")
+		f.write("\t" + str(np.percentile(self.src_scores, 25)))
+		f.write("\t" + str(np.percentile(self.src_scores, 50)))
+		f.write("\t" + str(np.percentile(self.src_scores, 75)))
+
+		f.write("\t" + str(np.percentile(self.trg_scores, 25)))
+		f.write("\t" + str(np.percentile(self.trg_scores, 50)))
+		f.write("\t" + str(np.percentile(self.trg_scores, 75)))
+		f.write("\n")
+
+		f.close()
+
 	#
 	def process_tu(self, tu, num_of_finished_scans):
 		src_set = set([x[0] for x in tu.alignment])
@@ -53,17 +109,31 @@ def process_tu(self, tu, num_of_finished_scans):
 		trg_size = float(len(tu.trg_tokens))
 
 		if src_size == 0 or trg_size == 0:
-			return
+			return [0.0, 0.0]
 
 		self.n += 1
 		src_ratio = float(len(src_set)) / src_size
+		# if src_ratio > 1:
+		# 	print src_set
+		# 	print tu.src_tokens
 		trg_ratio = float(len(trg_set)) / trg_size
+		# if trg_ratio > 1:
+		# 	print trg_set
+		# 	print tu.trg_tokens
+
+		src_ratio = min(src_ratio, 1.0)
+		trg_ratio = min(trg_ratio, 1.0)
 
 		self.src_sum += src_ratio
 		self.src_sum_sq += src_ratio * src_ratio
 		self.trg_sum += trg_ratio
 		self.trg_sum_sq += trg_ratio * trg_ratio
 
+		self.src_scores.append(src_ratio)
+		self.trg_scores.append(trg_ratio)
+
+		return [src_ratio, trg_ratio]
+
 	#
 	def do_after_a_full_scan(self, num_of_finished_scans):
 		pass
@@ -84,6 +154,7 @@ def decide(self, tu):
 		src_ratio = abs(src_ratio - self.src_mean)
 		trg_ratio = abs(trg_ratio - self.trg_mean)
 
-		if src_ratio > 2 * self.src_var or trg_ratio > 2 * self.trg_var:
+		if src_ratio > self.var_mult * self.src_var or trg_ratio > self.var_mult * self.trg_var:
+		# if src_ratio < self.s_thresh or trg_ratio < self.t_thresh:
 			return 'reject'
 		return 'accept'

filters/AlignedSequenceLength/AlignedSequenceLength.py

@@ -1,10 +1,15 @@
 # sys.path.append(os.getcwd() + '/..') # Uncomment for standalone running
 from abstract_filter import *
+import os.path
 import math
+import numpy as np
 
 
 class AlignedSequenceLength(AbstractFilter):
 	def __init__(self):
+		self.var_mult = 2
+		# self.var_mult = 100 - self.var_mult
+
 		self.num_of_scans = 1
 		self.src_language = ""
 		self.trg_language = ""
@@ -22,15 +27,42 @@ def __init__(self):
 		self.trg_mean = 0.0
 		self.trg_var = 0.0
 
+		self.src_scores = []
+		self.trg_scores = []
+		self.s_thresh = 0.0
+		self.t_thresh = 0.0
+
 	#
-	def initialize(self, source_language, target_language):
+	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
-		self.src_language = source_language
-		self.trg_language = target_language
+		self.src_language = extra_args['source language']
+		self.trg_language = extra_args['target language']
+		self.normalize = extra_args['normalize scores']
+		self.model_filename = "models/" + extra_args['input filename'] + "__AlignedSequenceLength.stats"
+		if self.normalize:
+			self.model_filename += "_n"
+
+		if os.path.isfile(self.model_filename):
+			self.num_of_scans = 0
+
+			f = open(self.model_filename, 'r')
+			l = f.readline().strip().split("\t")
+			self.src_mean = float(l[1])
+			self.src_var = float(l[2])
+
+			l = f.readline().strip().split("\t")
+			self.trg_mean = float(l[1])
+			self.trg_var = float(l[2])
+
+			f.close()
+			print "Loaded stats from the model file."
 
 		return
 
 	def finalize(self):
+		if self.num_of_scans == 0:
+			return
+
 		if self.src_n <= 1:
 			self.src_n = 2.0
 		self.src_mean = self.src_sum / self.src_n
@@ -43,6 +75,30 @@ def finalize(self):
 		self.trg_var = (self.trg_sum_sq - (self.trg_sum * self.trg_sum) / self.trg_n) / (self.trg_n - 1)
 		self.trg_var = math.sqrt(self.trg_var)
 
+		f = open(self.model_filename, 'w')
+
+		f.write("source\t" + str(self.src_mean) + "\t" + str(self.src_var) + "\n")
+		f.write("target\t" + str(self.trg_mean) + "\t" + str(self.trg_var) + "\n")
+
+		f.close()
+
+		self.s_thresh = np.percentile(self.src_scores, self.var_mult)
+		self.t_thresh = np.percentile(self.trg_scores, self.var_mult)
+
+		f = open("models/quartiles", "a")
+
+		f.write("AlignedSequenceLength")
+		f.write("\t" + str(np.percentile(self.src_scores, 25)))
+		f.write("\t" + str(np.percentile(self.src_scores, 50)))
+		f.write("\t" + str(np.percentile(self.src_scores, 75)))
+
+		f.write("\t" + str(np.percentile(self.trg_scores, 25)))
+		f.write("\t" + str(np.percentile(self.trg_scores, 50)))
+		f.write("\t" + str(np.percentile(self.trg_scores, 75)))
+		f.write("\n")
+
+		f.close()
+
 	#
 	def process_tu(self, tu, num_of_finished_scans):
 		src_set = set([x[0] for x in tu.alignment])
@@ -51,37 +107,57 @@ def process_tu(self, tu, num_of_finished_scans):
 		trg_size = float(len(tu.trg_tokens))
 
 		if src_size == 0 or trg_size == 0:
-			return
+			return [0.0, 0.0]
 
 		src_bar = set([i for i in range(int(src_size))])
 		trg_bar = set([i for i in range(int(trg_size))])
 		src_set = src_bar - src_set
 		trg_set = trg_bar - trg_set
 
 		last = -1
+		n = 0.0
+		smean = 0.0
 		for current in src_set:
 			if current - last > 1:
-				self.src_n += 1
-				self.src_sum += (current - last - 1)
-				self.src_sum_sq += (current - last - 1) * (current - last - 1)
+				n += 1
+				smean += (current - last - 1)
 			last = current
 		if src_size - last > 1:
-			self.src_n += 1
-			self.src_sum += (src_size - last - 1)
-			self.src_sum_sq += (src_size - last - 1) * (src_size - last - 1)
+			n += 1
+			smean += (src_size - last - 1)
+
+		smean /= max(n, 1.0)
+		if self.normalize:
+			smean = min(smean, 4.0) / 4.0
+
+		self.src_n += 1
+		self.src_sum += smean
+		self.src_sum_sq += smean * smean
 
 		last = -1
+		n = 0.0
+		tmean = 0.0
 		for current in trg_set:
 			if current - last > 1:
-				self.trg_n += 1
-				self.trg_sum += (current - last - 1)
-				self.trg_sum_sq += (current - last - 1) * (current - last - 1)
+				n += 1
+				tmean += (current - last - 1)
 			last = current
 		if trg_size - last > 1:
-			self.trg_n += 1
-			self.trg_sum += (trg_size - last - 1)
-			self.trg_sum_sq += (trg_size - last - 1) * (trg_size - last - 1)
+			n += 1
+			tmean += (trg_size - last - 1)
+
+		tmean /= max(n, 1.0)
+		if self.normalize:
+			tmean = min(tmean, 4.0) / 4.0
+
+		self.trg_n += 1
+		self.trg_sum += tmean
+		self.trg_sum_sq += tmean * tmean
+
+		self.src_scores.append(smean)
+		self.trg_scores.append(tmean)
 
+		return [smean, tmean]
 
 	#
 	def do_after_a_full_scan(self, num_of_finished_scans):
@@ -112,6 +188,8 @@ def decide(self, tu):
 		if n < 1:
 			n = 1.0
 		src_mean /= n
+		if self.normalize:
+			src_mean = min(src_mean, 4.0) / 4.0
 
 		n = 0.0
 		trg_mean = 0.0
@@ -127,10 +205,13 @@ def decide(self, tu):
 		if n < 1:
 			n = 1.0
 		trg_mean /= n
+		if self.normalize:
+			trg_mean = min(trg_mean, 4.0) / 4.0
 
 		src_mean = abs(src_mean - self.src_mean)
 		trg_mean = abs(trg_mean - self.trg_mean)
 
-		if src_mean > 2 * self.src_var or trg_mean > 2 * self.trg_var:
+		if src_mean > self.var_mult * self.src_var or trg_mean > self.var_mult * self.trg_var:
+		# if src_mean < self.s_thresh or trg_mean < self.t_thresh:
 			return 'reject'
 		return 'accept'