add initial value for model_exist in filters

Masoud Jalili · Masoud Jalili · commit f20ba1af9e16 · 2017-04-16T21:43:43.000+04:30
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/filters/AlignedSequenceLength/AlignedSequenceLength.py b/filters/AlignedSequenceLength/AlignedSequenceLength.py
@@ -32,6 +32,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/BigramAlignedProportion/BigramAlignedProportion.py b/filters/BigramAlignedProportion/BigramAlignedProportion.py
@@ -31,6 +31,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/FirstUnalignedWord/FirstUnalignedWord.py b/filters/FirstUnalignedWord/FirstUnalignedWord.py
@@ -30,6 +30,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/LastUnalignedWord/LastUnalignedWord.py b/filters/LastUnalignedWord/LastUnalignedWord.py
@@ -31,6 +31,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/LengthRatio/LengthRatio.py b/filters/LengthRatio/LengthRatio.py
@@ -22,6 +22,8 @@ def __init__(self):
 		self.scores = []
 		self.thresh = 0.0
 
+		self.model_exist = False
+
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
 		self.src_language = extra_args['source language']
diff --git a/filters/LengthStats/LengthStats.py b/filters/LengthStats/LengthStats.py
@@ -35,14 +35,15 @@ def finalize(self):
 		print 'src word mean:', self.src_wsum / self.n
 		print 'trg word mean:', self.trg_wsum / self.n
 
-
 	def process_tu(self, tu, num_of_finished_scans):
 		self.src_wsum += len(tu.src_tokens)
 		self.trg_wsum += len(tu.trg_tokens)
 		self.src_sum += len(tu.src_phrase)
 		self.trg_sum += len(tu.trg_phrase)
 		self.n += 1
 
+		return None
+
 	def do_after_a_full_scan(self, num_of_finished_scans):
 		return
 
diff --git a/filters/LongestAlignedSequence/LongestAlignedSequence.py b/filters/LongestAlignedSequence/LongestAlignedSequence.py
@@ -31,6 +31,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/LongestUnalignedSequence/LongestUnalignedSequence.py b/filters/LongestUnalignedSequence/LongestUnalignedSequence.py
@@ -30,6 +30,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/NumberOfUnalignedSequences/NumberOfUnalignedSequences.py b/filters/NumberOfUnalignedSequences/NumberOfUnalignedSequences.py
@@ -30,6 +30,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/ReverseLengthRatio/ReverseLengthRatio.py b/filters/ReverseLengthRatio/ReverseLengthRatio.py
@@ -22,7 +22,9 @@ def __init__(self):
 		self.scores = []
 		self.thresh = 0.0
 
-#
+		self.model_exist = False
+
+	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
 		self.src_language = extra_args['source language']
diff --git a/filters/ReverseWordRatio/ReverseWordRatio.py b/filters/ReverseWordRatio/ReverseWordRatio.py
@@ -22,7 +22,9 @@ def __init__(self):
 		self.scores = []
 		self.thresh = 0.0
 
-#
+		self.model_exist = False
+
+	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
 		self.src_language = extra_args['source language']
diff --git a/filters/UnalignedSequenceLength/UnalignedSequenceLength.py b/filters/UnalignedSequenceLength/UnalignedSequenceLength.py
@@ -31,6 +31,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/WE_Average/WE_Average.py b/filters/WE_Average/WE_Average.py
@@ -37,6 +37,8 @@ def __init__(self):
 		self.mean = 0.0
 		self.var = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 3
diff --git a/filters/WE_BestAlignScore/WE_BestAlignScore.py b/filters/WE_BestAlignScore/WE_BestAlignScore.py
@@ -37,7 +37,9 @@ def __init__(self):
 		self.mean = 0.0
 		self.var = 0.0
 
-#
+		self.model_exist = False
+
+	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 3
 		self.src_language = extra_args['source language']
diff --git a/filters/WE_Median/WE_Median.py b/filters/WE_Median/WE_Median.py
@@ -37,6 +37,8 @@ def __init__(self):
 		self.mean = 0.0
 		self.var = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 3
diff --git a/filters/WE_ScoreAlign_BestForRest/WE_ScoreAlign_BestForRest.py b/filters/WE_ScoreAlign_BestForRest/WE_ScoreAlign_BestForRest.py
@@ -38,6 +38,8 @@ def __init__(self):
 		self.mean = 0.0
 		self.var = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 3
diff --git a/filters/WE_ScoreOtherAlignment/WE_ScoreOtherAlignment.py b/filters/WE_ScoreOtherAlignment/WE_ScoreOtherAlignment.py
@@ -37,6 +37,8 @@ def __init__(self):
 		self.mean = 0.0
 		self.var = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 3
diff --git a/filters/WordLength/WordLength.py b/filters/WordLength/WordLength.py
@@ -32,6 +32,8 @@ def __init__(self):
 		self.s_thresh = 0.0
 		self.t_thresh = 0.0
 
+		self.model_exist = False
+
 	#
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
diff --git a/filters/WordRatio/WordRatio.py b/filters/WordRatio/WordRatio.py
@@ -23,12 +23,14 @@ def __init__(self):
 		self.scores = []
 		self.thresh = 0.0
 
+		self.model_exist = False
+
 	def initialize(self, source_language, target_language, extra_args):
 		self.num_of_scans = 1
 		self.src_language = extra_args['source language']
 		self.trg_language = extra_args['target language']
 		self.normalize = extra_args['normalize scores']
-		self.model_filename = "models/UnalignedSequenceLength.stats"
+		self.model_filename = "models/WordRatio.stats"
 		if self.normalize:
 			self.model_filename += "_n"
 
diff --git a/tm_manager.py b/tm_manager.py
@@ -494,7 +494,8 @@ def run(self):
 				for filter_tuple in active_filters:
 					try:
 						score = filter_tuple[1].process_tu(copy(tu), filter_tuple[2])
-						scores.append(score)
+						if score is not None:
+							scores.append(score)
 					except Exception, e:
 						print "The filter", filter_tuple[0], "has problems processing the TU in line:", line_no
 						print "The Exception:"

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,8 @@ def __init__(self):`
`32`	`32`	`self.s_thresh = 0.0`
`33`	`33`	`self.t_thresh = 0.0`
`34`	`34`
	`35`	`+ self.model_exist = False`
	`36`	`+`
`35`	`37`	`#`
`36`	`38`	`def initialize(self, source_language, target_language, extra_args):`
`37`	`39`	`self.num_of_scans = 1`
Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,8 @@ def __init__(self):`
`31`	`31`	`self.s_thresh = 0.0`
`32`	`32`	`self.t_thresh = 0.0`
`33`	`33`
	`34`	`+ self.model_exist = False`
	`35`	`+`
`34`	`36`	`#`
`35`	`37`	`def initialize(self, source_language, target_language, extra_args):`
`36`	`38`	`self.num_of_scans = 1`
Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,8 @@ def __init__(self):`
`30`	`30`	`self.s_thresh = 0.0`
`31`	`31`	`self.t_thresh = 0.0`
`32`	`32`
	`33`	`+ self.model_exist = False`
	`34`	`+`
`33`	`35`	`#`
`34`	`36`	`def initialize(self, source_language, target_language, extra_args):`
`35`	`37`	`self.num_of_scans = 1`
Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,8 @@ def __init__(self):`
`37`	`37`	`self.mean = 0.0`
`38`	`38`	`self.var = 0.0`
`39`	`39`
	`40`	`+ self.model_exist = False`
	`41`	`+`
`40`	`42`	`#`
`41`	`43`	`def initialize(self, source_language, target_language, extra_args):`
`42`	`44`	`self.num_of_scans = 3`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,8 @@ def __init__(self):`
`38`	`38`	`self.mean = 0.0`
`39`	`39`	`self.var = 0.0`
`40`	`40`
	`41`	`+ self.model_exist = False`
	`42`	`+`
`41`	`43`	`#`
`42`	`44`	`def initialize(self, source_language, target_language, extra_args):`
`43`	`45`	`self.num_of_scans = 3`