Print out some debug information

zhongkaifu · Sep 11, 2023 · a61fd87 · a61fd87
1 parent f1d5a6d
commit a61fd87
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 9 deletions.
diff --git a/Seq2SeqSharp/Corpus/MonoCorpus.cs b/Seq2SeqSharp/Corpus/MonoCorpus.cs
@@ -359,7 +359,11 @@ public IEnumerator<T> GetEnumerator()
                                     Logger.WriteLine($"Processing batch '{batchIdx}/{m_batchNumInTotal}'."); // The '{i}th' record in this batch is: Target = '{tgtLine}'");
                                     currentBatchPercent++;
                                 }
-                            }                            
+                            }
+                            else if (batchIdx % 10000 == 0)
+                            {
+                                Logger.WriteLine($"Processing batch '{batchIdx}'");
+                            }
 
                             SntPair sntPair = new SntPair(tgtLine, tgtLine);
                             currentTokenCountsInBatch += sntPair.GetTgtTokenCount();

diff --git a/Seq2SeqSharp/Models/Model.cs b/Seq2SeqSharp/Models/Model.cs
@@ -211,8 +211,8 @@ public void AddWeights(string name, float[] weights)
                 }
 
                 (double min, double max, double distortion) = vq.BuildCodebook(vqSize);
-                if (Math.Abs(max - min) <= 2.0)
-                {
+             //   if (Math.Abs(max - min) <= 2.0)
+             //   {
                     Name2CodeBook.Add(name, vq.CodeBook);
 
                     byte[] bweights = new byte[weights.Length / 2];
@@ -225,12 +225,12 @@ public void AddWeights(string name, float[] weights)
                     }
 
                     Name2WeightsVQ.Add(name, bweights);
-                }
-                else
-                {
-                    Logger.WriteLine($"Distortion({distortion}) is too large, so we keep the original values.");
-                    Name2Weights.Add(name, weights);
-                }
+             //   }
+             //   else
+             //   {
+             //       Logger.WriteLine($"Distortion({distortion}) is too large, so we keep the original values.");
+             //       Name2Weights.Add(name, weights);
+             //   }
             }
             else
             {