1

Author: [email protected]

app/ngram_interpret.py

@@ -115,6 +115,9 @@ def compute_per_sentence_attr_score(target_df, class_pos, ngram, save_path, lang
 
 
 # python ngram_interpret.py --path '../result/interpret/interpret_cn_novel_5billion_cn_roberta_debug_0_text_len_128_debug_N_10000_use_all_zero_bs_token_attr.csv'
+# python ngram_interpret.py --path '../result/interpret/interpret_cn_novel_5billion_cn_roberta_debug_0_text_len_128_debug_N_10000_use_all_zero_bs_token_attr.csv'
+# python ngram_interpret.py --path '../result/interpret/interpret_cn_novel_5billion_cn_roberta_debug_0_text_len_128_debug_N_10000_use_all_zero_bs_token_attr.csv'
+
 # python ngram_interpret.py --path '../result/interpret/interpret_en_grover_en_roberta_debug_0_text_len_256_debug_N_800_use_pad_bs_token_attr.csv'
 # python ngram_interpret.py --path '../result/interpret/interpret_en_writing_prompt_en_roberta_debug_0_text_len_128_debug_N_800_use_pad_bs_token_attr.csv'
 # python ngram_interpret.py --path '../result/interpret/interpret_en_grover_en_roberta_debug_0_text_len_256_debug_N_10000_use_all_zero_bs_token_attr.csv'
@@ -128,7 +131,7 @@ def main():
     language = basename.split('_')[1]
 
     df = pd.read_csv(path)
-    ngrams = [1, 2, 3, 4, 5, 6]
+    ngrams = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
     # ngrams = [1, 2, 3, 4, 5, 6]
 
     # token_freq_dict = collections.Counter(df['token'].values)

app/ngram_interpret.sh

@@ -0,0 +1,11 @@
+# bash ngram_interpret.sh
+
+set -e
+
+python ngram_interpret.py --path ../result/interpret/interpret_cn_novel_5billion_cn_roberta_debug_0_text_len_128_debug_N_10000_use_all_zero_bs_token_attr.csv
+python ngram_interpret.py --path ../result/interpret/interpret_cn_novel_5billion_cn_roberta_debug_0_text_len_128_debug_N_10000_use_pad_bs_token_attr.csv
+python ngram_interpret.py --path ../result/interpret/interpret_en_grover_en_roberta_debug_0_text_len_256_debug_N_10000_use_all_zero_bs_token_attr.csv
+python ngram_interpret.py --path ../result/interpret/interpret_en_grover_en_roberta_debug_0_text_len_256_debug_N_10000_use_pad_bs_token_attr.csv
+python ngram_interpret.py --path ../result/interpret/interpret_en_writing_prompt_en_roberta_debug_0_text_len_128_debug_N_10000_use_all_zero_bs_token_attr.csv
+python ngram_interpret.py --path ../result/interpret/interpret_en_writing_prompt_en_roberta_debug_0_text_len_128_debug_N_10000_use_pad_bs_token_attr.csv
+

app/run_all.sh

@@ -5,22 +5,22 @@
 #repeat=${5:-1}
 
 # apply to test (pretrain)
-bash train_on_cn_novel_origin.sh                      '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_reorder_shuffle.sh             '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_char_deduplicate.sh            '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_reorder_freq_high2low.sh       '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_reorder_freq_low2high.sh       '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_reorder_shuffle+deduplicate.sh '16 256 1024 0' 1 0 0 15
-bash train_on_cn_novel_likelihood_rank.sh             '16 256 1024 0' 1 0 0 15
+bash train_on_cn_novel_origin.sh                      '16 32 64 128 256 512 0' 1 0 0 15
+bash train_on_cn_novel_reorder_shuffle.sh             '16 32 64 128 256 512 0' 1 0 0 15
+bash train_on_cn_novel_char_deduplicate.sh            '16 32 64 128 256 512 0' 1 0 0 15
+#bash train_on_cn_novel_reorder_freq_high2low.sh       '16 32 64 128 256 512 0' 1 0 0 15
+#bash train_on_cn_novel_reorder_freq_low2high.sh       '16 32 64 128 256 512 0' 1 0 0 15
+bash train_on_cn_novel_reorder_shuffle+deduplicate.sh '16 32 64 128 256 512 0' 1 0 0 15
+bash train_on_cn_novel_likelihood_rank.sh             '16 32 64 128 256 512 0' 1 0 0 15
 
 # apply to test (not pretrain)
-bash train_on_cn_novel_origin.sh                      '16 256 1024 0' 1 0 1 15 #  non pre-train
-bash train_on_cn_novel_reorder_shuffle.sh             '16 256 1024 0' 1 0 1 15
-bash train_on_cn_novel_char_deduplicate.sh            '16 256 1024 0' 1 0 1 15
-bash train_on_cn_novel_reorder_freq_high2low.sh       '16 256 1024 0' 1 0 1 15
-bash train_on_cn_novel_reorder_freq_low2high.sh       '16 256 1024 0' 1 0 1 15
-bash train_on_cn_novel_reorder_shuffle+deduplicate.sh '16 256 1024 0' 1 0 1 15
-bash train_on_cn_novel_likelihood_rank.sh             '16 256 1024 0' 1 0 1 15
+bash train_on_cn_novel_origin.sh                      '16 32 64 128 256 512 0' 1 0 1 15 #  non pre-train
+bash train_on_cn_novel_reorder_shuffle.sh             '16 32 64 128 256 512 0' 1 0 1 15
+bash train_on_cn_novel_char_deduplicate.sh            '16 32 64 128 256 512 0' 1 0 1 15
+#bash train_on_cn_novel_reorder_freq_high2low.sh       '16 32 64 128 256 512 0' 1 0 1 15
+#bash train_on_cn_novel_reorder_freq_low2high.sh       '16 32 64 128 256 512 0' 1 0 1 15
+bash train_on_cn_novel_reorder_shuffle+deduplicate.sh '16 32 64 128 256 512 0' 1 0 1 15
+
 
 #char_freq_ranges=${1:-0}
 #is_debug=${2:-0}

app/run_en_all_no_spacy.sh

@@ -10,18 +10,18 @@
 
 
 # Grover
-# bash train_on_en.sh 'en_grover' '400 800 1600 0' 1 0 0 15 1 'None'
-# bash train_on_en.sh 'en_grover' '400 800 1600 0' 1 0 1 15 1 'None' # NON PRETRAIN
-# bash train_on_en.sh 'en_grover' '16 256 1024 0' 1 0 0 15 1 'reorder_shuffle char_deduplicate'
-# bash train_on_en.sh 'en_grover' '16 256 1024 0' 1 0 0 15 1 'reorder_freq_low2high'
-# bash train_on_en.sh 'en_grover' '16 256 1024 0' 1 0 0 15 1 'reorder_freq_high2low'
+# bash train_on_en.sh 'en_grover' '400 800 1600 3200 6400 12800 0' 1 0 0 15 1 'None'
+# bash train_on_en.sh 'en_grover' '400 800 1600 3200 6400 12800 0' 1 0 0 15 1 'char_deduplicate'
+# bash train_on_en.sh 'en_grover' '400 800 1600 3200 6400 12800 0' 1 0 0 15 1 'reorder_shuffle'
+# bash train_on_en.sh 'en_grover' '400 800 1600 3200 6400 12800 0' 1 0 0 15 1 'reorder_shuffle char_deduplicate'
+# bash train_on_en.sh 'en_grover' '400 800 1600 3200 6400 12800 0' 1 0 0 15 1 'likelihood_rank'
+
 
 # en_writing_prompt
-# bash train_on_en.sh 'en_writing_prompt' '20 40 80 160 0' 1 0 0 15 1 'None'
-# bash train_on_en.sh 'en_writing_prompt' '20 40 80 160 0' 1 0 1 15 1 'None' # NON PRETRAIN
-# bash train_on_en.sh 'en_writing_prompt' '16 256 1024 0' 1 0 0 15 1 'reorder_shuffle char_deduplicate'
-# bash train_on_en.sh 'en_writing_prompt' '16 256 1024 0' 1 0 0 15 1 'reorder_freq_low2high'  # a8c02b65
-# bash train_on_en.sh 'en_writing_prompt' '16 256 1024 0' 1 0 0 15 1 'reorder_freq_high2low'
+# bash train_on_en.sh 'en_writing_prompt' '10 20 40 80 160 320 0' 1 0 0 15 1 'None'
+# bash train_on_en.sh 'en_writing_prompt' '10 20 40 80 160 320 0' 1 0 0 15 1 'char_deduplicate'
+# bash train_on_en.sh 'en_writing_prompt' '10 20 40 80 160 320 0' 1 0 0 15 1 'reorder_shuffle'
+# bash train_on_en.sh 'en_writing_prompt' '10 20 40 80 160 320 0' 1 0 0 15 1 'reorder_shuffle char_deduplicate'
 
 dataset_name=${1:-0}
 is_debug=${2:-0}

app/run_story_interpret_all.sh

@@ -0,0 +1,20 @@
+# bash run_story_interpret_all.sh
+
+set -e
+#bash save_model_for_interpret.sh en_grover en_roberta 0
+#bash save_model_for_interpret.sh en_writing_prompt en_roberta 0
+# bash run_story_interpret.sh 500 1 128 cn_novel_5billion interpret_cn_novel_5billion_cn_roberta_debug_0 bert 100 1
+# bash run_story_interpret.sh 800 1 256 en_grover interpret_en_grover_en_roberta_debug_0 roberta 40 1
+# bash run_story_interpret.sh 800 1 128 en_writing_prompt interpret_en_writing_prompt_en_roberta_debug_0 roberta 100 1
+
+#bash run_story_interpret.sh 10000 1 128 cn_novel_5billion interpret_cn_novel_5billion_cn_roberta_debug_0 bert 100 0
+#bash run_story_interpret.sh 10000 1 256 en_grover interpret_en_grover_en_roberta_debug_0 roberta 40 0
+#bash run_story_interpret.sh 10000 1 128 en_writing_prompt interpret_en_writing_prompt_en_roberta_debug_0 roberta 100 0
+
+#bash run_story_interpret.sh 10000 1 256 en_grover interpret_en_grover_en_roberta_debug_0 roberta 500 0
+#bash run_story_interpret.sh 10000 1 128 cn_novel_5billion interpret_cn_novel_5billion_cn_roberta_debug_0 bert 500 0
+#bash run_story_interpret.sh 10000 1 128 en_writing_prompt interpret_en_writing_prompt_en_roberta_debug_0 roberta 500 0
+
+bash run_story_interpret.sh 10000 1 256 en_grover interpret_en_grover_en_roberta_debug_0 roberta 500 1
+bash run_story_interpret.sh 10000 1 128 cn_novel_5billion interpret_cn_novel_5billion_cn_roberta_debug_0 bert 500 1
+bash run_story_interpret.sh 10000 1 128 en_writing_prompt interpret_en_writing_prompt_en_roberta_debug_0 roberta 500 1

core/interpreter.py

@@ -110,12 +110,14 @@ def interpret_encoded_inputs(self,
                                                                      target=all_1_labels,
                                                                      baselines=all_pad_embedding,
                                                                      n_steps=self.n_steps,
-                                                                     return_convergence_delta=True)
+                                                                     return_convergence_delta=True,
+                                                                     internal_batch_size=32)
         else:
             label1_attributions_ig, label1_delta = self.ig.attribute(inputs=input_embedding,
                                                                      target=all_1_labels,
                                                                      n_steps=self.n_steps,
-                                                                     return_convergence_delta=True)
+                                                                     return_convergence_delta=True,
+                                                                     internal_batch_size=32)
         label1_attributions_ig = label1_attributions_ig.detach().cpu()
         label1_delta = label1_delta.detach().cpu()
         # print("label1_delta: ", label1_delta)