nrimsky · flores-o · Aug 17, 2024 · Sep 5, 2024 · Sep 5, 2024 · Sep 5, 2024
diff --git a/README.md b/README.md
@@ -36,12 +36,13 @@ See `models/implementations/gpt2small.py` for an example of how to use this clas
 
 ### `models`
 
-Model implementations. Currently only [gpt2](https://huggingface.co/openai-community/gpt2) is implemented as a basic example that will load on your laptop. You can add more models by following the same pattern.
+Model implementations. Currently gpt2 and llama3 are implemented. You can add more models by following the same pattern.
 
 ### `notebooks`
 
 Jupyter notebooks demonstrating basic use-cases.
 
 ## To do
 
-- Integrate with SAE feature steering
+- Integrate with SAE feature steering
+- Add Gemma 2 implementation https://huggingface.co/google/gemma-2-9b
diff --git a/lmexp/activations/refusal/activations_neg_0_gpt2.pt b/lmexp/activations/refusal/activations_neg_0_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_10_gpt2.pt b/lmexp/activations/refusal/activations_neg_10_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_11_gpt2.pt b/lmexp/activations/refusal/activations_neg_11_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_1_gpt2.pt b/lmexp/activations/refusal/activations_neg_1_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_2_gpt2.pt b/lmexp/activations/refusal/activations_neg_2_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_3_gpt2.pt b/lmexp/activations/refusal/activations_neg_3_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_4_gpt2.pt b/lmexp/activations/refusal/activations_neg_4_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_5_gpt2.pt b/lmexp/activations/refusal/activations_neg_5_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_6_gpt2.pt b/lmexp/activations/refusal/activations_neg_6_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_7_gpt2.pt b/lmexp/activations/refusal/activations_neg_7_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_8_gpt2.pt b/lmexp/activations/refusal/activations_neg_8_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_neg_9_gpt2.pt b/lmexp/activations/refusal/activations_neg_9_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_0_gpt2.pt b/lmexp/activations/refusal/activations_pos_0_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_10_gpt2.pt b/lmexp/activations/refusal/activations_pos_10_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_11_gpt2.pt b/lmexp/activations/refusal/activations_pos_11_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_1_gpt2.pt b/lmexp/activations/refusal/activations_pos_1_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_2_gpt2.pt b/lmexp/activations/refusal/activations_pos_2_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_3_gpt2.pt b/lmexp/activations/refusal/activations_pos_3_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_4_gpt2.pt b/lmexp/activations/refusal/activations_pos_4_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_5_gpt2.pt b/lmexp/activations/refusal/activations_pos_5_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_6_gpt2.pt b/lmexp/activations/refusal/activations_pos_6_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_7_gpt2.pt b/lmexp/activations/refusal/activations_pos_7_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_8_gpt2.pt b/lmexp/activations/refusal/activations_pos_8_gpt2.pt
diff --git a/lmexp/activations/refusal/activations_pos_9_gpt2.pt b/lmexp/activations/refusal/activations_pos_9_gpt2.pt
diff --git a/...AB_DATA_PER_LAYER_model_name=openai-community-gpt2_behavior=refusal_type=ab.png b/...AB_DATA_PER_LAYER_model_name=openai-community-gpt2_behavior=refusal_type=ab.png
diff --git a/.../analysis/AB_DATA_PER_LAYER_model_name=openai-community-gpt2_behavior=refusal_type=ab.txt b/.../analysis/AB_DATA_PER_LAYER_model_name=openai-community-gpt2_behavior=refusal_type=ab.txt
@@ -0,0 +1,13 @@
+Layer	Mult_1	Mult_-1
+0	28.92	28.93
+1	29.01	28.84
+2	29.25	28.60
+3	29.36	28.50
+4	29.47	28.39
+5	29.10	28.75
+6	28.80	29.05
+7	28.78	29.08
+8	29.05	28.82
+9	28.71	29.17
+10	28.57	29.32
+11	28.76	29.09
diff --git a/...ysis/LAYER_SWEEPS_model_name=openai-community-gpt2_behavior=refusal_type=ab.png b/...ysis/LAYER_SWEEPS_model_name=openai-community-gpt2_behavior=refusal_type=ab.png
diff --git a/.../model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.png b/.../model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.png
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.tex b/...p/analysis/model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \worse{0.56} & \same{0.56} & \better{0.56} \\ 
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.txt b/...p/analysis/model_name=openai-community-gpt2_layer=0_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5649	0.5650	0.5650
diff --git a/...model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.png b/...model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.png
diff --git a/.../analysis/model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.tex b/.../analysis/model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \worse{0.56} & \same{0.56} & \better{0.57} \\ 
diff --git a/.../analysis/model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.txt b/.../analysis/model_name=openai-community-gpt2_layer=10_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5638	0.5650	0.5660
diff --git a/...model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.png b/...model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.png
diff --git a/.../analysis/model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.tex b/.../analysis/model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \worse{0.56} & \same{0.56} & \better{0.57} \\ 
diff --git a/.../analysis/model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.txt b/.../analysis/model_name=openai-community-gpt2_layer=11_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5642	0.5650	0.5657
diff --git a/.../model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.png b/.../model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.png
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.tex b/...p/analysis/model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \better{0.57} & \same{0.56} & \worse{0.56} \\ 
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.txt b/...p/analysis/model_name=openai-community-gpt2_layer=1_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5654	0.5650	0.5645
diff --git a/.../model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.png b/.../model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.png
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.tex b/...p/analysis/model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \better{0.57} & \same{0.56} & \worse{0.56} \\ 
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.txt b/...p/analysis/model_name=openai-community-gpt2_layer=2_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5668	0.5650	0.5631
diff --git a/.../model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.png b/.../model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.png
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.tex b/...p/analysis/model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \better{0.57} & \same{0.56} & \worse{0.56} \\ 
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.txt b/...p/analysis/model_name=openai-community-gpt2_layer=3_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5675	0.5650	0.5624
diff --git a/.../model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.png b/.../model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.png
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.tex b/...p/analysis/model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.tex
@@ -0,0 +1 @@
+Misconceptions & \better{0.57} & \same{0.56} & \worse{0.56} \\ 
diff --git a/...p/analysis/model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.txt b/...p/analysis/model_name=openai-community-gpt2_layer=4_behavior=refusal_type=truthful_qa.txt
@@ -0,0 +1,2 @@
+Category	Mult_-1.0	Mult_0.0	Mult_1.0
+Misconceptions	0.5680	0.5650	0.5619
diff --git a/.../analysis/model_name=openai-community-gpt2_layer=5_behavior=refusal_type=ab.png b/.../analysis/model_name=openai-community-gpt2_layer=5_behavior=refusal_type=ab.png
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Misconceptions & \worse{0.56} & \same{0.56} & \better{0.56} \\
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5649 0.5650 0.5650
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5638 0.5650 0.5660
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5642 0.5650 0.5657
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Misconceptions & \better{0.57} & \same{0.56} & \worse{0.56} \\
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5654 0.5650 0.5645
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5668 0.5650 0.5631
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5675 0.5650 0.5624
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Category Mult_-1.0 Mult_0.0 Mult_1.0
		Misconceptions 0.5680 0.5650 0.5619