Revision to Process all script (#10)

* fix python_tokenizer tok_name error

* remove import which is not required in python_tokenizer.py

* remove empty value tokens

* modified process_all.sh to take more argument

bin/process_all.sh

@@ -3,9 +3,41 @@
 semester=$1
 course=$2
 gradeable=$3
-language=$4
-window=$5
 
+prev_argument=""
+prior_term_gradeables=()
+ignore_submissions=()
+for argument in "$@"
+do
+	if [[ $argument == --* ]]
+	then
+		prev_argument=$argument
+	else
+	    case $prev_argument in
+		"--language")
+			language=$argument
+		  	;;
+		"--window")
+		  	window=$argument
+		  	;;
+		"--threshold")
+		  	threshold=$argument
+		  	;;
+		"--regrex")
+		  	regrex=$argument
+		  	;;
+		"--provided_code_path")
+		  	provided_code_path=$argument
+		  	;;
+		"--prior_term_gradeables")
+			prior_term_gradeables+=("$argument")
+		  	;;
+		"--ignore_submissions")
+			ignore_submissions+=("$argument")
+		  	;;
+		esac
+	fi		
+done
 
 /usr/local/submitty/Lichen/bin/concatenate_all.py  $semester $course $gradeable 
 /usr/local/submitty/Lichen/bin/tokenize_all.py     $semester $course $gradeable  --${language}