init commit

Author: SeanLee97

README.md

@@ -1,2 +1,22 @@
 # LS-LLaMA
 LABEL SUPERVISED LLAMA FINETUNING
+
+
+## Usage
+
+Load Pretrained Models
+
+```python
+from transformers import AutoTokenizer
+from modeling_llama import (
+    LlamaForSequenceClassification, LlamaForTokenClassification,
+    UnmaskingLlamaForSequenceClassification, UnmaskingLlamaForTokenClassification,
+)
+
+
+model_id = 'meta-llama/Llama-2-7b'
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = LlamaForSequenceClassification.from_pretrained(model_id)
+```
+
+More usage please refer to `unllama_seq_clf.py`, `unllama_token_clf.py`, `llama_seq_clf.py`, `llama_token_clf.py`.

data/ontonotesv5/dev.jsonl

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from typing import List, Any, Dict
+from datasets import load_dataset
+from transformers import AutoTokenizer
+from transformers.data import  *
+from transformers import TrainingArguments, Trainer
+from peft import get_peft_model, LoraConfig, TaskType
+import evaluate
+import numpy as np
+
+from modeling_llama import LlamaForSequenceClassification
+
+
+if len(sys.argv) != 3:
+    print('usage python %.py dataset model_size')
+    sys.exit()
+
+
+dataset, model_size = sys.argv[1], sys.argv[2]
+epochs = 10
+batch_size = 8
+learning_rate = 5e-5
+lora_r = 12
+max_length = 64
+if model_size.lower() == '7b':
+    model_id = 'NousResearch/Llama-2-7b-hf'
+elif model_size.lower() == '13b':
+    model_id = 'NousResearch/Llama-2-13b-hf'
+
+test_name = 'test'
+text_name = None
+if dataset == 'agnews':
+    id2label = {0: "World", 1: "Sports", 2: "Business", 3: "Sci/Tech"}
+    label2id = {v: k for k, v in id2label.items()}
+    ds = load_dataset("ag_news")
+    text_name = 'text'
+elif dataset == 'twitterfin':
+    id2label = {0: "Bearish", 1: "Bullish", 2: "Neutral"}
+    label2id = {v: k for k, v in id2label.items()}
+    ds = load_dataset("zeroshot/twitter-financial-news-sentiment")
+    test_name = 'validation'
+    text_name = 'text'
+elif dataset == 'sst2':
+    id2label = {0: "negative", 1: "positive"}
+    label2id = {v: k for k, v in id2label.items()}
+    ds = load_dataset("sst2")
+    test_name = 'validation'
+    text_name = 'sentence'
+elif dataset in ['amazon_de', 'amazon_en', 'amazon_es', 'amazon_fr', 'amazon_ja', 'amazon_zh']:
+    max_length = 200
+    batch_size = 4
+    lang = dataset.split('_')[1]
+    id2label = {0: 'furniture', 1: 'baby_product', 2: 'jewelry', 3: 'musical_instruments', 4: 'industrial_supplies', 5: 'pc', 6: 'other', 7: 'pet_products', 8: 'book', 9: 'apparel', 10: 'automotive', 11: 'digital_video_download', 12: 'beauty', 13: 'toy', 14: 'shoes', 15: 'personal_care_appliances', 16: 'camera', 17: 'digital_ebook_purchase', 18: 'watch', 19: 'drugstore', 20: 'grocery', 21: 'kitchen', 22: 'home', 23: 'office_product', 24: 'home_improvement', 25: 'electronics', 26: 'video_games', 27: 'sports', 28: 'luggage', 29: 'lawn_and_garden', 30: 'wireless'}
+    label2id = {v: k for k, v in id2label.items()}
+    ds = load_dataset("amazon_reviews_multi", lang)
+    ds = ds.rename_column('product_category', 'label')
+    text_name = ['review_title', 'review_body']
+    # reimplement DataCollatorWithPaddingAmazon
+    class DataCollatorWithPaddingAmazon(DataCollatorWithPadding):
+        def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
+            # print('>>> features>>>', features)
+            new_features = []
+            for v in features:
+                label = v.pop('label')
+                v['label'] = label2id[label]
+                new_features.append(v)
+            features = new_features
+            batch = self.tokenizer.pad(
+                features,
+                padding=self.padding,
+                max_length=self.max_length,
+                pad_to_multiple_of=self.pad_to_multiple_of,
+                return_tensors=self.return_tensors,
+            )
+            if "label" in batch:
+                batch["labels"] = batch["label"]
+                del batch["label"]
+            if "label_ids" in batch:
+                batch["labels"] = batch["label_ids"]
+                del batch["label_ids"]
+            return batch
+
+    DataCollatorWithPadding = DataCollatorWithPaddingAmazon
+else:
+    raise NotImplementedError
+
+accuracy = evaluate.load("accuracy")
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = LlamaForSequenceClassification.from_pretrained(
+    model_id, num_labels=len(label2id), id2label=id2label, label2id=label2id
+).bfloat16()
+peft_config = LoraConfig(task_type=TaskType.SEQ_CLS, inference_mode=False, r=lora_r, lora_alpha=32, lora_dropout=0.1)
+model = get_peft_model(model, peft_config)
+model.print_trainable_parameters()
+
+
+def compute_metrics(eval_pred):
+    predictions, labels = eval_pred
+    predictions = np.argmax(predictions, axis=1)
+    return accuracy.compute(predictions=predictions, references=labels)
+
+
+def preprocess_function(examples):
+    global text_name
+    if isinstance(text_name, str):
+        d = examples[text_name]
+    else:
+        d = examples[text_name[0]]
+        for n in text_name[1:]:
+            nd = examples[n]
+            assert len(d) == len(nd)
+            for i, t in enumerate(nd):
+                d[i] += '\n' + t
+
+    return tokenizer(d, padding='longest', max_length=max_length, truncation=True)
+
+
+tokenized_ds = ds.map(preprocess_function, batched=True)
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+
+training_args = TrainingArguments(
+    output_dir="clf",
+    learning_rate=learning_rate,
+    per_device_train_batch_size=batch_size,
+    per_device_eval_batch_size=batch_size,
+    num_train_epochs=epochs,
+    weight_decay=0.01,
+    evaluation_strategy="epoch",
+    save_strategy="no",
+    load_best_model_at_end=False,
+    push_to_hub=False,
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_ds["train"],
+    eval_dataset=tokenized_ds[test_name],
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+    compute_metrics=compute_metrics,
+)
+
+trainer.train()

data/ontonotesv5/test.jsonl

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+import json
+import sys
+import numpy as np
+import evaluate
+from datasets import load_dataset, Dataset, DatasetDict
+from transformers import AutoTokenizer
+from transformers import DataCollatorForTokenClassification
+from transformers import TrainingArguments, Trainer
+from peft import get_peft_model, LoraConfig, TaskType
+
+from modeling_llama import LlamaForTokenClassification
+
+
+def load_ontonotesv5():
+    ret = {}
+    for split_name in ['train', 'dev', 'test']:
+        data = []
+        with open(f'./data/NER/ontonotesv5/{split_name}.jsonl', 'r') as reader:
+            for line in reader:
+                data.append(json.loads(line))
+        ret[split_name] = Dataset.from_list(data)
+    return DatasetDict(ret)
+
+
+if len(sys.argv) != 3:
+    print('usage python %.py task model_size')
+    sys.exit()
+
+task, model_size = sys.argv[1], sys.argv[2].lower()
+print(f'handling task {task}')
+
+epochs = 10
+batch_size = 8
+learning_rate = 1e-4
+max_length = 64
+if model_size == '7b':
+    model_id = 'NousResearch/Llama-2-7b-hf'
+    lora_r = 12
+elif model_size == '13b':
+    model_id = 'NousResearch/Llama-2-13b-hf'
+    lora_r = 12
+else:
+    raise NotImplementedError
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+seqeval = evaluate.load("seqeval")
+if task == 'wnut_17':
+    ds = load_dataset("wnut_17")
+    label2id = { "O": 0, "B-corporation": 1, "I-corporation": 2, "B-creative-work": 3, "I-creative-work": 4, "B-group": 5, "I-group": 6, "B-location": 7, "I-location": 8, "B-person": 9, "I-person": 10, "B-product": 11, "I-product": 12, }
+elif task == 'conll2003':
+    ds = load_dataset("conll2003")
+    label2id = {'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}
+elif task == 'ontonotesv5':
+    ds = load_ontonotesv5()
+    label2id = {'O': 0, 'B-NORP': 1, 'B-PERSON': 2, 'B-WORK_OF_ART': 3, 'B-QUANTITY': 4, 'B-EVENT': 5, 'B-DATE': 6, 'B-TIME': 7, 'B-PERCENT': 8, 'B-LANGUAGE': 9, 'B-ORG': 10, 'B-CARDINAL': 11, 'B-LAW': 12, 'B-GPE': 13, 'B-PRODUCT': 14, 'B-LOC': 15, 'B-MONEY': 16, 'B-ORDINAL': 17, 'B-FAC': 18}
+else:
+    raise NotImplementedError
+id2label = {v: k for k, v in label2id.items()}
+label_list = list(label2id.keys()) # ds["train"].features[f"ner_tags"].feature.names
+model = LlamaForTokenClassification.from_pretrained(
+    model_id, num_labels=len(label2id), id2label=id2label, label2id=label2id
+).bfloat16()
+peft_config = LoraConfig(task_type=TaskType.TOKEN_CLS, inference_mode=False, r=lora_r, lora_alpha=32, lora_dropout=0.1)
+model = get_peft_model(model, peft_config)
+model.print_trainable_parameters()
+
+
+def tokenize_and_align_labels(examples):
+    tokenized_inputs = tokenizer(examples["tokens"], is_split_into_words=True, padding='longest', max_length=max_length, truncation=True)
+
+    labels = []
+    for i, label in enumerate(examples[f"ner_tags"]):
+        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
+        previous_word_idx = None
+        label_ids = []
+        for word_idx in word_ids:  # Set the special tokens to -100.
+            if word_idx is None:
+                label_ids.append(-100)
+            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
+                label_ids.append(label[word_idx])
+            else:
+                label_ids.append(-100)
+            previous_word_idx = word_idx
+        labels.append(label_ids)
+
+    tokenized_inputs["labels"] = labels
+    return tokenized_inputs
+
+
+tokenized_ds = ds.map(tokenize_and_align_labels, batched=True)
+data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)
+
+
+def compute_metrics(p):
+    predictions, labels = p
+    predictions = np.argmax(predictions, axis=2)
+
+    true_predictions = [
+        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
+        for prediction, label in zip(predictions, labels)
+    ]
+    true_labels = [
+        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
+        for prediction, label in zip(predictions, labels)
+    ]
+
+    results = seqeval.compute(predictions=true_predictions, references=true_labels)
+    return {
+        "precision": results["overall_precision"],
+        "recall": results["overall_recall"],
+        "f1": results["overall_f1"],
+        "accuracy": results["overall_accuracy"],
+    }
+
+
+training_args = TrainingArguments(
+    output_dir="my_awesome_ds_model",
+    learning_rate=learning_rate,
+    per_device_train_batch_size=batch_size,
+    per_device_eval_batch_size=batch_size,
+    num_train_epochs=epochs,
+    weight_decay=0.01,
+    evaluation_strategy="epoch",
+    save_strategy="epoch",
+    load_best_model_at_end=True,
+    push_to_hub=False,
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_ds["train"],
+    eval_dataset=tokenized_ds["test"],
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+    compute_metrics=compute_metrics,
+)
+
+trainer.train()