Pushing deberta-v3-large-emotion to hub

README.md

@@ -0,0 +1,80 @@
+---
+license: mit
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: deberta-v3-large-emotion-lr7e-6-gas1-ls0.1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deberta-v3-large-emotion-lr7e-6-gas1-ls0.1
+
+This model is a fine-tuned version of [microsoft/deberta-v3-large](https://huggingface.co/microsoft/deberta-v3-large) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.8311
+- Accuracy: 0.8235
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 7e-06
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 50
+- num_epochs: 10.0
+- label_smoothing_factor: 0.1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| 1.2787        | 0.49  | 100  | 1.1127          | 0.4866   |
+| 1.089         | 0.98  | 200  | 0.9668          | 0.7139   |
+| 0.9134        | 1.47  | 300  | 0.8720          | 0.7834   |
+| 0.8618        | 1.96  | 400  | 0.7726          | 0.7941   |
+| 0.686         | 2.45  | 500  | 0.7337          | 0.8209   |
+| 0.6333        | 2.94  | 600  | 0.7350          | 0.8235   |
+| 0.5765        | 3.43  | 700  | 0.7561          | 0.8235   |
+| 0.5502        | 3.92  | 800  | 0.7273          | 0.8476   |
+| 0.5049        | 4.41  | 900  | 0.8137          | 0.8102   |
+| 0.4695        | 4.9   | 1000 | 0.7581          | 0.8289   |
+| 0.4657        | 5.39  | 1100 | 0.8404          | 0.8048   |
+| 0.4549        | 5.88  | 1200 | 0.7800          | 0.8369   |
+| 0.4305        | 6.37  | 1300 | 0.8575          | 0.8235   |
+| 0.4209        | 6.86  | 1400 | 0.8572          | 0.8102   |
+| 0.3983        | 7.35  | 1500 | 0.8392          | 0.8316   |
+| 0.4139        | 7.84  | 1600 | 0.8152          | 0.8209   |
+| 0.393         | 8.33  | 1700 | 0.8261          | 0.8289   |
+| 0.3979        | 8.82  | 1800 | 0.8328          | 0.8235   |
+| 0.3928        | 9.31  | 1900 | 0.8364          | 0.8209   |
+| 0.3848        | 9.8   | 2000 | 0.8322          | 0.8235   |
+
+
+### Framework versions
+
+- Transformers 4.20.0.dev0
+- Pytorch 1.9.0
+- Datasets 2.2.2
+- Tokenizers 0.11.6

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.8235294222831726,
+    "eval_loss": 0.8311316967010498,
+    "eval_runtime": 2.1275,
+    "eval_samples": 374,
+    "eval_samples_per_second": 175.794,
+    "eval_steps_per_second": 11.281,
+    "train_loss": 0.5818920486113605,
+    "train_runtime": 787.9964,
+    "train_samples": 3257,
+    "train_samples_per_second": 41.333,
+    "train_steps_per_second": 2.589
+}

config.json

@@ -0,0 +1,47 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-large",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": 0,
+    "1": 1,
+    "2": 2,
+    "3": 3
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "0": 0,
+    "1": 1,
+    "2": 2,
+    "3": 3
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.0.dev0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "eval_accuracy": 0.8475936055183411,
+    "eval_loss": 0.5155109763145447,
+    "eval_runtime": 3.3452,
+    "eval_samples": 374,
+    "eval_samples_per_second": 111.803,
+    "eval_steps_per_second": 7.174
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e182efbb57bc6e4ecda054f4a41e886d02bcafcf1de64adf784d0e47c61f27ab
+size 1740401579

run_test.sh

@@ -0,0 +1 @@
+jbsub -queue x86_1h -cores 4+1 -mem 30g -require a100 -o outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/test.log /dccstor/tslm/envs/anaconda3/envs/tslm-gen/bin/python train_clf.py --model_name_or_path outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/best_checkpoint --train_file data/tweet_eval/emotion/train.csv --validation_file data/tweet_eval/emotion/validation.csv --test_file data/tweet_eval/emotion/test.csv --do_eval --do_predict --report_to none --per_device_eval_batch_size 16 --max_seq_length 256 --output_dir outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/best_checkpoint

run_train.sh

@@ -0,0 +1 @@
+jbsub -queue x86_6h -cores 4+1 -mem 30g -require a100 -o outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/train.log /dccstor/tslm/envs/anaconda3/envs/tslm-gen/bin/python train_clf.py --model_name_or_path microsoft/deberta-v3-large --train_file data/tweet_eval/emotion/train.csv --validation_file data/tweet_eval/emotion/validation.csv --do_train --do_eval --per_device_train_batch_size 16 --per_device_eval_batch_size 16 --max_seq_length 256 --learning_rate 7e-6 --output_dir outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1 --evaluation_strategy steps --save_strategy no --warmup_steps 50 --num_train_epochs 10 --overwrite_output_dir --logging_steps 100 --gradient_accumulation_steps 1 --label_smoothing_factor 0.1 --report_to clearml --metric_for_best_model accuracy --logging_dir outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/tb \; rm -rf outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/tb \; rm -rf outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/checkpoint-* \; . outputs/train/tweet_eval2/emotion/deberta-v3-large-emotion-lr7e-6-gas1-ls0.1/run_test.sh

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

test_results.json

@@ -0,0 +1,8 @@
+{
+    "eval_accuracy": 0.8627727031707764,
+    "eval_loss": 0.46840450167655945,
+    "eval_runtime": 8.8065,
+    "eval_samples_per_second": 161.359,
+    "eval_steps_per_second": 10.106,
+    "test_samples": 1421
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "name_or_path": "microsoft/deberta-v3-large",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

trainer_state.json

@@ -0,0 +1,325 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.0,
+  "global_step": 2040,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.49,
+      "learning_rate": 6.824120603015075e-06,
+      "loss": 1.2787,
+      "step": 100
+    },
+    {
+      "epoch": 0.49,
+      "eval_accuracy": 0.48663100600242615,
+      "eval_loss": 1.1127219200134277,
+      "eval_runtime": 2.143,
+      "eval_samples_per_second": 174.524,
+      "eval_steps_per_second": 11.199,
+      "step": 100
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.472361809045226e-06,
+      "loss": 1.089,
+      "step": 200
+    },
+    {
+      "epoch": 0.98,
+      "eval_accuracy": 0.7139037251472473,
+      "eval_loss": 0.9668397903442383,
+      "eval_runtime": 2.1362,
+      "eval_samples_per_second": 175.077,
+      "eval_steps_per_second": 11.235,
+      "step": 200
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 6.1206030150753765e-06,
+      "loss": 0.9134,
+      "step": 300
+    },
+    {
+      "epoch": 1.47,
+      "eval_accuracy": 0.7834224700927734,
+      "eval_loss": 0.8720457553863525,
+      "eval_runtime": 2.1335,
+      "eval_samples_per_second": 175.301,
+      "eval_steps_per_second": 11.249,
+      "step": 300
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 5.7688442211055275e-06,
+      "loss": 0.8618,
+      "step": 400
+    },
+    {
+      "epoch": 1.96,
+      "eval_accuracy": 0.7941176295280457,
+      "eval_loss": 0.772637665271759,
+      "eval_runtime": 2.1363,
+      "eval_samples_per_second": 175.072,
+      "eval_steps_per_second": 11.235,
+      "step": 400
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 5.4170854271356785e-06,
+      "loss": 0.686,
+      "step": 500
+    },
+    {
+      "epoch": 2.45,
+      "eval_accuracy": 0.8208556175231934,
+      "eval_loss": 0.7337380647659302,
+      "eval_runtime": 2.1375,
+      "eval_samples_per_second": 174.975,
+      "eval_steps_per_second": 11.228,
+      "step": 500
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 5.0653266331658295e-06,
+      "loss": 0.6333,
+      "step": 600
+    },
+    {
+      "epoch": 2.94,
+      "eval_accuracy": 0.8235294222831726,
+      "eval_loss": 0.7350101470947266,
+      "eval_runtime": 2.1343,
+      "eval_samples_per_second": 175.234,
+      "eval_steps_per_second": 11.245,
+      "step": 600
+    },
+    {
+      "epoch": 3.43,
+      "learning_rate": 4.71356783919598e-06,
+      "loss": 0.5765,
+      "step": 700
+    },
+    {
+      "epoch": 3.43,
+      "eval_accuracy": 0.8235294222831726,
+      "eval_loss": 0.7560638785362244,
+      "eval_runtime": 2.1248,
+      "eval_samples_per_second": 176.019,
+      "eval_steps_per_second": 11.295,
+      "step": 700
+    },
+    {
+      "epoch": 3.92,
+      "learning_rate": 4.361809045226131e-06,
+      "loss": 0.5502,
+      "step": 800
+    },
+    {
+      "epoch": 3.92,
+      "eval_accuracy": 0.8475936055183411,
+      "eval_loss": 0.727317750453949,
+      "eval_runtime": 2.1509,
+      "eval_samples_per_second": 173.882,
+      "eval_steps_per_second": 11.158,
+      "step": 800
+    },
+    {
+      "epoch": 4.41,
+      "learning_rate": 4.010050251256282e-06,
+      "loss": 0.5049,
+      "step": 900
+    },
+    {
+      "epoch": 4.41,
+      "eval_accuracy": 0.8101603984832764,
+      "eval_loss": 0.8136795163154602,
+      "eval_runtime": 2.1343,
+      "eval_samples_per_second": 175.233,
+      "eval_steps_per_second": 11.245,
+      "step": 900
+    },
+    {
+      "epoch": 4.9,
+      "learning_rate": 3.6582914572864323e-06,
+      "loss": 0.4695,
+      "step": 1000
+    },
+    {
+      "epoch": 4.9,
+      "eval_accuracy": 0.8288770318031311,
+      "eval_loss": 0.7581244707107544,
+      "eval_runtime": 2.1366,
+      "eval_samples_per_second": 175.041,
+      "eval_steps_per_second": 11.233,
+      "step": 1000
+    },
+    {
+      "epoch": 5.39,
+      "learning_rate": 3.306532663316583e-06,
+      "loss": 0.4657,
+      "step": 1100
+    },
+    {
+      "epoch": 5.39,
+      "eval_accuracy": 0.8048128485679626,
+      "eval_loss": 0.8404039144515991,
+      "eval_runtime": 2.1382,
+      "eval_samples_per_second": 174.912,
+      "eval_steps_per_second": 11.224,
+      "step": 1100
+    },
+    {
+      "epoch": 5.88,
+      "learning_rate": 2.9547738693467334e-06,
+      "loss": 0.4549,
+      "step": 1200
+    },
+    {
+      "epoch": 5.88,
+      "eval_accuracy": 0.8368983864784241,
+      "eval_loss": 0.7800447940826416,
+      "eval_runtime": 2.143,
+      "eval_samples_per_second": 174.523,
+      "eval_steps_per_second": 11.199,
+      "step": 1200
+    },
+    {
+      "epoch": 6.37,
+      "learning_rate": 2.6030150753768844e-06,
+      "loss": 0.4305,
+      "step": 1300
+    },
+    {
+      "epoch": 6.37,
+      "eval_accuracy": 0.8235294222831726,
+      "eval_loss": 0.8575029969215393,
+      "eval_runtime": 2.1375,
+      "eval_samples_per_second": 174.967,
+      "eval_steps_per_second": 11.228,
+      "step": 1300
+    },
+    {
+      "epoch": 6.86,
+      "learning_rate": 2.251256281407035e-06,
+      "loss": 0.4209,
+      "step": 1400
+    },
+    {
+      "epoch": 6.86,
+      "eval_accuracy": 0.8101603984832764,
+      "eval_loss": 0.8572390675544739,
+      "eval_runtime": 2.1484,
+      "eval_samples_per_second": 174.083,
+      "eval_steps_per_second": 11.171,
+      "step": 1400
+    },
+    {
+      "epoch": 7.35,
+      "learning_rate": 1.899497487437186e-06,
+      "loss": 0.3983,
+      "step": 1500
+    },
+    {
+      "epoch": 7.35,
+      "eval_accuracy": 0.8315507769584656,
+      "eval_loss": 0.8391810655593872,
+      "eval_runtime": 2.1399,
+      "eval_samples_per_second": 174.771,
+      "eval_steps_per_second": 11.215,
+      "step": 1500
+    },
+    {
+      "epoch": 7.84,
+      "learning_rate": 1.5477386934673368e-06,
+      "loss": 0.4139,
+      "step": 1600
+    },
+    {
+      "epoch": 7.84,
+      "eval_accuracy": 0.8208556175231934,
+      "eval_loss": 0.8151516318321228,
+      "eval_runtime": 2.1644,
+      "eval_samples_per_second": 172.797,
+      "eval_steps_per_second": 11.089,
+      "step": 1600
+    },
+    {
+      "epoch": 8.33,
+      "learning_rate": 1.1959798994974873e-06,
+      "loss": 0.393,
+      "step": 1700
+    },
+    {
+      "epoch": 8.33,
+      "eval_accuracy": 0.8288770318031311,
+      "eval_loss": 0.8261328935623169,
+      "eval_runtime": 2.171,
+      "eval_samples_per_second": 172.27,
+      "eval_steps_per_second": 11.055,
+      "step": 1700
+    },
+    {
+      "epoch": 8.82,
+      "learning_rate": 8.442211055276381e-07,
+      "loss": 0.3979,
+      "step": 1800
+    },
+    {
+      "epoch": 8.82,
+      "eval_accuracy": 0.8235294222831726,
+      "eval_loss": 0.8327566385269165,
+      "eval_runtime": 2.1392,
+      "eval_samples_per_second": 174.828,
+      "eval_steps_per_second": 11.219,
+      "step": 1800
+    },
+    {
+      "epoch": 9.31,
+      "learning_rate": 4.924623115577889e-07,
+      "loss": 0.3928,
+      "step": 1900
+    },
+    {
+      "epoch": 9.31,
+      "eval_accuracy": 0.8208556175231934,
+      "eval_loss": 0.8364331126213074,
+      "eval_runtime": 2.138,
+      "eval_samples_per_second": 174.931,
+      "eval_steps_per_second": 11.226,
+      "step": 1900
+    },
+    {
+      "epoch": 9.8,
+      "learning_rate": 1.4070351758793969e-07,
+      "loss": 0.3848,
+      "step": 2000
+    },
+    {
+      "epoch": 9.8,
+      "eval_accuracy": 0.8235294222831726,
+      "eval_loss": 0.8322352170944214,
+      "eval_runtime": 2.142,
+      "eval_samples_per_second": 174.602,
+      "eval_steps_per_second": 11.204,
+      "step": 2000
+    },
+    {
+      "epoch": 10.0,
+      "step": 2040,
+      "total_flos": 1.517670726457344e+16,
+      "train_loss": 0.5818920486113605,
+      "train_runtime": 787.9964,
+      "train_samples_per_second": 41.333,
+      "train_steps_per_second": 2.589
+    }
+  ],
+  "max_steps": 2040,
+  "num_train_epochs": 10,
+  "total_flos": 1.517670726457344e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4d0aab9d5096465be0dc70b5c8ad406c93d795069341635b649369d4c01830
+size 3311