End of training

README.md

@@ -0,0 +1,76 @@
+---
+license: apache-2.0
+base_model: retrieva-jp/bert-1.3b
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+- precision
+- recall
+- f1
+model-index:
+- name: out
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/rspeech3399/huggingface/runs/zfb205ui)
+# out
+
+This model is a fine-tuned version of [retrieva-jp/bert-1.3b](https://huggingface.co/retrieva-jp/bert-1.3b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4714
+- Accuracy: 0.737
+- Precision: 0.7423
+- Recall: 0.726
+- F1: 0.7341
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- training_steps: 1000
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Accuracy | Precision | Recall | F1     |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:---------:|:------:|:------:|
+| 0.6438        | 0.0080 | 100  | 0.6116          | 0.663    | 0.8721    | 0.382  | 0.5313 |
+| 0.5113        | 0.0160 | 200  | 0.5442          | 0.699    | 0.6736    | 0.772  | 0.7195 |
+| 0.4512        | 0.0240 | 300  | 0.5119          | 0.717    | 0.8359    | 0.54   | 0.6561 |
+| 0.3916        | 0.0321 | 400  | 0.4936          | 0.702    | 0.7295    | 0.642  | 0.6830 |
+| 0.3806        | 0.0401 | 500  | 0.4763          | 0.715    | 0.7708    | 0.612  | 0.6823 |
+| 0.3581        | 0.0481 | 600  | 0.4597          | 0.754    | 0.75      | 0.762  | 0.7560 |
+| 0.3308        | 0.0561 | 700  | 0.4690          | 0.742    | 0.7738    | 0.684  | 0.7261 |
+| 0.3458        | 0.0641 | 800  | 0.4703          | 0.737    | 0.7423    | 0.726  | 0.7341 |
+| 0.3475        | 0.0721 | 900  | 0.4728          | 0.737    | 0.7495    | 0.712  | 0.7303 |
+| 0.3435        | 0.0801 | 1000 | 0.4714          | 0.737    | 0.7423    | 0.726  | 0.7341 |
+
+
+### Framework versions
+
+- Transformers 4.43.1
+- Pytorch 2.3.1+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

config.json

@@ -0,0 +1,45 @@
+{
+  "_name_or_path": "retrieva-jp/bert-1.3b",
+  "architectures": [
+    "RetrievaBertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "retrieva-jp/bert-1.3b--configuration_retrieva_bert.RetrievaBertConfig",
+    "AutoModel": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertModel",
+    "AutoModelForCausalLM": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForCausalLM",
+    "AutoModelForMaskedLM": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForMaskedLM",
+    "AutoModelForMultipleChoice": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForMultipleChoice",
+    "AutoModelForNextSentencePrediction": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForNextSentencePrediction",
+    "AutoModelForPreTraining": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForPreTraining",
+    "AutoModelForQuestionAnswering": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForQuestionAnswering",
+    "AutoModelForSequenceClassification": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForSequenceClassification",
+    "AutoModelForTokenClassification": "retrieva-jp/bert-1.3b--modeling_retrieva_bert.RetrievaBertForTokenClassification"
+  },
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-12,
+  "lm_head_hidden_act": "gelu",
+  "max_position_embeddings": 2048,
+  "max_sequence_length": 2048,
+  "mlp_bias": true,
+  "model_type": "retrieva-bert",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 1,
+  "pad_token_id": 4,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "rope_theta": 10000.0,
+  "rotary_percent": 1.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "type_vocab_size": 0,
+  "use_cache": true,
+  "vocab_size": 99584
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8578ce95e087b33386071c999621a00eea2fa03f1fc4fccef7fef600e867915
+size 2602680660

special_tokens_map.json

@@ -0,0 +1,54 @@
+{
+  "additional_special_tokens": [
+    "<EOD|LLM-jp>"
+  ],
+  "bos_token": {
+    "content": "<s|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<CLS|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<MASK|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<PAD|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<SEP|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk|LLM-jp>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,84 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<MASK|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<PAD|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<CLS|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<SEP|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<EOD|LLM-jp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<EOD|LLM-jp>"
+  ],
+  "bos_token": "<s|LLM-jp>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<CLS|LLM-jp>",
+  "eod_token": "</s|LLM-jp>",
+  "eos_token": "</s|LLM-jp>",
+  "extra_ids": 0,
+  "mask_token": "<MASK|LLM-jp>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD|LLM-jp>",
+  "sep_token": "<SEP|LLM-jp>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk|LLM-jp>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddb61849c08709a6eb8922610c0b71174ac8273f6151d49607e26609db230e6e
+size 5176