Upload folder using huggingface_hub

README.md

@@ -0,0 +1,89 @@
+---
+license: bigscience-openrail-m
+base_model: ehsanaghaei/SecureBERT
+tags:
+- generated_from_trainer
+metrics:
+- precision
+- recall
+- f1
+- accuracy
+model-index:
+- name: our_data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# our_data
+
+This model is a fine-tuned version of [ehsanaghaei/SecureBERT](https://huggingface.co/ehsanaghaei/SecureBERT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.4668
+- Precision: 0.4762
+- Recall: 0.5291
+- F1: 0.5013
+- Accuracy: 0.7376
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 10.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Precision | Recall | F1     | Accuracy |
+|:-------------:|:-----:|:-----:|:---------------:|:---------:|:------:|:------:|:--------:|
+| 1.9177        | 0.4   | 500   | 1.6839          | 0.06      | 0.0278 | 0.0380 | 0.6004   |
+| 1.4976        | 0.81  | 1000  | 1.4936          | 0.2281    | 0.2659 | 0.2456 | 0.6313   |
+| 1.2309        | 1.21  | 1500  | 1.2915          | 0.2650    | 0.3148 | 0.2878 | 0.6657   |
+| 1.0546        | 1.61  | 2000  | 1.2454          | 0.2950    | 0.3796 | 0.3320 | 0.6804   |
+| 0.9405        | 2.01  | 2500  | 1.2377          | 0.3613    | 0.3532 | 0.3572 | 0.6916   |
+| 0.7501        | 2.42  | 3000  | 1.1723          | 0.3607    | 0.4180 | 0.3873 | 0.7171   |
+| 0.7133        | 2.82  | 3500  | 1.1584          | 0.3632    | 0.4444 | 0.3998 | 0.7160   |
+| 0.5896        | 3.22  | 4000  | 1.2288          | 0.4103    | 0.4444 | 0.4267 | 0.7306   |
+| 0.5353        | 3.63  | 4500  | 1.2319          | 0.3978    | 0.4815 | 0.4357 | 0.7254   |
+| 0.5432        | 4.03  | 5000  | 1.2173          | 0.4269    | 0.4868 | 0.4549 | 0.7306   |
+| 0.4062        | 4.43  | 5500  | 1.2832          | 0.4398    | 0.5026 | 0.4691 | 0.7272   |
+| 0.4485        | 4.83  | 6000  | 1.2196          | 0.4212    | 0.5093 | 0.4611 | 0.7412   |
+| 0.3614        | 5.24  | 6500  | 1.3155          | 0.4325    | 0.4960 | 0.4621 | 0.7325   |
+| 0.3308        | 5.64  | 7000  | 1.3501          | 0.4184    | 0.5119 | 0.4604 | 0.7354   |
+| 0.3645        | 6.04  | 7500  | 1.3391          | 0.4359    | 0.5172 | 0.4731 | 0.7366   |
+| 0.2982        | 6.45  | 8000  | 1.3889          | 0.4093    | 0.5225 | 0.4590 | 0.7315   |
+| 0.2845        | 6.85  | 8500  | 1.4109          | 0.4452    | 0.5159 | 0.4779 | 0.7377   |
+| 0.2482        | 7.25  | 9000  | 1.4668          | 0.4762    | 0.5291 | 0.5013 | 0.7376   |
+| 0.2636        | 7.66  | 9500  | 1.4925          | 0.4540    | 0.5357 | 0.4915 | 0.7341   |
+| 0.2605        | 8.06  | 10000 | 1.4916          | 0.4586    | 0.5344 | 0.4936 | 0.7405   |
+| 0.1989        | 8.46  | 10500 | 1.5096          | 0.4661    | 0.5370 | 0.4991 | 0.7387   |
+| 0.2415        | 8.86  | 11000 | 1.4698          | 0.4603    | 0.5450 | 0.4991 | 0.7443   |
+| 0.2488        | 9.27  | 11500 | 1.4736          | 0.4578    | 0.5304 | 0.4914 | 0.7455   |
+| 0.2129        | 9.67  | 12000 | 1.5067          | 0.4640    | 0.5450 | 0.5012 | 0.7439   |
+
+
+### Framework versions
+
+- Transformers 4.36.0.dev0
+- Pytorch 2.1.0+cu118
+- Datasets 2.15.0
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,25 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7375506314033834,
+    "eval_f1": 0.5012531328320803,
+    "eval_loss": 1.4667584896087646,
+    "eval_precision": 0.47619047619047616,
+    "eval_recall": 0.5291005291005291,
+    "eval_runtime": 2.5734,
+    "eval_samples": 304,
+    "eval_samples_per_second": 118.132,
+    "eval_steps_per_second": 59.066,
+    "predict_accuracy": 0.7877786952931461,
+    "predict_f1": 0.5529573590096286,
+    "predict_loss": 1.0716973543167114,
+    "predict_precision": 0.5193798449612403,
+    "predict_recall": 0.5911764705882353,
+    "predict_runtime": 1.1,
+    "predict_samples_per_second": 119.096,
+    "predict_steps_per_second": 60.002,
+    "train_loss": 0.5683070008357046,
+    "train_runtime": 1353.7746,
+    "train_samples": 2481,
+    "train_samples_per_second": 18.327,
+    "train_steps_per_second": 9.167
+}

config.json

@@ -0,0 +1,115 @@
+{
+  "_name_or_path": "ehsanaghaei/SecureBERT",
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "finetuning_task": "ner",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "B-ATTACK_MOTIVATION",
+    "1": "B-ATTACK_PATTERN",
+    "2": "B-ATTACK_RESOURCE_LEVEL",
+    "3": "B-ATTACK_SOPHISTICATION_LEVEL",
+    "4": "B-ATTACK_TOOL",
+    "5": "B-CAMPAIGN",
+    "6": "B-COURSE_OF_ACTION",
+    "7": "B-GENERAL_IDENTITY",
+    "8": "B-GENERAL_TOOL",
+    "9": "B-IMPACT",
+    "10": "B-INDICATOR",
+    "11": "B-INFRASTRUCTURE",
+    "12": "B-INTRUSION_SET",
+    "13": "B-LOCATION",
+    "14": "B-MALWARE",
+    "15": "B-MALWARE_ANALYSIS",
+    "16": "B-OBSERVED_DATA",
+    "17": "B-THREAT_ACTOR",
+    "18": "B-VICTIM_IDENTITY",
+    "19": "B-VULNERABILITY",
+    "20": "I-ATTACK_MOTIVATION",
+    "21": "I-ATTACK_PATTERN",
+    "22": "I-ATTACK_RESOURCE_LEVEL",
+    "23": "I-ATTACK_SOPHISTICATION_LEVEL",
+    "24": "I-ATTACK_TOOL",
+    "25": "I-CAMPAIGN",
+    "26": "I-COURSE_OF_ACTION",
+    "27": "I-GENERAL_IDENTITY",
+    "28": "I-GENERAL_TOOL",
+    "29": "I-IMPACT",
+    "30": "I-INDICATOR",
+    "31": "I-INFRASTRUCTURE",
+    "32": "I-INTRUSION_SET",
+    "33": "I-LOCATION",
+    "34": "I-MALWARE",
+    "35": "I-MALWARE_ANALYSIS",
+    "36": "I-OBSERVED_DATA",
+    "37": "I-THREAT_ACTOR",
+    "38": "I-VICTIM_IDENTITY",
+    "39": "I-VULNERABILITY",
+    "40": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-ATTACK_MOTIVATION": 0,
+    "B-ATTACK_PATTERN": 1,
+    "B-ATTACK_RESOURCE_LEVEL": 2,
+    "B-ATTACK_SOPHISTICATION_LEVEL": 3,
+    "B-ATTACK_TOOL": 4,
+    "B-CAMPAIGN": 5,
+    "B-COURSE_OF_ACTION": 6,
+    "B-GENERAL_IDENTITY": 7,
+    "B-GENERAL_TOOL": 8,
+    "B-IMPACT": 9,
+    "B-INDICATOR": 10,
+    "B-INFRASTRUCTURE": 11,
+    "B-INTRUSION_SET": 12,
+    "B-LOCATION": 13,
+    "B-MALWARE": 14,
+    "B-MALWARE_ANALYSIS": 15,
+    "B-OBSERVED_DATA": 16,
+    "B-THREAT_ACTOR": 17,
+    "B-VICTIM_IDENTITY": 18,
+    "B-VULNERABILITY": 19,
+    "I-ATTACK_MOTIVATION": 20,
+    "I-ATTACK_PATTERN": 21,
+    "I-ATTACK_RESOURCE_LEVEL": 22,
+    "I-ATTACK_SOPHISTICATION_LEVEL": 23,
+    "I-ATTACK_TOOL": 24,
+    "I-CAMPAIGN": 25,
+    "I-COURSE_OF_ACTION": 26,
+    "I-GENERAL_IDENTITY": 27,
+    "I-GENERAL_TOOL": 28,
+    "I-IMPACT": 29,
+    "I-INDICATOR": 30,
+    "I-INFRASTRUCTURE": 31,
+    "I-INTRUSION_SET": 32,
+    "I-LOCATION": 33,
+    "I-MALWARE": 34,
+    "I-MALWARE_ANALYSIS": 35,
+    "I-OBSERVED_DATA": 36,
+    "I-THREAT_ACTOR": 37,
+    "I-VICTIM_IDENTITY": 38,
+    "I-VULNERABILITY": 39,
+    "O": 40
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

eval_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7375506314033834,
+    "eval_f1": 0.5012531328320803,
+    "eval_loss": 1.4667584896087646,
+    "eval_precision": 0.47619047619047616,
+    "eval_recall": 0.5291005291005291,
+    "eval_runtime": 2.5734,
+    "eval_samples": 304,
+    "eval_samples_per_second": 118.132,
+    "eval_steps_per_second": 59.066
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c12ae782862818ffa9ef401e592c362b2794a0c76aad2c1dcc5eec6abe855687
+size 496370212

predict_results.json

@@ -0,0 +1,10 @@
+{
+    "predict_accuracy": 0.7877786952931461,
+    "predict_f1": 0.5529573590096286,
+    "predict_loss": 1.0716973543167114,
+    "predict_precision": 0.5193798449612403,
+    "predict_recall": 0.5911764705882353,
+    "predict_runtime": 1.1,
+    "predict_samples_per_second": 119.096,
+    "predict_steps_per_second": 60.002
+}

predictions.txt

@@ -0,0 +1,131 @@
+O O O B-ATTACK_TOOL O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O B-THREAT_ACTOR O O O O O O O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-CAMPAIGN O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O
+O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+O O O O O O O O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O I-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS O O O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O B-THREAT_ACTOR I-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O O O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O O O O B-VICTIM_IDENTITY O O O O O B-MALWARE O
+O O O B-MALWARE O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O
+O B-MALWARE B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O O O O O O
+B-MALWARE O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O
+O O O O O O O B-THREAT_ACTOR O O O B-THREAT_ACTOR O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O B-VICTIM_IDENTITY O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O
+O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-MALWARE I-MALWARE O O O B-VULNERABILITY I-INFRASTRUCTURE O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O B-IMPACT I-IMPACT I-IMPACT B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O O O O O O O O O O O B-VICTIM_IDENTITY O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O B-THREAT_ACTOR O O O O O O O
+B-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O
+O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O
+O O O O O O O B-VICTIM_IDENTITY O O O O O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O O O
+O O B-MALWARE O B-THREAT_ACTOR O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O I-ATTACK_PATTERN O O O
+O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-THREAT_ACTOR I-THREAT_ACTOR O
+O O O B-INTRUSION_SET I-INTRUSION_SET O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O B-GENERAL_IDENTITY O O O O O B-GENERAL_IDENTITY O
+B-THREAT_ACTOR O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O
+B-MALWARE O O O O O O O B-MALWARE I-MALWARE I-MALWARE O O O O O O O B-ATTACK_TOOL O O O O O O O O O O O O O O O O O O
+O O O O O O O B-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O
+O O O O B-CAMPAIGN I-THREAT_ACTOR I-CAMPAIGN O B-VICTIM_IDENTITY O O O O O O O O O B-GENERAL_IDENTITY O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O
+O O O O I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-INFRASTRUCTURE I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O
+O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-INFRASTRUCTURE I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O
+O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O B-THREAT_ACTOR O
+O O B-THREAT_ACTOR O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O
+O O O O B-CAMPAIGN O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O B-MALWARE O O O B-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O B-CAMPAIGN O
+B-THREAT_ACTOR I-THREAT_ACTOR O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O B-MALWARE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O
+B-THREAT_ACTOR O B-ATTACK_TOOL O O O O O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-ATTACK_TOOL O
+B-CAMPAIGN O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-INFRASTRUCTURE O I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN O O I-ATTACK_PATTERN O I-ATTACK_TOOL I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-INFRASTRUCTURE O O O O O O O O
+B-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O I-ATTACK_MOTIVATION O
+O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O B-THREAT_ACTOR I-CAMPAIGN O O O O O O O O O O O
+O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-ATTACK_TOOL I-ATTACK_TOOL O
+O O O O O B-THREAT_ACTOR O B-ATTACK_TOOL O O O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O I-MALWARE_ANALYSIS O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O B-ATTACK_TOOL I-ATTACK_TOOL I-INFRASTRUCTURE O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+B-MALWARE I-MALWARE I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O O O O O O O O O O O
+O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-ATTACK_TOOL O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O O O
+O O O B-CAMPAIGN I-INTRUSION_SET O B-VICTIM_IDENTITY I-VICTIM_IDENTITY
+O O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-MALWARE O O O O O O O O O O O O B-ATTACK_TOOL O B-ATTACK_TOOL O O O O O O O O
+O O O O O B-MALWARE I-MALWARE O
+O O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O O B-INFRASTRUCTURE B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O O O O
+O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O I-MALWARE I-MALWARE B-MALWARE I-MALWARE I-MALWARE O O B-MALWARE O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O O O O O O O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O
+B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY
+B-OBSERVED_DATA O I-ATTACK_PATTERN I-ATTACK_PATTERN I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O I-OBSERVED_DATA I-OBSERVED_DATA O I-OBSERVED_DATA I-OBSERVED_DATA O O O O O O O
+B-ATTACK_TOOL O O O O O O O O O O O O O O O
+O O O O O B-GENERAL_IDENTITY O O O O B-LOCATION O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O O O
+O O O O O O B-MALWARE I-MALWARE O O O O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O O B-MALWARE O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-OBSERVED_DATA O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O I-OBSERVED_DATA O O O O O O O O
+O O O O O O O B-MALWARE O O O
+O O O O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O O O O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O O B-MALWARE O O O O O O O O O O
+B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O B-THREAT_ACTOR O O O O O O O O O I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O B-THREAT_ACTOR O O O B-THREAT_ACTOR O O O O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O O B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O O O I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O O O O O B-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_TOOL I-ATTACK_TOOL O O B-ATTACK_TOOL I-ATTACK_TOOL O
+O I-CAMPAIGN O O O O O O O O O O O O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O
+O O O O O O B-MALWARE I-MALWARE O O O O
+O O O O O O O O O O O O O B-ATTACK_PATTERN O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O B-THREAT_ACTOR O O O O O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O O O O O O I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O O O O O O O O O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O
+O B-IMPACT I-IMPACT I-IMPACT O O O O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O B-LOCATION I-LOCATION O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O B-THREAT_ACTOR O
+O O O O O O B-COURSE_OF_ACTION O O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-ATTACK_TOOL O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O
+O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O B-GENERAL_IDENTITY O O B-VULNERABILITY I-VULNERABILITY O O O O O O O O O O O O O O O O
+O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O O O O O O O O O O I-IMPACT I-IMPACT O O O O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-INFRASTRUCTURE O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-GENERAL_TOOL O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O B-GENERAL_TOOL O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O O O O B-MALWARE O O O B-MALWARE O B-MALWARE O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O O O B-ATTACK_PATTERN O
+O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O B-THREAT_ACTOR O B-INTRUSION_SET I-INTRUSION_SET O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O B-LOCATION I-VICTIM_IDENTITY O
+O O O O O O O O O O O O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O B-ATTACK_MOTIVATION O
+B-THREAT_ACTOR O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_PATTERN I-ATTACK_PATTERN I-CAMPAIGN O O O O O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O
+O O O O O O O O O B-MALWARE O O O O B-CAMPAIGN I-CAMPAIGN O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O B-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_TOOL O O O O O O O O O O O
+O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-OBSERVED_DATA B-OBSERVED_DATA B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O B-VULNERABILITY O O O O O O O I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O O B-MALWARE O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O

runs/Nov13_12-57-19_75b3cadc803a/events.out.tfevents.1699880246.75b3cadc803a.46504.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e94a229368645c6be2fdd638c5c1f2267b2e330b6c256ab47f24c85c6fb08224
+size 35326

runs/Nov13_12-57-19_75b3cadc803a/events.out.tfevents.1699881570.75b3cadc803a.46504.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9a6209faa298d67dfab999756c10db5bb79167ff3e5814f061bc490383cd43
+size 560

runs/Nov20_20-36-36_4ba4cc77ee77/events.out.tfevents.1700512602.4ba4cc77ee77.10806.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c9e5acbf19b6c714dc231b1132883cdfb9747bb452798f9c04410c6a3033cd1
+size 30964

runs/Nov20_20-36-36_4ba4cc77ee77/events.out.tfevents.1700513674.4ba4cc77ee77.10806.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6be25217bc8683288866900dddd6ce318d1c1a5df0a3e0f07dbeb6c13ea2927f
+size 560

runs/Nov21_15-11-53_84c49691a4fe/events.out.tfevents.1700579521.84c49691a4fe.37829.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3f4d800517e4dd734236210c865470539fb965a44b1f78e1ee2715c1e9be1b8
+size 34109

runs/Nov21_15-11-53_84c49691a4fe/events.out.tfevents.1700580757.84c49691a4fe.37829.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d216e7388a204c46f1a83876c79bbf9becef4b18d05afb2573f739b6f69b0f
+size 560

runs/Nov30_23-10-58_7680f78f04a8/events.out.tfevents.1701385870.7680f78f04a8.12426.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c115caa68fc039ff9125df7c551ebb373f7009a19efc75bda66aa34b54e9a58
+size 22186

runs/Nov30_23-10-58_7680f78f04a8/events.out.tfevents.1701387228.7680f78f04a8.12426.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebe262b55f31c805fcfa89786a0e833444ea00bf325222ed836673275e124a11
+size 560

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "max_len": 512,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 10.0,
+    "train_loss": 0.5683070008357046,
+    "train_runtime": 1353.7746,
+    "train_samples": 2481,
+    "train_samples_per_second": 18.327,
+    "train_steps_per_second": 9.167
+}

trainer_state.json

@@ -0,0 +1,461 @@
+{
+  "best_metric": 0.5012531328320803,
+  "best_model_checkpoint": "/content/our_data/checkpoint-9000",
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 12410,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.91941982272361e-05,
+      "loss": 1.9177,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_accuracy": 0.6004288777698356,
+      "eval_f1": 0.0379746835443038,
+      "eval_loss": 1.6839170455932617,
+      "eval_precision": 0.06,
+      "eval_recall": 0.027777777777777776,
+      "eval_runtime": 3.4052,
+      "eval_samples_per_second": 89.275,
+      "eval_steps_per_second": 44.637,
+      "step": 500
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.83883964544722e-05,
+      "loss": 1.4976,
+      "step": 1000
+    },
+    {
+      "epoch": 0.81,
+      "eval_accuracy": 0.63128425065523,
+      "eval_f1": 0.24557116676847893,
+      "eval_loss": 1.4935959577560425,
+      "eval_precision": 0.22814982973893302,
+      "eval_recall": 0.26587301587301587,
+      "eval_runtime": 2.3845,
+      "eval_samples_per_second": 127.488,
+      "eval_steps_per_second": 63.744,
+      "step": 1000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.75825946817083e-05,
+      "loss": 1.2309,
+      "step": 1500
+    },
+    {
+      "epoch": 1.21,
+      "eval_accuracy": 0.6657136049559209,
+      "eval_f1": 0.2877871825876663,
+      "eval_loss": 1.2914698123931885,
+      "eval_precision": 0.2650334075723831,
+      "eval_recall": 0.3148148148148148,
+      "eval_runtime": 3.4851,
+      "eval_samples_per_second": 87.229,
+      "eval_steps_per_second": 43.615,
+      "step": 1500
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 1.67767929089444e-05,
+      "loss": 1.0546,
+      "step": 2000
+    },
+    {
+      "epoch": 1.61,
+      "eval_accuracy": 0.6803669287586371,
+      "eval_f1": 0.33198380566801616,
+      "eval_loss": 1.2454315423965454,
+      "eval_precision": 0.2949640287769784,
+      "eval_recall": 0.37962962962962965,
+      "eval_runtime": 2.4542,
+      "eval_samples_per_second": 123.867,
+      "eval_steps_per_second": 61.934,
+      "step": 2000
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.59709911361805e-05,
+      "loss": 0.9405,
+      "step": 2500
+    },
+    {
+      "epoch": 2.01,
+      "eval_accuracy": 0.6915654038599,
+      "eval_f1": 0.35719063545150503,
+      "eval_loss": 1.2377290725708008,
+      "eval_precision": 0.36129905277401897,
+      "eval_recall": 0.3531746031746032,
+      "eval_runtime": 3.3956,
+      "eval_samples_per_second": 89.527,
+      "eval_steps_per_second": 44.763,
+      "step": 2500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.5165189363416601e-05,
+      "loss": 0.7501,
+      "step": 3000
+    },
+    {
+      "epoch": 2.42,
+      "eval_accuracy": 0.7170598046223493,
+      "eval_f1": 0.3872549019607843,
+      "eval_loss": 1.1723062992095947,
+      "eval_precision": 0.3607305936073059,
+      "eval_recall": 0.41798941798941797,
+      "eval_runtime": 2.4666,
+      "eval_samples_per_second": 123.248,
+      "eval_steps_per_second": 61.624,
+      "step": 3000
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.4359387590652701e-05,
+      "loss": 0.7133,
+      "step": 3500
+    },
+    {
+      "epoch": 2.82,
+      "eval_accuracy": 0.7159876101977604,
+      "eval_f1": 0.39976204640095175,
+      "eval_loss": 1.1583572626113892,
+      "eval_precision": 0.36324324324324325,
+      "eval_recall": 0.4444444444444444,
+      "eval_runtime": 2.4767,
+      "eval_samples_per_second": 122.743,
+      "eval_steps_per_second": 61.371,
+      "step": 3500
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 1.35535858178888e-05,
+      "loss": 0.5896,
+      "step": 4000
+    },
+    {
+      "epoch": 3.22,
+      "eval_accuracy": 0.7306409340004766,
+      "eval_f1": 0.42666666666666664,
+      "eval_loss": 1.2287709712982178,
+      "eval_precision": 0.41025641025641024,
+      "eval_recall": 0.4444444444444444,
+      "eval_runtime": 3.2475,
+      "eval_samples_per_second": 93.609,
+      "eval_steps_per_second": 46.805,
+      "step": 4000
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 1.27477840451249e-05,
+      "loss": 0.5353,
+      "step": 4500
+    },
+    {
+      "epoch": 3.63,
+      "eval_accuracy": 0.7253990945913747,
+      "eval_f1": 0.4356672651107121,
+      "eval_loss": 1.2319059371948242,
+      "eval_precision": 0.3978142076502732,
+      "eval_recall": 0.48148148148148145,
+      "eval_runtime": 2.3963,
+      "eval_samples_per_second": 126.861,
+      "eval_steps_per_second": 63.431,
+      "step": 4500
+    },
+    {
+      "epoch": 4.03,
+      "learning_rate": 1.1941982272361e-05,
+      "loss": 0.5432,
+      "step": 5000
+    },
+    {
+      "epoch": 4.03,
+      "eval_accuracy": 0.7306409340004766,
+      "eval_f1": 0.4548825710754017,
+      "eval_loss": 1.2172613143920898,
+      "eval_precision": 0.42691415313225056,
+      "eval_recall": 0.48677248677248675,
+      "eval_runtime": 3.3895,
+      "eval_samples_per_second": 89.689,
+      "eval_steps_per_second": 44.845,
+      "step": 5000
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 1.11361804995971e-05,
+      "loss": 0.4062,
+      "step": 5500
+    },
+    {
+      "epoch": 4.43,
+      "eval_accuracy": 0.7271860852990231,
+      "eval_f1": 0.4691358024691359,
+      "eval_loss": 1.283239722251892,
+      "eval_precision": 0.4398148148148148,
+      "eval_recall": 0.5026455026455027,
+      "eval_runtime": 2.4143,
+      "eval_samples_per_second": 125.916,
+      "eval_steps_per_second": 62.958,
+      "step": 5500
+    },
+    {
+      "epoch": 4.83,
+      "learning_rate": 1.0330378726833199e-05,
+      "loss": 0.4485,
+      "step": 6000
+    },
+    {
+      "epoch": 4.83,
+      "eval_accuracy": 0.7412437455325233,
+      "eval_f1": 0.4610778443113772,
+      "eval_loss": 1.2196030616760254,
+      "eval_precision": 0.4212253829321663,
+      "eval_recall": 0.5092592592592593,
+      "eval_runtime": 2.7872,
+      "eval_samples_per_second": 109.069,
+      "eval_steps_per_second": 54.534,
+      "step": 6000
+    },
+    {
+      "epoch": 5.24,
+      "learning_rate": 9.5245769540693e-06,
+      "loss": 0.3614,
+      "step": 6500
+    },
+    {
+      "epoch": 5.24,
+      "eval_accuracy": 0.732547057421968,
+      "eval_f1": 0.46210720887245843,
+      "eval_loss": 1.3155299425125122,
+      "eval_precision": 0.43252595155709345,
+      "eval_recall": 0.49603174603174605,
+      "eval_runtime": 3.3893,
+      "eval_samples_per_second": 89.694,
+      "eval_steps_per_second": 44.847,
+      "step": 6500
+    },
+    {
+      "epoch": 5.64,
+      "learning_rate": 8.7187751813054e-06,
+      "loss": 0.3308,
+      "step": 7000
+    },
+    {
+      "epoch": 5.64,
+      "eval_accuracy": 0.7354062425542054,
+      "eval_f1": 0.4604402141582391,
+      "eval_loss": 1.3501168489456177,
+      "eval_precision": 0.4183783783783784,
+      "eval_recall": 0.5119047619047619,
+      "eval_runtime": 2.4288,
+      "eval_samples_per_second": 125.165,
+      "eval_steps_per_second": 62.583,
+      "step": 7000
+    },
+    {
+      "epoch": 6.04,
+      "learning_rate": 7.9129734085415e-06,
+      "loss": 0.3645,
+      "step": 7500
+    },
+    {
+      "epoch": 6.04,
+      "eval_accuracy": 0.7365975696926376,
+      "eval_f1": 0.4730792498487599,
+      "eval_loss": 1.3390766382217407,
+      "eval_precision": 0.4358974358974359,
+      "eval_recall": 0.5171957671957672,
+      "eval_runtime": 2.391,
+      "eval_samples_per_second": 127.141,
+      "eval_steps_per_second": 63.571,
+      "step": 7500
+    },
+    {
+      "epoch": 6.45,
+      "learning_rate": 7.107171635777599e-06,
+      "loss": 0.2982,
+      "step": 8000
+    },
+    {
+      "epoch": 6.45,
+      "eval_accuracy": 0.7314748629973791,
+      "eval_f1": 0.4590354445090064,
+      "eval_loss": 1.3889434337615967,
+      "eval_precision": 0.40932642487046633,
+      "eval_recall": 0.5224867724867724,
+      "eval_runtime": 3.0054,
+      "eval_samples_per_second": 101.151,
+      "eval_steps_per_second": 50.576,
+      "step": 8000
+    },
+    {
+      "epoch": 6.85,
+      "learning_rate": 6.301369863013699e-06,
+      "loss": 0.2845,
+      "step": 8500
+    },
+    {
+      "epoch": 6.85,
+      "eval_accuracy": 0.7376697641172266,
+      "eval_f1": 0.47794117647058826,
+      "eval_loss": 1.4109262228012085,
+      "eval_precision": 0.4452054794520548,
+      "eval_recall": 0.5158730158730159,
+      "eval_runtime": 2.4417,
+      "eval_samples_per_second": 124.505,
+      "eval_steps_per_second": 62.253,
+      "step": 8500
+    },
+    {
+      "epoch": 7.25,
+      "learning_rate": 5.495568090249799e-06,
+      "loss": 0.2482,
+      "step": 9000
+    },
+    {
+      "epoch": 7.25,
+      "eval_accuracy": 0.7375506314033834,
+      "eval_f1": 0.5012531328320803,
+      "eval_loss": 1.4667584896087646,
+      "eval_precision": 0.47619047619047616,
+      "eval_recall": 0.5291005291005291,
+      "eval_runtime": 2.4653,
+      "eval_samples_per_second": 123.313,
+      "eval_steps_per_second": 61.657,
+      "step": 9000
+    },
+    {
+      "epoch": 7.66,
+      "learning_rate": 4.689766317485899e-06,
+      "loss": 0.2636,
+      "step": 9500
+    },
+    {
+      "epoch": 7.66,
+      "eval_accuracy": 0.73409578270193,
+      "eval_f1": 0.49150485436893204,
+      "eval_loss": 1.4925192594528198,
+      "eval_precision": 0.45403587443946186,
+      "eval_recall": 0.5357142857142857,
+      "eval_runtime": 3.4322,
+      "eval_samples_per_second": 88.572,
+      "eval_steps_per_second": 44.286,
+      "step": 9500
+    },
+    {
+      "epoch": 8.06,
+      "learning_rate": 3.883964544721999e-06,
+      "loss": 0.2605,
+      "step": 10000
+    },
+    {
+      "epoch": 8.06,
+      "eval_accuracy": 0.7405289492494639,
+      "eval_f1": 0.49358582773365917,
+      "eval_loss": 1.4916423559188843,
+      "eval_precision": 0.4585698070374574,
+      "eval_recall": 0.5343915343915344,
+      "eval_runtime": 2.4755,
+      "eval_samples_per_second": 122.805,
+      "eval_steps_per_second": 61.403,
+      "step": 10000
+    },
+    {
+      "epoch": 8.46,
+      "learning_rate": 3.0781627719580986e-06,
+      "loss": 0.1989,
+      "step": 10500
+    },
+    {
+      "epoch": 8.46,
+      "eval_accuracy": 0.7387419585418156,
+      "eval_f1": 0.4990780577750461,
+      "eval_loss": 1.5096321105957031,
+      "eval_precision": 0.4661308840413318,
+      "eval_recall": 0.5370370370370371,
+      "eval_runtime": 3.3245,
+      "eval_samples_per_second": 91.444,
+      "eval_steps_per_second": 45.722,
+      "step": 10500
+    },
+    {
+      "epoch": 8.86,
+      "learning_rate": 2.2723609991941985e-06,
+      "loss": 0.2415,
+      "step": 11000
+    },
+    {
+      "epoch": 8.86,
+      "eval_accuracy": 0.744341196092447,
+      "eval_f1": 0.4990914597213809,
+      "eval_loss": 1.4698182344436646,
+      "eval_precision": 0.46033519553072627,
+      "eval_recall": 0.544973544973545,
+      "eval_runtime": 2.5811,
+      "eval_samples_per_second": 117.779,
+      "eval_steps_per_second": 58.889,
+      "step": 11000
+    },
+    {
+      "epoch": 9.27,
+      "learning_rate": 1.4665592264302982e-06,
+      "loss": 0.2488,
+      "step": 11500
+    },
+    {
+      "epoch": 9.27,
+      "eval_accuracy": 0.7455325232308792,
+      "eval_f1": 0.491421568627451,
+      "eval_loss": 1.4736005067825317,
+      "eval_precision": 0.4577625570776256,
+      "eval_recall": 0.5304232804232805,
+      "eval_runtime": 3.3981,
+      "eval_samples_per_second": 89.462,
+      "eval_steps_per_second": 44.731,
+      "step": 11500
+    },
+    {
+      "epoch": 9.67,
+      "learning_rate": 6.607574536663981e-07,
+      "loss": 0.2129,
+      "step": 12000
+    },
+    {
+      "epoch": 9.67,
+      "eval_accuracy": 0.7438646652370741,
+      "eval_f1": 0.5012165450121654,
+      "eval_loss": 1.506749153137207,
+      "eval_precision": 0.46396396396396394,
+      "eval_recall": 0.544973544973545,
+      "eval_runtime": 2.854,
+      "eval_samples_per_second": 106.517,
+      "eval_steps_per_second": 53.258,
+      "step": 12000
+    },
+    {
+      "epoch": 10.0,
+      "step": 12410,
+      "total_flos": 503702005049490.0,
+      "train_loss": 0.5683070008357046,
+      "train_runtime": 1353.7746,
+      "train_samples_per_second": 18.327,
+      "train_steps_per_second": 9.167
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 12410,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "total_flos": 503702005049490.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54dbc352a92b7aedc19bf07ab796de02e755a77b802fad540649dcf5026c98b0
+size 4664