phatjk commited on
Commit
746280e
·
1 Parent(s): 972e874

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +4 -0
  2. special_tokens_map.json +16 -0
  3. vocab.json +91 -75
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "</s>": 92,
3
+ "<s>": 91
4
+ }
special_tokens_map.json CHANGED
@@ -1,4 +1,20 @@
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
  "pad_token": "[PAD]",
 
1
  {
2
+ "additional_special_tokens": [
3
+ {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": true,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "</s>",
12
+ "lstrip": false,
13
+ "normalized": true,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
17
+ ],
18
  "bos_token": "<s>",
19
  "eos_token": "</s>",
20
  "pad_token": "[PAD]",
vocab.json CHANGED
@@ -1,77 +1,93 @@
1
  {
2
- "A": 53,
3
- "B": 47,
4
- "C": 14,
5
- "D": 62,
6
- "E": 15,
7
- "G": 19,
8
- "H": 31,
9
- "I": 0,
10
- "K": 64,
11
- "L": 23,
12
- "M": 18,
13
- "N": 10,
14
- "O": 50,
15
- "P": 46,
16
- "Q": 44,
17
- "R": 70,
18
- "S": 68,
19
- "T": 49,
20
- "U": 29,
21
- "V": 3,
22
- "X": 1,
23
- "Y": 63,
24
- "[PAD]": 74,
25
- "[UNK]": 73,
26
- "|": 2,
27
- "À": 13,
28
- "Á": 71,
29
- "Â": 8,
30
- "Ã": 28,
31
- "È": 65,
32
- "Ê": 61,
33
- "Ì": 58,
34
- "Í": 7,
35
- "Ò": 56,
36
- "Ó": 66,
37
- "Ô": 11,
38
- "Ù": 38,
39
- "Ú": 42,
40
- "Ý": 34,
41
- "Đ": 6,
42
- "Ĩ": 36,
43
- "Ơ": 41,
44
- "Ư": 45,
45
- "": 51,
46
- "": 40,
47
- "": 9,
48
- "": 35,
49
- "": 67,
50
- "": 59,
51
- "": 16,
52
- "": 69,
53
- "": 37,
54
- "": 24,
55
- "": 5,
56
- "": 57,
57
- "": 12,
58
- "": 17,
59
- "": 33,
60
- "": 52,
61
- "": 27,
62
- "": 4,
63
- "": 22,
64
- "": 60,
65
- "": 39,
66
- "": 25,
67
- "": 72,
68
- "": 26,
69
- "": 54,
70
- "": 55,
71
- "": 30,
72
- "": 48,
73
- "": 43,
74
- "": 21,
75
- "": 20,
76
- "": 32
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  }
 
1
  {
2
+ "[PAD]": 90,
3
+ "[UNK]": 89,
4
+ "a": 86,
5
+ "b": 81,
6
+ "c": 82,
7
+ "d": 19,
8
+ "e": 71,
9
+ "g": 4,
10
+ "h": 21,
11
+ "i": 36,
12
+ "k": 49,
13
+ "l": 58,
14
+ "m": 54,
15
+ "n": 46,
16
+ "o": 80,
17
+ "p": 76,
18
+ "q": 12,
19
+ "r": 57,
20
+ "s": 48,
21
+ "t": 45,
22
+ "u": 85,
23
+ "v": 73,
24
+ "x": 22,
25
+ "y": 43,
26
+ "|": 69,
27
+ "à": 3,
28
+ "á": 42,
29
+ "â": 50,
30
+ "ã": 31,
31
+ "è": 52,
32
+ "é": 30,
33
+ "ê": 37,
34
+ "ì": 32,
35
+ "í": 61,
36
+ "ò": 13,
37
+ "ó": 40,
38
+ "ô": 67,
39
+ "õ": 20,
40
+ "ù": 35,
41
+ "ú": 65,
42
+ "ý": 47,
43
+ "ă": 75,
44
+ "đ": 39,
45
+ "ĩ": 79,
46
+ "ũ": 1,
47
+ "ơ": 29,
48
+ "ư": 87,
49
+ "": 15,
50
+ "": 5,
51
+ "": 53,
52
+ "": 16,
53
+ "": 64,
54
+ "": 18,
55
+ "": 33,
56
+ "": 55,
57
+ "": 68,
58
+ "": 28,
59
+ "": 14,
60
+ "": 77,
61
+ "": 72,
62
+ "": 78,
63
+ "": 17,
64
+ "ế": 23,
65
+ "": 41,
66
+ "": 24,
67
+ "": 10,
68
+ "": 59,
69
+ "": 2,
70
+ "": 38,
71
+ "": 83,
72
+ "": 60,
73
+ "": 44,
74
+ "": 70,
75
+ "": 25,
76
+ "": 66,
77
+ "ộ": 84,
78
+ "ớ": 8,
79
+ "ờ": 62,
80
+ "ở": 11,
81
+ "ỡ": 6,
82
+ "ợ": 9,
83
+ "ụ": 88,
84
+ "ủ": 7,
85
+ "ứ": 0,
86
+ "ừ": 56,
87
+ "ử": 63,
88
+ "ữ": 26,
89
+ "ự": 27,
90
+ "ỳ": 34,
91
+ "ỷ": 51,
92
+ "ỹ": 74
93
  }