Hugging Face's logo

BioMike
/
go-tokenizer-9k

Transformers
Model card Files
xet
Community
BioMike commited on
Commit
934717e
·
verified ·
1 Parent(s): 353da68

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -6
tokenizer.json CHANGED
@@ -94,19 +94,19 @@
94
  "single": [
95
  {
96
  "SpecialToken": {
97
- "id": "[BOS]",
98
  "type_id": 0
99
  }
100
  },
101
  {
102
  "SpecialToken": {
103
- "id": "[PROMPT]",
104
  "type_id": 0
105
  }
106
  },
107
  {
108
  "SpecialToken": {
109
- "id": "[TIMESTAMP]",
110
  "type_id": 0
111
  }
112
  },
@@ -126,19 +126,19 @@
126
  "pair": [
127
  {
128
  "SpecialToken": {
129
- "id": "[BOS]",
130
  "type_id": 0
131
  }
132
  },
133
  {
134
  "SpecialToken": {
135
- "id": "[PROMPT]",
136
  "type_id": 0
137
  }
138
  },
139
  {
140
  "SpecialToken": {
141
- "id": "[TIMESTAMP]",
142
  "type_id": 0
143
  }
144
  },
 
94
  "single": [
95
  {
96
  "SpecialToken": {
97
+ "id": "[PROMPT]",
98
  "type_id": 0
99
  }
100
  },
101
  {
102
  "SpecialToken": {
103
+ "id": "[TIMESTAMP]",
104
  "type_id": 0
105
  }
106
  },
107
  {
108
  "SpecialToken": {
109
+ "id": "[BOS]",
110
  "type_id": 0
111
  }
112
  },
 
126
  "pair": [
127
  {
128
  "SpecialToken": {
129
+ "id": "[PROMPT]",
130
  "type_id": 0
131
  }
132
  },
133
  {
134
  "SpecialToken": {
135
+ "id": "[TIMESTAMP]",
136
  "type_id": 0
137
  }
138
  },
139
  {
140
  "SpecialToken": {
141
+ "id": "[BOS]",
142
  "type_id": 0
143
  }
144
  },