Upload 3 files

special_tokens_map.json

@@ -1 +1,11 @@
-{"eos_token": "<eos>", "pad_token": "<pad>", "additional_special_tokens": ["<|endoftext|>", "<|sep|>", "<|acc|>", "<|tel|>", "<|rrn|>"]}
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<pad>",
+      "content": "<|unused0|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "<eos>",
+      "content": "<|unused1|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -107,8 +107,8 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "vocab": {
-      "<pad>": 0,
-      "<eos>": 1,
+      "<|unused0|>": 0,
+      "<|unused1|>": 1,
       "<|endoftext|>": 2,
       "<|sep|>": 3,
       "!": 4,
@@ -59851,4 +59851,4 @@
       "ìºIJ롤ëĿ¼ ìĿ´ëĤĺ"
     ]
   }
-}
+}

tokenizer_config.json

@@ -1 +1,6 @@
-{"pad_token": "<pad>", "eos_token": "<eos>", "tokenizer_class": "PreTrainedTokenizerFast"}
+{
+  "name_or_path": "EleutherAI/polyglot-ko-5.8b",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}