Sync config token ids

config.json

@@ -135,19 +135,24 @@
     "torch_dtype": "float32",
     "vocab_size": 1024
   },
+  "audio_end_id": 45,
   "audio_pool_step": 2,
+  "audio_start_id": 43,
   "auto_map": {
     "AutoConfig": "configuration_minicpm.MiniCPMOConfig",
     "AutoModel": "modeling_minicpmo.MiniCPMO",
     "AutoModelForCausalLM": "modeling_minicpmo.MiniCPMO"
   },
   "batch_vision_input": true,
-  "bos_token_id": 151643,
+  "bos_token_id": 1,
   "chunk_input": true,
   "drop_vision_last_layer": false,
-  "eos_token_id": 151645,
+  "eos_token_id": 1,
   "hidden_act": "silu",
   "hidden_size": 48,
+  "im_end_id": 3,
+  "im_start_id": 2,
+  "image_id": 23,
   "image_size": 448,
   "init_audio": true,
   "init_tts": true,
@@ -162,6 +167,7 @@
   "num_heads": 4,
   "num_hidden_layers": 2,
   "num_key_value_heads": 4,
+  "pad_token_id": 1,
   "patch_size": 14,
   "query_num": 64,
   "rms_norm_eps": 1e-06,
@@ -170,7 +176,9 @@
     "max_slice_nums": 9,
     "model_type": "minicpmv"
   },
+  "slice_end_id": 34,
   "slice_mode": true,
+  "slice_start_id": 33,
   "sliding_window": null,
   "stream_input": false,
   "tie_word_embeddings": false,
@@ -188,6 +196,7 @@
     "num_mel_bins": 10,
     "num_text_tokens": 1024
   },
+  "unk_token_id": 0,
   "use_cache": true,
   "use_image_id": true,
   "use_sliding_window": false,

generation_config.json

@@ -1,6 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 151643,
-  "eos_token_id": 151645,
+  "bos_token_id": 1,
+  "eos_token_id": 1,
+  "pad_token_id": 1,
   "transformers_version": "4.44.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c738167284d94b6a41654596a1bb5cb419616d832085f2de67280df0ba3f51b
+oid sha256:59a686f1801e1ed4d44e8e1ac7fc175dcae2b581570dea016e586c9f59e35c75
 size 5535352

modeling_minicpmo.py

@@ -56,7 +56,7 @@ from transformers.cache_utils import StaticCache
 from transformers.modeling_outputs import BaseModelOutputWithPast
 from transformers.modeling_outputs import ModelOutput
 from transformers.models.whisper.modeling_whisper import ACT2FN
-from transformers.models.whisper.modeling_whisper import WhisperAttention
+from transformers.models.whisper.modeling_whisper import WHISPER_ATTENTION_CLASSES
 from transformers.models.whisper.modeling_whisper import WhisperConfig
 from transformers.models.whisper.modeling_whisper import WhisperEncoder
 
@@ -1890,7 +1890,7 @@ class MiniCPMWhisperEncoderLayer(nn.Module):
     def __init__(self, config: WhisperConfig, layer_idx: int = None):
         super().__init__()
         self.embed_dim = config.d_model
-        self.self_attn = WhisperAttention(
+        self.self_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation](
             embed_dim=self.embed_dim,
             num_heads=config.encoder_attention_heads,
             dropout=config.attention_dropout,