Hugging Face's logo

zhangj1an
/
AudioX

Text-to-Audio
Diffusers
Safetensors
AudioXPipeline
diffusion_cond
audio-generation
music-generation
Model card Files
xet
 Community
zhangj1an commited on
Commit
d55e834
·
verified ·
1 Parent(s): 27d5612

Repack weights: transformer + vae safetensors (Diffusers layout); drop legacy conditioners shard

Browse files
Files changed (4) hide show
  1. conditioners/diffusion_pytorch_model.safetensors +0 -3
  2. config.json +176 -124
  3. model_index.json +2 -8
  4. transformer/diffusion_pytorch_model.safetensors +2 -2
conditioners/diffusion_pytorch_model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3faa075c250e638d5d9c04ac1330f88e57a3268fe4459a106be1933f623a9307
3
- size 1104169076
 
 
 
 
config.json CHANGED
@@ -1,136 +1,188 @@
1
  {
2
- "model_type": "diffusion_cond",
3
- "sample_size": 485100,
4
- "sample_rate": 44100,
5
- "video_fps": 5,
6
- "audio_channels": 2,
7
- "model": {
8
- "pretransform": {
9
- "type": "autoencoder",
10
- "iterate_batch": true,
11
- "config": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  "encoder": {
13
- "type": "oobleck",
14
- "requires_grad": false,
15
- "config": {
16
- "in_channels": 2,
17
- "channels": 128,
18
- "c_mults": [1, 2, 4, 8, 16],
19
- "strides": [2, 4, 4, 8, 8],
20
- "latent_dim": 128,
21
- "use_snake": true
22
- }
 
 
 
 
 
 
 
 
 
 
 
 
23
  },
24
  "decoder": {
25
- "type": "oobleck",
26
- "config": {
27
- "out_channels": 2,
28
- "channels": 128,
29
- "c_mults": [1, 2, 4, 8, 16],
30
- "strides": [2, 4, 4, 8, 8],
31
- "latent_dim": 64,
32
- "use_snake": true,
33
- "final_tanh": false
34
- }
 
 
 
 
 
 
 
 
 
 
 
 
35
  },
36
  "bottleneck": {
37
- "type": "vae"
38
  },
39
  "latent_dim": 64,
40
  "downsampling_ratio": 2048,
41
  "io_channels": 2
42
- }
43
- },
44
- "conditioning": {
45
- "configs": [
46
- {
47
- "id": "video_prompt",
48
- "type": "clip-with-sync-w-empty-feat",
49
- "config": {
50
- "clip_model_name": "clip-vit-base-patch32"
51
- }
52
- },
53
- {
54
- "id": "text_prompt",
55
- "type": "t5",
56
- "config": {
57
- "t5_model_name": "t5-base",
58
- "max_length": 128
59
- }
60
- },
61
- {
62
- "id": "audio_prompt",
63
- "type": "audio_autoencoder_v2",
64
- "config": {
65
- "sample_rate": 44100,
66
- "pretransform_config": {
67
- "type": "autoencoder",
68
- "iterate_batch": true,
69
- "config": {
70
- "encoder": {
71
- "type": "oobleck",
72
- "requires_grad": false,
73
- "config": {
74
- "in_channels": 2,
75
- "channels": 128,
76
- "c_mults": [1, 2, 4, 8, 16],
77
- "strides": [2, 4, 4, 8, 8],
78
- "latent_dim": 128,
79
- "use_snake": true
80
- }
81
- },
82
- "decoder": {
83
- "type": "oobleck",
84
- "config": {
85
- "out_channels": 2,
86
- "channels": 128,
87
- "c_mults": [1, 2, 4, 8, 16],
88
- "strides": [2, 4, 4, 8, 8],
89
- "latent_dim": 64,
90
- "use_snake": true,
91
- "final_tanh": false
92
- }
93
- },
94
- "bottleneck": {
95
- "type": "vae"
96
- },
97
- "latent_dim": 64,
98
- "downsampling_ratio": 2048,
99
- "io_channels": 2
100
- }
101
- },
102
- "pretransform_ckpt_path": "./model/VAE.ckpt",
103
- "latent_seq_len": 215,
104
- "mask_ratio_start": 0,
105
- "mask_ratio_end": 0
106
- }
107
- }
108
- ],
109
- "cond_dim": 768
110
- },
111
- "diffusion": {
112
- "cross_attention_cond_ids": ["video_prompt", "text_prompt", "audio_prompt"],
113
- "global_cond_ids": [],
114
- "type": "mmdit",
115
- "gate": true,
116
- "gate_type": "MAF",
117
- "gate_type_config": {
118
- "num_experts_per_modality": 64,
119
- "num_heads": 24,
120
- "num_fusion_layers": 8
121
  },
122
- "config": {
123
- "io_channels": 64,
124
- "embed_dim": 1536,
125
- "depth": 24,
126
- "num_heads": 24,
127
- "cond_token_dim": 768,
128
- "global_cond_dim": 768,
129
- "project_cond_tokens": false,
130
- "transformer_type": "continuous_transformer",
131
- "video_fps": 5
132
- }
133
- },
134
- "io_channels": 64
135
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  }
 
1
  {
2
+ "model_type": "diffusion_cond",
3
+ "sample_size": 485100,
4
+ "sample_rate": 44100,
5
+ "video_fps": 5,
6
+ "audio_channels": 2,
7
+ "model": {
8
+ "pretransform": {
9
+ "type": "autoencoder",
10
+ "iterate_batch": true,
11
+ "config": {
12
+ "encoder": {
13
+ "type": "oobleck",
14
+ "requires_grad": false,
15
+ "config": {
16
+ "in_channels": 2,
17
+ "channels": 128,
18
+ "c_mults": [
19
+ 1,
20
+ 2,
21
+ 4,
22
+ 8,
23
+ 16
24
+ ],
25
+ "strides": [
26
+ 2,
27
+ 4,
28
+ 4,
29
+ 8,
30
+ 8
31
+ ],
32
+ "latent_dim": 128,
33
+ "use_snake": true
34
+ }
35
+ },
36
+ "decoder": {
37
+ "type": "oobleck",
38
+ "config": {
39
+ "out_channels": 2,
40
+ "channels": 128,
41
+ "c_mults": [
42
+ 1,
43
+ 2,
44
+ 4,
45
+ 8,
46
+ 16
47
+ ],
48
+ "strides": [
49
+ 2,
50
+ 4,
51
+ 4,
52
+ 8,
53
+ 8
54
+ ],
55
+ "latent_dim": 64,
56
+ "use_snake": true,
57
+ "final_tanh": false
58
+ }
59
+ },
60
+ "bottleneck": {
61
+ "type": "vae"
62
+ },
63
+ "latent_dim": 64,
64
+ "downsampling_ratio": 2048,
65
+ "io_channels": 2
66
+ }
67
+ },
68
+ "conditioning": {
69
+ "configs": [
70
+ {
71
+ "id": "video_prompt",
72
+ "type": "clip-with-sync-w-empty-feat",
73
+ "config": {
74
+ "clip_model_name": "openai/clip-vit-base-patch32"
75
+ }
76
+ },
77
+ {
78
+ "id": "text_prompt",
79
+ "type": "t5",
80
+ "config": {
81
+ "t5_model_name": "t5-base",
82
+ "max_length": 128
83
+ }
84
+ },
85
+ {
86
+ "id": "audio_prompt",
87
+ "type": "audio_autoencoder_v2",
88
+ "config": {
89
+ "sample_rate": 44100,
90
+ "pretransform_config": {
91
+ "type": "autoencoder",
92
+ "iterate_batch": true,
93
+ "config": {
94
  "encoder": {
95
+ "type": "oobleck",
96
+ "requires_grad": false,
97
+ "config": {
98
+ "in_channels": 2,
99
+ "channels": 128,
100
+ "c_mults": [
101
+ 1,
102
+ 2,
103
+ 4,
104
+ 8,
105
+ 16
106
+ ],
107
+ "strides": [
108
+ 2,
109
+ 4,
110
+ 4,
111
+ 8,
112
+ 8
113
+ ],
114
+ "latent_dim": 128,
115
+ "use_snake": true
116
+ }
117
  },
118
  "decoder": {
119
+ "type": "oobleck",
120
+ "config": {
121
+ "out_channels": 2,
122
+ "channels": 128,
123
+ "c_mults": [
124
+ 1,
125
+ 2,
126
+ 4,
127
+ 8,
128
+ 16
129
+ ],
130
+ "strides": [
131
+ 2,
132
+ 4,
133
+ 4,
134
+ 8,
135
+ 8
136
+ ],
137
+ "latent_dim": 64,
138
+ "use_snake": true,
139
+ "final_tanh": false
140
+ }
141
  },
142
  "bottleneck": {
143
+ "type": "vae"
144
  },
145
  "latent_dim": 64,
146
  "downsampling_ratio": 2048,
147
  "io_channels": 2
148
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  },
150
+ "pretransform_ckpt_path": "./model/VAE.ckpt",
151
+ "latent_seq_len": 215,
152
+ "mask_ratio_start": 0,
153
+ "mask_ratio_end": 0
154
+ }
155
+ }
156
+ ],
157
+ "cond_dim": 768
158
+ },
159
+ "diffusion": {
160
+ "cross_attention_cond_ids": [
161
+ "video_prompt",
162
+ "text_prompt",
163
+ "audio_prompt"
164
+ ],
165
+ "global_cond_ids": [],
166
+ "type": "mmdit",
167
+ "gate": true,
168
+ "gate_type": "MAF",
169
+ "gate_type_config": {
170
+ "num_experts_per_modality": 64,
171
+ "num_heads": 24,
172
+ "num_fusion_layers": 8
173
+ },
174
+ "config": {
175
+ "io_channels": 64,
176
+ "embed_dim": 1536,
177
+ "depth": 24,
178
+ "num_heads": 24,
179
+ "cond_token_dim": 768,
180
+ "global_cond_dim": 768,
181
+ "project_cond_tokens": false,
182
+ "transformer_type": "continuous_transformer",
183
+ "video_fps": 5
184
+ }
185
+ },
186
+ "io_channels": 64
187
+ }
188
  }
model_index.json CHANGED
@@ -1,9 +1,3 @@
1
  {
2
- "_class_name": "AudioXPipeline",
3
- "config": "config.json",
4
- "weight_layout": "vllm_omni_component_sharded",
5
- "source_checkpoint": "model.ckpt",
6
- "transformer_weights": "transformer/diffusion_pytorch_model.safetensors",
7
- "conditioners_weights": "conditioners/diffusion_pytorch_model.safetensors",
8
- "vae_weights": "vae/diffusion_pytorch_model.safetensors"
9
- }
 
1
  {
2
+ "_class_name": "AudioXPipeline"
3
+ }
 
 
 
 
 
 
transformer/diffusion_pytorch_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:be59e269e46b2973d578708b358e5a1fdad550eedcf31e4354cdae62318fcdde
3
- size 9114498996
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c546d9ecbbb079fe541b099e3e6d6082ddc9dd1506f364ac1a0e93642eee9dc
3
+ size 10218666856