Alikestocode commited on
Commit
ecf6a69
Β·
1 Parent(s): 022b2da

Fix QuantizationConfig: use config_groups with BaseQuantizationConfig

Browse files

- QuantizationConfig requires config_groups dict, not direct parameters
- Use BaseQuantizationConfig for default group with num_bits=4
- Fixes missing config_groups field error

Files changed (1) hide show
  1. quantize_to_awq_colab.ipynb +11 -5
quantize_to_awq_colab.ipynb CHANGED
@@ -254,15 +254,21 @@
254
  " print(f\" β†’ Starting quantization with LLM Compressor...\")\n",
255
  " print(f\" β†’ This may take 30-60 minutes depending on model size...\")\n",
256
  " \n",
257
- " # AWQModifier requires quantization_config with num_bits\n",
258
  " # Create quantization config for 4-bit AWQ\n",
259
- " from compressed_tensors.quantization import QuantizationConfig\n",
260
  " \n",
261
  " print(f\" β†’ Creating quantization config for 4-bit AWQ...\")\n",
 
 
262
  " quant_config = QuantizationConfig(\n",
263
- " num_bits=4, # 4-bit quantization\n",
264
- " group_size=128, # Group size\n",
265
- " zero_point=True # Zero-point quantization\n",
 
 
 
 
266
  " )\n",
267
  " \n",
268
  " print(f\" β†’ Creating AWQModifier with quantization config...\")\n",
 
254
  " print(f\" β†’ Starting quantization with LLM Compressor...\")\n",
255
  " print(f\" β†’ This may take 30-60 minutes depending on model size...\")\n",
256
  " \n",
257
+ " # AWQModifier requires quantization_config with config_groups\n",
258
  " # Create quantization config for 4-bit AWQ\n",
259
+ " from compressed_tensors.quantization import QuantizationConfig, BaseQuantizationConfig\n",
260
  " \n",
261
  " print(f\" β†’ Creating quantization config for 4-bit AWQ...\")\n",
262
+ " # QuantizationConfig requires config_groups - a dict mapping layer names to configs\n",
263
+ " # For AWQ, we use a default config group that applies to all layers\n",
264
  " quant_config = QuantizationConfig(\n",
265
+ " config_groups={\n",
266
+ " \"default\": BaseQuantizationConfig(\n",
267
+ " num_bits=4, # 4-bit quantization\n",
268
+ " group_size=128, # Group size\n",
269
+ " zero_point=True # Zero-point quantization\n",
270
+ " )\n",
271
+ " }\n",
272
  " )\n",
273
  " \n",
274
  " print(f\" β†’ Creating AWQModifier with quantization config...\")\n",