diff --git a/docs/source/reference/launcher.md b/docs/source/reference/launcher.md index 01f15648..c8d2a4c6 100644 --- a/docs/source/reference/launcher.md +++ b/docs/source/reference/launcher.md @@ -55,7 +55,9 @@ Options: ## QUANTIZE ```shell --quantize - Whether you want the model to be quantized + Quantization method to use for the model. It is not necessary to specify this option for pre-quantized models, since the quantization method is read from the model configuration. + + Marlin kernels will be used automatically for GPTQ/AWQ models. [env: QUANTIZE=] diff --git a/flake.nix b/flake.nix index 07348e74..260b2554 100644 --- a/flake.nix +++ b/flake.nix @@ -157,6 +157,7 @@ pyright pytest pytest-asyncio + redocly ruff syrupy ]); diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 2cdccfe0..175244ff 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -367,7 +367,11 @@ struct Args { #[clap(long, env)] num_shard: Option, - /// Whether you want the model to be quantized. + /// Quantization method to use for the model. It is not necessary to specify this option + /// for pre-quantized models, since the quantization method is read from the model + /// configuration. + /// + /// Marlin kernels will be used automatically for GPTQ/AWQ models. #[clap(long, env, value_enum)] quantize: Option,