preemo_text-generation-infe.../aml/deployment.yaml

39 lines
976 B
YAML
Raw Normal View History

2022-10-15 12:21:50 -06:00
$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json
name: bloom-deployment
endpoint_name: bloom-inference
2022-10-17 02:39:59 -06:00
model: azureml:bloom:1
2022-10-15 12:21:50 -06:00
model_mount_path: /var/azureml-model
environment_variables:
MODEL_BASE_PATH: /var/azureml-model/bloom
MODEL_ID: bigscience/bloom
2022-10-15 12:21:50 -06:00
NUM_GPUS: 8
environment:
image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1
2022-10-15 12:21:50 -06:00
inference_config:
liveness_route:
port: 3000
path: /health
readiness_route:
port: 3000
path: /health
scoring_route:
port: 3000
path: /generate
instance_type: Standard_ND96amsr_A100_v4
request_settings:
request_timeout_ms: 90000
2022-10-17 02:39:59 -06:00
max_concurrent_requests_per_instance: 256
2022-10-15 12:21:50 -06:00
liveness_probe:
2022-10-17 06:59:00 -06:00
initial_delay: 600
timeout: 90
2022-10-17 06:59:00 -06:00
period: 120
2022-10-15 12:21:50 -06:00
success_threshold: 1
failure_threshold: 5
2022-10-15 12:21:50 -06:00
readiness_probe:
2022-10-17 06:59:00 -06:00
initial_delay: 600
timeout: 90
2022-10-17 06:59:00 -06:00
period: 120
2022-10-15 12:21:50 -06:00
success_threshold: 1
failure_threshold: 5
2022-10-15 12:21:50 -06:00
instance_count: 1