This repository has been archived on 2024-10-27. You can view files and clone it, but cannot push or open issues or pull requests.
local-llm-server/llm_server/llm/info.py

38 lines
1.3 KiB
Python
Raw Normal View History

import requests
2024-05-07 12:20:53 -06:00
from llm_server.config.global_config import GlobalConfig
def get_running_model(backend_url: str, mode: str):
if mode == 'ooba':
try:
2024-05-07 12:20:53 -06:00
backend_response = requests.get(f'{backend_url}/api/v1/model', timeout=GlobalConfig.get().backend_request_timeout, verify=GlobalConfig.get().verify_ssl)
r_json = backend_response.json()
return r_json['result'], None
except Exception as e:
2023-08-23 16:02:57 -06:00
return False, e
elif mode == 'vllm':
2023-09-11 20:47:19 -06:00
try:
2024-05-07 12:20:53 -06:00
backend_response = requests.get(f'{backend_url}/model', timeout=GlobalConfig.get().backend_request_timeout, verify=GlobalConfig.get().verify_ssl)
2023-09-11 20:47:19 -06:00
r_json = backend_response.json()
return r_json['model'], None
2023-09-11 20:47:19 -06:00
except Exception as e:
return False, e
else:
raise Exception
def get_info(backend_url: str, mode: str):
if mode == 'ooba':
return {}
# raise NotImplementedError
elif mode == 'vllm':
try:
2024-05-07 12:20:53 -06:00
r = requests.get(f'{backend_url}/info', verify=GlobalConfig.get().verify_ssl, timeout=GlobalConfig.get().backend_request_timeout)
j = r.json()
except Exception as e:
return {}
return j
else:
raise Exception