2119 lines
57 KiB
JSON
2119 lines
57 KiB
JSON
{
|
|
"openapi": "3.0.3",
|
|
"info": {
|
|
"title": "Text Generation Inference",
|
|
"description": "Text Generation Webserver",
|
|
"contact": {
|
|
"name": "Olivier Dehaene"
|
|
},
|
|
"license": {
|
|
"name": "Apache 2.0",
|
|
"url": "https://www.apache.org/licenses/LICENSE-2.0"
|
|
},
|
|
"version": "2.2.1-dev0"
|
|
},
|
|
"paths": {
|
|
"/": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`",
|
|
"operationId": "compat_generate",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/CompatGenerateRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Generated Text",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/GenerateResponse"
|
|
}
|
|
},
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/StreamResponse"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"422": {
|
|
"description": "Input validation error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Input validation error"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"424": {
|
|
"description": "Generation Error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Request failed during generation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"429": {
|
|
"description": "Model is overloaded",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Model is overloaded"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"500": {
|
|
"description": "Incomplete generation",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Incomplete generation"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/generate": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Generate tokens",
|
|
"operationId": "generate",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/GenerateRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Generated Text",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/GenerateResponse"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"422": {
|
|
"description": "Input validation error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Input validation error"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"424": {
|
|
"description": "Generation Error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Request failed during generation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"429": {
|
|
"description": "Model is overloaded",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Model is overloaded"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"500": {
|
|
"description": "Incomplete generation",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Incomplete generation"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/generate_stream": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Generate a stream of token using Server-Sent Events",
|
|
"operationId": "generate_stream",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/GenerateRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Generated Text",
|
|
"content": {
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/StreamResponse"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"422": {
|
|
"description": "Input validation error",
|
|
"content": {
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Input validation error"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"424": {
|
|
"description": "Generation Error",
|
|
"content": {
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Request failed during generation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"429": {
|
|
"description": "Model is overloaded",
|
|
"content": {
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Model is overloaded"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"500": {
|
|
"description": "Incomplete generation",
|
|
"content": {
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Incomplete generation"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/health": {
|
|
"get": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Health check method",
|
|
"operationId": "health",
|
|
"responses": {
|
|
"200": {
|
|
"description": "Everything is working fine"
|
|
},
|
|
"503": {
|
|
"description": "Text generation inference is down",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "unhealthy",
|
|
"error_type": "healthcheck"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/info": {
|
|
"get": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Text Generation Inference endpoint info",
|
|
"operationId": "get_model_info",
|
|
"responses": {
|
|
"200": {
|
|
"description": "Served model info",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/Info"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/metrics": {
|
|
"get": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Prometheus metrics scrape endpoint",
|
|
"operationId": "metrics",
|
|
"responses": {
|
|
"200": {
|
|
"description": "Prometheus Metrics",
|
|
"content": {
|
|
"text/plain": {
|
|
"schema": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/tokenize": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Tokenize inputs",
|
|
"operationId": "tokenize",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/GenerateRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Tokenized ids",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/TokenizeResponse"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"404": {
|
|
"description": "No tokenizer found",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "No fast tokenizer available"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/v1/chat/completions": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Generate tokens",
|
|
"operationId": "chat_completions",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ChatRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Generated Chat Completion",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ChatCompletion"
|
|
}
|
|
},
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ChatCompletionChunk"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"422": {
|
|
"description": "Input validation error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Input validation error"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"424": {
|
|
"description": "Generation Error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Request failed during generation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"429": {
|
|
"description": "Model is overloaded",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Model is overloaded"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"500": {
|
|
"description": "Incomplete generation",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Incomplete generation"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"/v1/completions": {
|
|
"post": {
|
|
"tags": [
|
|
"Text Generation Inference"
|
|
],
|
|
"summary": "Generate tokens",
|
|
"operationId": "completions",
|
|
"requestBody": {
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/CompletionRequest"
|
|
}
|
|
}
|
|
},
|
|
"required": true
|
|
},
|
|
"responses": {
|
|
"200": {
|
|
"description": "Generated Chat Completion",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/CompletionFinal"
|
|
}
|
|
},
|
|
"text/event-stream": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/Chunk"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"422": {
|
|
"description": "Input validation error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Input validation error"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"424": {
|
|
"description": "Generation Error",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Request failed during generation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"429": {
|
|
"description": "Model is overloaded",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Model is overloaded"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"500": {
|
|
"description": "Incomplete generation",
|
|
"content": {
|
|
"application/json": {
|
|
"schema": {
|
|
"$ref": "#/components/schemas/ErrorResponse"
|
|
},
|
|
"example": {
|
|
"error": "Incomplete generation"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"components": {
|
|
"schemas": {
|
|
"BestOfSequence": {
|
|
"type": "object",
|
|
"required": [
|
|
"generated_text",
|
|
"finish_reason",
|
|
"generated_tokens",
|
|
"prefill",
|
|
"tokens"
|
|
],
|
|
"properties": {
|
|
"finish_reason": {
|
|
"$ref": "#/components/schemas/FinishReason"
|
|
},
|
|
"generated_text": {
|
|
"type": "string",
|
|
"example": "test"
|
|
},
|
|
"generated_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 1,
|
|
"minimum": 0
|
|
},
|
|
"prefill": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/PrefillToken"
|
|
}
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": 42,
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Token"
|
|
}
|
|
},
|
|
"top_tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Token"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletion": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"created",
|
|
"model",
|
|
"system_fingerprint",
|
|
"choices",
|
|
"usage"
|
|
],
|
|
"properties": {
|
|
"choices": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/ChatCompletionComplete"
|
|
}
|
|
},
|
|
"created": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": "1706270835",
|
|
"minimum": 0
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "string",
|
|
"example": "mistralai/Mistral-7B-Instruct-v0.2"
|
|
},
|
|
"system_fingerprint": {
|
|
"type": "string"
|
|
},
|
|
"usage": {
|
|
"$ref": "#/components/schemas/Usage"
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionChoice": {
|
|
"type": "object",
|
|
"required": [
|
|
"index",
|
|
"delta"
|
|
],
|
|
"properties": {
|
|
"delta": {
|
|
"$ref": "#/components/schemas/ChatCompletionDelta"
|
|
},
|
|
"finish_reason": {
|
|
"type": "string",
|
|
"nullable": true
|
|
},
|
|
"index": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"logprobs": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/ChatCompletionLogprobs"
|
|
}
|
|
],
|
|
"nullable": true
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionChunk": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"created",
|
|
"model",
|
|
"system_fingerprint",
|
|
"choices"
|
|
],
|
|
"properties": {
|
|
"choices": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/ChatCompletionChoice"
|
|
}
|
|
},
|
|
"created": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": "1706270978",
|
|
"minimum": 0
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "string",
|
|
"example": "mistralai/Mistral-7B-Instruct-v0.2"
|
|
},
|
|
"system_fingerprint": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionComplete": {
|
|
"type": "object",
|
|
"required": [
|
|
"index",
|
|
"message",
|
|
"finish_reason"
|
|
],
|
|
"properties": {
|
|
"finish_reason": {
|
|
"type": "string"
|
|
},
|
|
"index": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"logprobs": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/ChatCompletionLogprobs"
|
|
}
|
|
],
|
|
"nullable": true
|
|
},
|
|
"message": {
|
|
"$ref": "#/components/schemas/OutputMessage"
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionDelta": {
|
|
"oneOf": [
|
|
{
|
|
"$ref": "#/components/schemas/TextMessage"
|
|
},
|
|
{
|
|
"$ref": "#/components/schemas/ToolCallDelta"
|
|
}
|
|
]
|
|
},
|
|
"ChatCompletionLogprob": {
|
|
"type": "object",
|
|
"required": [
|
|
"token",
|
|
"logprob",
|
|
"top_logprobs"
|
|
],
|
|
"properties": {
|
|
"logprob": {
|
|
"type": "number",
|
|
"format": "float"
|
|
},
|
|
"token": {
|
|
"type": "string"
|
|
},
|
|
"top_logprobs": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/ChatCompletionTopLogprob"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionLogprobs": {
|
|
"type": "object",
|
|
"required": [
|
|
"content"
|
|
],
|
|
"properties": {
|
|
"content": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/ChatCompletionLogprob"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ChatCompletionTopLogprob": {
|
|
"type": "object",
|
|
"required": [
|
|
"token",
|
|
"logprob"
|
|
],
|
|
"properties": {
|
|
"logprob": {
|
|
"type": "number",
|
|
"format": "float"
|
|
},
|
|
"token": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"ChatRequest": {
|
|
"type": "object",
|
|
"required": [
|
|
"messages"
|
|
],
|
|
"properties": {
|
|
"frequency_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
|
|
"example": "1.0",
|
|
"nullable": true
|
|
},
|
|
"logit_bias": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "number",
|
|
"format": "float"
|
|
},
|
|
"description": "UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.",
|
|
"nullable": true
|
|
},
|
|
"logprobs": {
|
|
"type": "boolean",
|
|
"description": "Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.",
|
|
"example": "false",
|
|
"nullable": true
|
|
},
|
|
"max_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "The maximum number of tokens that can be generated in the chat completion.",
|
|
"example": "32",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"messages": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Message"
|
|
},
|
|
"description": "A list of messages comprising the conversation so far.",
|
|
"example": "[{\"role\": \"user\", \"content\": \"What is Deep Learning?\"}]"
|
|
},
|
|
"model": {
|
|
"type": "string",
|
|
"description": "[UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
|
|
"example": "mistralai/Mistral-7B-Instruct-v0.2",
|
|
"nullable": true
|
|
},
|
|
"n": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.",
|
|
"example": "2",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"presence_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics",
|
|
"example": 0.1,
|
|
"nullable": true
|
|
},
|
|
"response_format": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/GrammarType"
|
|
}
|
|
],
|
|
"default": "null",
|
|
"nullable": true
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": 42,
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"stop": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Up to 4 sequences where the API will stop generating further tokens.",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"stream": {
|
|
"type": "boolean"
|
|
},
|
|
"temperature": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.",
|
|
"example": 1.0,
|
|
"nullable": true
|
|
},
|
|
"tool_choice": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/ToolChoice"
|
|
}
|
|
],
|
|
"nullable": true
|
|
},
|
|
"tool_prompt": {
|
|
"type": "string",
|
|
"description": "A prompt to be appended before the tools",
|
|
"example": "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\"",
|
|
"nullable": true
|
|
},
|
|
"tools": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Tool"
|
|
},
|
|
"description": "A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of\nfunctions the model may generate JSON inputs for.",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"top_logprobs": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.",
|
|
"example": "5",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"top_p": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.",
|
|
"example": 0.95,
|
|
"nullable": true
|
|
}
|
|
}
|
|
},
|
|
"Chunk": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"created",
|
|
"choices",
|
|
"model",
|
|
"system_fingerprint"
|
|
],
|
|
"properties": {
|
|
"choices": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/CompletionComplete"
|
|
}
|
|
},
|
|
"created": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"minimum": 0
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "string"
|
|
},
|
|
"system_fingerprint": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"CompatGenerateRequest": {
|
|
"type": "object",
|
|
"required": [
|
|
"inputs"
|
|
],
|
|
"properties": {
|
|
"inputs": {
|
|
"type": "string",
|
|
"example": "My name is Olivier and I"
|
|
},
|
|
"parameters": {
|
|
"$ref": "#/components/schemas/GenerateParameters"
|
|
},
|
|
"stream": {
|
|
"type": "boolean",
|
|
"default": "false"
|
|
}
|
|
}
|
|
},
|
|
"Completion": {
|
|
"oneOf": [
|
|
{
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/Chunk"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"object"
|
|
],
|
|
"properties": {
|
|
"object": {
|
|
"type": "string",
|
|
"enum": [
|
|
"text_completion"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/CompletionFinal"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"object"
|
|
],
|
|
"properties": {
|
|
"object": {
|
|
"type": "string",
|
|
"enum": [
|
|
"text_completion"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"discriminator": {
|
|
"propertyName": "object"
|
|
}
|
|
},
|
|
"CompletionComplete": {
|
|
"type": "object",
|
|
"required": [
|
|
"index",
|
|
"text",
|
|
"finish_reason"
|
|
],
|
|
"properties": {
|
|
"finish_reason": {
|
|
"type": "string"
|
|
},
|
|
"index": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"logprobs": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "number",
|
|
"format": "float"
|
|
},
|
|
"nullable": true
|
|
},
|
|
"text": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"CompletionFinal": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"created",
|
|
"model",
|
|
"system_fingerprint",
|
|
"choices",
|
|
"usage"
|
|
],
|
|
"properties": {
|
|
"choices": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/CompletionComplete"
|
|
}
|
|
},
|
|
"created": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": "1706270835",
|
|
"minimum": 0
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "string",
|
|
"example": "mistralai/Mistral-7B-Instruct-v0.2"
|
|
},
|
|
"system_fingerprint": {
|
|
"type": "string"
|
|
},
|
|
"usage": {
|
|
"$ref": "#/components/schemas/Usage"
|
|
}
|
|
}
|
|
},
|
|
"CompletionRequest": {
|
|
"type": "object",
|
|
"required": [
|
|
"prompt"
|
|
],
|
|
"properties": {
|
|
"frequency_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
|
|
"example": "1.0",
|
|
"nullable": true
|
|
},
|
|
"max_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "The maximum number of tokens that can be generated in the chat completion.",
|
|
"default": "32",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"model": {
|
|
"type": "string",
|
|
"description": "UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
|
|
"example": "mistralai/Mistral-7B-Instruct-v0.2",
|
|
"nullable": true
|
|
},
|
|
"prompt": {
|
|
"$ref": "#/components/schemas/Prompt"
|
|
},
|
|
"repetition_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"nullable": true
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": 42,
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"stop": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Up to 4 sequences where the API will stop generating further tokens.",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"stream": {
|
|
"type": "boolean"
|
|
},
|
|
"suffix": {
|
|
"type": "string",
|
|
"description": "The text to append to the prompt. This is useful for completing sentences or generating a paragraph of text.\nplease see the completion_template field in the model's tokenizer_config.json file for completion template.",
|
|
"nullable": true
|
|
},
|
|
"temperature": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both.",
|
|
"example": 1.0,
|
|
"nullable": true
|
|
},
|
|
"top_p": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.",
|
|
"example": 0.95,
|
|
"nullable": true
|
|
}
|
|
}
|
|
},
|
|
"DeltaToolCall": {
|
|
"type": "object",
|
|
"required": [
|
|
"index",
|
|
"id",
|
|
"type",
|
|
"function"
|
|
],
|
|
"properties": {
|
|
"function": {
|
|
"$ref": "#/components/schemas/Function"
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"index": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"type": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"Details": {
|
|
"type": "object",
|
|
"required": [
|
|
"finish_reason",
|
|
"generated_tokens",
|
|
"prefill",
|
|
"tokens"
|
|
],
|
|
"properties": {
|
|
"best_of_sequences": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/BestOfSequence"
|
|
},
|
|
"nullable": true
|
|
},
|
|
"finish_reason": {
|
|
"$ref": "#/components/schemas/FinishReason"
|
|
},
|
|
"generated_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 1,
|
|
"minimum": 0
|
|
},
|
|
"prefill": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/PrefillToken"
|
|
}
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": 42,
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Token"
|
|
}
|
|
},
|
|
"top_tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Token"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ErrorResponse": {
|
|
"type": "object",
|
|
"required": [
|
|
"error",
|
|
"error_type"
|
|
],
|
|
"properties": {
|
|
"error": {
|
|
"type": "string"
|
|
},
|
|
"error_type": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"FinishReason": {
|
|
"type": "string",
|
|
"enum": [
|
|
"length",
|
|
"eos_token",
|
|
"stop_sequence"
|
|
],
|
|
"example": "Length"
|
|
},
|
|
"Function": {
|
|
"type": "object",
|
|
"required": [
|
|
"arguments"
|
|
],
|
|
"properties": {
|
|
"arguments": {
|
|
"type": "string"
|
|
},
|
|
"name": {
|
|
"type": "string",
|
|
"nullable": true
|
|
}
|
|
}
|
|
},
|
|
"FunctionDefinition": {
|
|
"type": "object",
|
|
"required": [
|
|
"name",
|
|
"arguments"
|
|
],
|
|
"properties": {
|
|
"arguments": {},
|
|
"description": {
|
|
"type": "string",
|
|
"nullable": true
|
|
},
|
|
"name": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"FunctionName": {
|
|
"type": "object",
|
|
"required": [
|
|
"name"
|
|
],
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"GenerateParameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"adapter_id": {
|
|
"type": "string",
|
|
"description": "Lora adapter id",
|
|
"default": "null",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"best_of": {
|
|
"type": "integer",
|
|
"description": "Generate best_of sequences and return the one if the highest token logprobs.",
|
|
"default": "null",
|
|
"example": 1,
|
|
"nullable": true,
|
|
"minimum": 0,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"decoder_input_details": {
|
|
"type": "boolean",
|
|
"description": "Whether to return decoder input token logprobs and ids.",
|
|
"default": "false"
|
|
},
|
|
"details": {
|
|
"type": "boolean",
|
|
"description": "Whether to return generation details.",
|
|
"default": "true"
|
|
},
|
|
"do_sample": {
|
|
"type": "boolean",
|
|
"description": "Activate logits sampling.",
|
|
"default": "false",
|
|
"example": true
|
|
},
|
|
"frequency_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "The parameter for frequency penalty. 1.0 means no penalty\nPenalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
|
|
"default": "null",
|
|
"example": 0.1,
|
|
"nullable": true,
|
|
"exclusiveMinimum": -2
|
|
},
|
|
"grammar": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/GrammarType"
|
|
}
|
|
],
|
|
"default": "null",
|
|
"nullable": true
|
|
},
|
|
"max_new_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "Maximum number of tokens to generate.",
|
|
"default": "100",
|
|
"example": "20",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"repetition_penalty": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "The parameter for repetition penalty. 1.0 means no penalty.\nSee [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.",
|
|
"default": "null",
|
|
"example": 1.03,
|
|
"nullable": true,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"return_full_text": {
|
|
"type": "boolean",
|
|
"description": "Whether to prepend the prompt to the generated text",
|
|
"default": "null",
|
|
"example": false,
|
|
"nullable": true
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"description": "Random sampling seed.",
|
|
"default": "null",
|
|
"example": "null",
|
|
"nullable": true,
|
|
"minimum": 0,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"stop": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Stop generating tokens if a member of `stop` is generated.",
|
|
"example": [
|
|
"photographer"
|
|
],
|
|
"maxItems": 4
|
|
},
|
|
"temperature": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "The value used to module the logits distribution.",
|
|
"default": "null",
|
|
"example": 0.5,
|
|
"nullable": true,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"top_k": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "The number of highest probability vocabulary tokens to keep for top-k-filtering.",
|
|
"default": "null",
|
|
"example": 10,
|
|
"nullable": true,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"top_n_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"description": "The number of highest probability vocabulary tokens to keep for top-n-filtering.",
|
|
"default": "null",
|
|
"example": 5,
|
|
"nullable": true,
|
|
"minimum": 0,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"top_p": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "Top-p value for nucleus sampling.",
|
|
"default": "null",
|
|
"example": 0.95,
|
|
"nullable": true,
|
|
"maximum": 1,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"truncate": {
|
|
"type": "integer",
|
|
"description": "Truncate inputs tokens to the given size.",
|
|
"default": "null",
|
|
"example": "null",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"typical_p": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"description": "Typical Decoding mass\nSee [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information.",
|
|
"default": "null",
|
|
"example": 0.95,
|
|
"nullable": true,
|
|
"maximum": 1,
|
|
"exclusiveMinimum": 0
|
|
},
|
|
"watermark": {
|
|
"type": "boolean",
|
|
"description": "Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226).",
|
|
"default": "false",
|
|
"example": true
|
|
}
|
|
}
|
|
},
|
|
"GenerateRequest": {
|
|
"type": "object",
|
|
"required": [
|
|
"inputs"
|
|
],
|
|
"properties": {
|
|
"inputs": {
|
|
"type": "string",
|
|
"example": "My name is Olivier and I"
|
|
},
|
|
"parameters": {
|
|
"$ref": "#/components/schemas/GenerateParameters"
|
|
}
|
|
}
|
|
},
|
|
"GenerateResponse": {
|
|
"type": "object",
|
|
"required": [
|
|
"generated_text"
|
|
],
|
|
"properties": {
|
|
"details": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/Details"
|
|
}
|
|
],
|
|
"nullable": true
|
|
},
|
|
"generated_text": {
|
|
"type": "string",
|
|
"example": "test"
|
|
}
|
|
}
|
|
},
|
|
"GrammarType": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"type",
|
|
"value"
|
|
],
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"json"
|
|
]
|
|
},
|
|
"value": {
|
|
"description": "A string that represents a [JSON Schema](https://json-schema.org/).\n\nJSON Schema is a declarative language that allows to annotate JSON documents\nwith types and descriptions."
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"type",
|
|
"value"
|
|
],
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"regex"
|
|
]
|
|
},
|
|
"value": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"discriminator": {
|
|
"propertyName": "type"
|
|
}
|
|
},
|
|
"Info": {
|
|
"type": "object",
|
|
"required": [
|
|
"model_id",
|
|
"model_dtype",
|
|
"model_device_type",
|
|
"max_concurrent_requests",
|
|
"max_best_of",
|
|
"max_stop_sequences",
|
|
"max_input_tokens",
|
|
"max_total_tokens",
|
|
"waiting_served_ratio",
|
|
"max_batch_total_tokens",
|
|
"max_waiting_tokens",
|
|
"validation_workers",
|
|
"max_client_batch_size",
|
|
"router",
|
|
"version"
|
|
],
|
|
"properties": {
|
|
"docker_label": {
|
|
"type": "string",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"max_batch_size": {
|
|
"type": "integer",
|
|
"example": "null",
|
|
"nullable": true,
|
|
"minimum": 0
|
|
},
|
|
"max_batch_total_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": "32000",
|
|
"minimum": 0
|
|
},
|
|
"max_best_of": {
|
|
"type": "integer",
|
|
"example": "2",
|
|
"minimum": 0
|
|
},
|
|
"max_client_batch_size": {
|
|
"type": "integer",
|
|
"example": "32",
|
|
"minimum": 0
|
|
},
|
|
"max_concurrent_requests": {
|
|
"type": "integer",
|
|
"description": "Router Parameters",
|
|
"example": "128",
|
|
"minimum": 0
|
|
},
|
|
"max_input_tokens": {
|
|
"type": "integer",
|
|
"example": "1024",
|
|
"minimum": 0
|
|
},
|
|
"max_stop_sequences": {
|
|
"type": "integer",
|
|
"example": "4",
|
|
"minimum": 0
|
|
},
|
|
"max_total_tokens": {
|
|
"type": "integer",
|
|
"example": "2048",
|
|
"minimum": 0
|
|
},
|
|
"max_waiting_tokens": {
|
|
"type": "integer",
|
|
"example": "20",
|
|
"minimum": 0
|
|
},
|
|
"model_device_type": {
|
|
"type": "string",
|
|
"example": "cuda"
|
|
},
|
|
"model_dtype": {
|
|
"type": "string",
|
|
"example": "torch.float16"
|
|
},
|
|
"model_id": {
|
|
"type": "string",
|
|
"description": "Model info",
|
|
"example": "bigscience/blomm-560m"
|
|
},
|
|
"model_pipeline_tag": {
|
|
"type": "string",
|
|
"example": "text-generation",
|
|
"nullable": true
|
|
},
|
|
"model_sha": {
|
|
"type": "string",
|
|
"example": "e985a63cdc139290c5f700ff1929f0b5942cced2",
|
|
"nullable": true
|
|
},
|
|
"router": {
|
|
"type": "string",
|
|
"description": "Router Info",
|
|
"example": "text-generation-router"
|
|
},
|
|
"sha": {
|
|
"type": "string",
|
|
"example": "null",
|
|
"nullable": true
|
|
},
|
|
"validation_workers": {
|
|
"type": "integer",
|
|
"example": "2",
|
|
"minimum": 0
|
|
},
|
|
"version": {
|
|
"type": "string",
|
|
"example": "0.5.0"
|
|
},
|
|
"waiting_served_ratio": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"example": "1.2"
|
|
}
|
|
}
|
|
},
|
|
"Message": {
|
|
"type": "object",
|
|
"required": [
|
|
"role",
|
|
"content"
|
|
],
|
|
"properties": {
|
|
"content": {
|
|
"$ref": "#/components/schemas/MessageContent"
|
|
},
|
|
"name": {
|
|
"type": "string",
|
|
"example": "\"David\"",
|
|
"nullable": true
|
|
},
|
|
"role": {
|
|
"type": "string",
|
|
"example": "user"
|
|
}
|
|
}
|
|
},
|
|
"MessageChunk": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"text"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"image_url",
|
|
"type"
|
|
],
|
|
"properties": {
|
|
"image_url": {
|
|
"$ref": "#/components/schemas/Url"
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"image_url"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"discriminator": {
|
|
"propertyName": "type"
|
|
}
|
|
},
|
|
"MessageContent": {
|
|
"oneOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/MessageChunk"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"OutputMessage": {
|
|
"oneOf": [
|
|
{
|
|
"$ref": "#/components/schemas/TextMessage"
|
|
},
|
|
{
|
|
"$ref": "#/components/schemas/ToolCallMessage"
|
|
}
|
|
]
|
|
},
|
|
"PrefillToken": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"text",
|
|
"logprob"
|
|
],
|
|
"properties": {
|
|
"id": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 0,
|
|
"minimum": 0
|
|
},
|
|
"logprob": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"example": -0.34,
|
|
"nullable": true
|
|
},
|
|
"text": {
|
|
"type": "string",
|
|
"example": "test"
|
|
}
|
|
}
|
|
},
|
|
"Prompt": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"SimpleToken": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"text",
|
|
"start",
|
|
"stop"
|
|
],
|
|
"properties": {
|
|
"id": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 0,
|
|
"minimum": 0
|
|
},
|
|
"start": {
|
|
"type": "integer",
|
|
"example": 0,
|
|
"minimum": 0
|
|
},
|
|
"stop": {
|
|
"type": "integer",
|
|
"example": 2,
|
|
"minimum": 0
|
|
},
|
|
"text": {
|
|
"type": "string",
|
|
"example": "test"
|
|
}
|
|
}
|
|
},
|
|
"StreamDetails": {
|
|
"type": "object",
|
|
"required": [
|
|
"finish_reason",
|
|
"generated_tokens"
|
|
],
|
|
"properties": {
|
|
"finish_reason": {
|
|
"$ref": "#/components/schemas/FinishReason"
|
|
},
|
|
"generated_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 1,
|
|
"minimum": 0
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"format": "int64",
|
|
"example": 42,
|
|
"nullable": true,
|
|
"minimum": 0
|
|
}
|
|
}
|
|
},
|
|
"StreamResponse": {
|
|
"type": "object",
|
|
"required": [
|
|
"index",
|
|
"token"
|
|
],
|
|
"properties": {
|
|
"details": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/StreamDetails"
|
|
}
|
|
],
|
|
"default": "null",
|
|
"nullable": true
|
|
},
|
|
"generated_text": {
|
|
"type": "string",
|
|
"default": "null",
|
|
"example": "test",
|
|
"nullable": true
|
|
},
|
|
"index": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"token": {
|
|
"$ref": "#/components/schemas/Token"
|
|
},
|
|
"top_tokens": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/Token"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"TextMessage": {
|
|
"type": "object",
|
|
"required": [
|
|
"role",
|
|
"content"
|
|
],
|
|
"properties": {
|
|
"content": {
|
|
"type": "string",
|
|
"example": "My name is David and I"
|
|
},
|
|
"role": {
|
|
"type": "string",
|
|
"example": "user"
|
|
}
|
|
}
|
|
},
|
|
"Token": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"text",
|
|
"logprob",
|
|
"special"
|
|
],
|
|
"properties": {
|
|
"id": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"example": 0,
|
|
"minimum": 0
|
|
},
|
|
"logprob": {
|
|
"type": "number",
|
|
"format": "float",
|
|
"example": -0.34,
|
|
"nullable": true
|
|
},
|
|
"special": {
|
|
"type": "boolean",
|
|
"example": "false"
|
|
},
|
|
"text": {
|
|
"type": "string",
|
|
"example": "test"
|
|
}
|
|
}
|
|
},
|
|
"TokenizeResponse": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/SimpleToken"
|
|
}
|
|
},
|
|
"Tool": {
|
|
"type": "object",
|
|
"required": [
|
|
"type",
|
|
"function"
|
|
],
|
|
"properties": {
|
|
"function": {
|
|
"$ref": "#/components/schemas/FunctionDefinition"
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"example": "function"
|
|
}
|
|
}
|
|
},
|
|
"ToolCall": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"type",
|
|
"function"
|
|
],
|
|
"properties": {
|
|
"function": {
|
|
"$ref": "#/components/schemas/FunctionDefinition"
|
|
},
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"ToolCallDelta": {
|
|
"type": "object",
|
|
"required": [
|
|
"role",
|
|
"tool_calls"
|
|
],
|
|
"properties": {
|
|
"role": {
|
|
"type": "string",
|
|
"example": "assistant"
|
|
},
|
|
"tool_calls": {
|
|
"$ref": "#/components/schemas/DeltaToolCall"
|
|
}
|
|
}
|
|
},
|
|
"ToolCallMessage": {
|
|
"type": "object",
|
|
"required": [
|
|
"role",
|
|
"tool_calls"
|
|
],
|
|
"properties": {
|
|
"role": {
|
|
"type": "string",
|
|
"example": "assistant"
|
|
},
|
|
"tool_calls": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/components/schemas/ToolCall"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ToolChoice": {
|
|
"allOf": [
|
|
{
|
|
"$ref": "#/components/schemas/ToolType"
|
|
}
|
|
],
|
|
"nullable": true
|
|
},
|
|
"ToolType": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"default": null,
|
|
"nullable": true
|
|
},
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"type": "object",
|
|
"required": [
|
|
"function"
|
|
],
|
|
"properties": {
|
|
"function": {
|
|
"$ref": "#/components/schemas/FunctionName"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"default": null,
|
|
"nullable": true
|
|
}
|
|
]
|
|
},
|
|
"Url": {
|
|
"type": "object",
|
|
"required": [
|
|
"url"
|
|
],
|
|
"properties": {
|
|
"url": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"Usage": {
|
|
"type": "object",
|
|
"required": [
|
|
"prompt_tokens",
|
|
"completion_tokens",
|
|
"total_tokens"
|
|
],
|
|
"properties": {
|
|
"completion_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"prompt_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
},
|
|
"total_tokens": {
|
|
"type": "integer",
|
|
"format": "int32",
|
|
"minimum": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"tags": [
|
|
{
|
|
"name": "Text Generation Inference",
|
|
"description": "Hugging Face Text Generation Inference API"
|
|
}
|
|
]
|
|
}
|