107 lines
2.9 KiB
Python
Executable File
107 lines
2.9 KiB
Python
Executable File
import asyncio
|
|
import json
|
|
import sys
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import websockets
|
|
except ImportError:
|
|
print("Websockets package not found. Make sure it's installed.")
|
|
|
|
script_path = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
|
|
def parse_bash_config(file_path):
|
|
config = {}
|
|
with open(file_path, 'r') as f:
|
|
for line in f:
|
|
if line.startswith('#') or '=' not in line:
|
|
continue
|
|
key, value = line.strip().split('=', 1)
|
|
if value.startswith('"') and value.endswith('"'):
|
|
value = value[1:-1]
|
|
elif value.startswith('(') and value.endswith(')'):
|
|
value = value[1:-1].split()
|
|
value = [v.strip('"') for v in value]
|
|
config[key] = value
|
|
return config
|
|
|
|
|
|
config = parse_bash_config(Path(script_path, 'config.sh'))
|
|
|
|
|
|
async def run(context):
|
|
request = {
|
|
'prompt': context,
|
|
'max_new_tokens': 250,
|
|
'auto_max_new_tokens': False,
|
|
'max_tokens_second': 0,
|
|
'preset': 'None',
|
|
'do_sample': True,
|
|
'temperature': 0.7,
|
|
'top_p': 0.1,
|
|
'typical_p': 1,
|
|
'epsilon_cutoff': 0,
|
|
'eta_cutoff': 0,
|
|
'tfs': 1,
|
|
'top_a': 0,
|
|
'repetition_penalty': 1.18,
|
|
'repetition_penalty_range': 0,
|
|
'top_k': 40,
|
|
'min_length': 0,
|
|
'no_repeat_ngram_size': 0,
|
|
'num_beams': 1,
|
|
'penalty_alpha': 0,
|
|
'length_penalty': 1,
|
|
'early_stopping': False,
|
|
'mirostat_mode': 0,
|
|
'mirostat_tau': 5,
|
|
'mirostat_eta': 0.1,
|
|
'guidance_scale': 1,
|
|
'negative_prompt': '',
|
|
'seed': -1,
|
|
'add_bos_token': True,
|
|
'truncation_length': 2048,
|
|
'ban_eos_token': False,
|
|
'custom_token_bans': '',
|
|
'skip_special_tokens': True,
|
|
'stopping_strings': []
|
|
}
|
|
|
|
async with websockets.connect(f'wss://{config["HOST"]}/api/v1/stream', ping_interval=None) as websocket:
|
|
await websocket.send(json.dumps(request))
|
|
|
|
yield context # Remove this if you just want to see the reply
|
|
|
|
while True:
|
|
incoming_data = await websocket.recv()
|
|
incoming_data = json.loads(incoming_data)
|
|
|
|
print(incoming_data)
|
|
|
|
match incoming_data['event']:
|
|
# case 'text_stream':
|
|
# yield incoming_data['text']
|
|
case 'stream_end':
|
|
return
|
|
|
|
|
|
async def print_response_stream(prompt):
|
|
try:
|
|
async for response in run(prompt):
|
|
print(response, end='')
|
|
sys.stdout.flush() # If we don't flush, we won't see tokens in realtime.
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
prompt = "Write a 300 word story about an apple tree.\n\n"
|
|
while True:
|
|
print('--> START <--')
|
|
asyncio.run(print_response_stream(prompt))
|
|
print('--> DONE <--')
|
|
time.sleep(2)
|