fix temperature, non-blocking generation

main.py

@@ -110,7 +110,7 @@ async def main():
                           log_full_response=config_data['logging'].get('log_full_response', False),
                           system_prompt=config_data['openai'].get('system_prompt'),
                           injected_system_prompt=config_data['openai'].get('injected_system_prompt', False),
-                          hyper_temperature=config_data['openai'].get('temperature', 0)
+                          openai_temperature=config_data['openai'].get('temperature', 0)
                           )
     client.add_event_callback(callbacks.message, RoomMessageText)
     client.add_event_callback(callbacks.invite_event_filtered_callback, InviteMemberEvent)

matrix_gpt/bot/bot_commands.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from types import ModuleType
 
@@ -22,6 +23,7 @@ class Command:
             openai_obj: ModuleType,
             openai_model: str,
             reply_in_thread,
+            openai_temperature: float = 0,
             system_prompt: str = None,
             injected_system_prompt: str = None,
             log_full_response: bool = False
@@ -54,6 +56,7 @@ class Command:
         self.injected_system_prompt = injected_system_prompt
         self.log_full_response = log_full_response
         self.openai_obj = openai_obj
+        self.openai_temperature = openai_temperature
 
     async def process(self):
         """Process the command"""
@@ -69,18 +72,22 @@ class Command:
         await self._process_chat()
 
     async def _process_chat(self):
-        await process_chat(
-            self.client,
-            self.room,
-            self.event,
-            self.command,
-            self.store,
-            openai_obj=self.openai_obj,
-            openai_model=self.openai_model,
-            system_prompt=self.system_prompt,
-            injected_system_prompt=self.injected_system_prompt,
-            log_full_response=self.log_full_response
-        )
+        async def inner():
+            await process_chat(
+                self.client,
+                self.room,
+                self.event,
+                self.command,
+                self.store,
+                openai_obj=self.openai_obj,
+                openai_model=self.openai_model,
+                openai_temperature=self.openai_temperature,
+                system_prompt=self.system_prompt,
+                injected_system_prompt=self.injected_system_prompt,
+                log_full_response=self.log_full_response
+            )
+
+        asyncio.get_event_loop().create_task(inner())
 
     async def _show_help(self):
         """Show the help text"""

matrix_gpt/bot/callbacks.py

@@ -1,4 +1,5 @@
 # https://github.com/anoadragon453/nio-template
+import asyncio
 import logging
 import time
 from types import ModuleType
@@ -26,7 +27,7 @@ class Callbacks:
                  system_prompt: str = None,
                  log_full_response: bool = False,
                  injected_system_prompt: str = False,
-                 hyper_temperature: float = 0
+                 openai_temperature: float = 0
                  ):
         """
         Args:
@@ -49,6 +50,7 @@ class Callbacks:
         self.log_full_response = log_full_response
         self.injected_system_prompt = injected_system_prompt
         self.openai_obj = openai_obj
+        self.openai_temperature = openai_temperature
 
     async def message(self, room: MatrixRoom, event: RoomMessageText) -> None:
         """Callback for when a message event is received
@@ -105,22 +107,25 @@ class Callbacks:
                             'content': thread_msg if not thread_msg.startswith(self.command_prefix) else thread_msg[len(self.command_prefix):].strip()
                         })  # if len(thread_content) >= 2 and thread_content[0].body.startswith(self.command_prefix):  # if thread_content[len(thread_content) - 2].sender == self.client.user
 
-            # message = Message(self.client, self.store, msg, room, event, self.reply_in_thread)
-            # await message.process()
-            # api_data.append({'role': 'user', 'content': msg})
-            await process_chat(
-                self.client,
-                room,
-                event,
-                api_data,
-                self.store,
-                openai_obj=self.openai_obj,
-                openai_model=self.openai_model,
-                thread_root_id=thread_content[0].event_id,
-                system_prompt=self.system_prompt,
-                log_full_response=self.log_full_response,
-                injected_system_prompt=self.injected_system_prompt
-            )
+            # TODO: process_chat() will set typing as false after generating.
+            # TODO: If there is still another query in-progress that typing state will be overwritten by the one that just finished.
+            async def inner():
+                await process_chat(
+                    self.client,
+                    room,
+                    event,
+                    api_data,
+                    self.store,
+                    openai_obj=self.openai_obj,
+                    openai_model=self.openai_model,
+                    openai_temperature=self.openai_temperature,
+                    thread_root_id=thread_content[0].event_id,
+                    system_prompt=self.system_prompt,
+                    log_full_response=self.log_full_response,
+                    injected_system_prompt=self.injected_system_prompt
+                )
+
+            asyncio.get_event_loop().create_task(inner())
             return
         elif msg.startswith(f'{self.command_prefix} ') or room.member_count == 2:
             # Otherwise if this is in a 1-1 with the bot or features a command prefix, treat it as a command.
@@ -133,6 +138,7 @@ class Callbacks:
                 event,
                 openai_obj=self.openai_obj,
                 openai_model=self.openai_model,
+                openai_temperature=self.openai_temperature,
                 reply_in_thread=self.reply_in_thread,
                 system_prompt=self.system_prompt,
                 injected_system_prompt=self.injected_system_prompt,

matrix_gpt/bot/chat_functions.py

@@ -1,5 +1,8 @@
+import asyncio
+import functools
 import logging
 import time
+import traceback
 from types import ModuleType
 from typing import List, Optional, Union
 
@@ -132,7 +135,21 @@ async def get_thread_content(client: AsyncClient, room: MatrixRoom, base_event:
     return messages
 
 
-async def process_chat(client, room, event, command, store, openai_obj: ModuleType, openai_model: str, openai_retries: int = 3, thread_root_id: str = None, system_prompt: str = None, log_full_response: bool = False, injected_system_prompt: str = False):
+async def process_chat(
+        client,
+        room,
+        event,
+        command,
+        store,
+        openai_obj: ModuleType,
+        openai_model: str,
+        openai_temperature: float,
+        openai_retries: int = 3,
+        thread_root_id: str = None,
+        system_prompt: str = None,
+        log_full_response: bool = False,
+        injected_system_prompt: str = False
+):
     if not store.check_seen_event(event.event_id):
         await client.room_typing(room.room_id, typing_state=True, timeout=9000)
         # if self.reply_in_thread:
@@ -158,25 +175,35 @@ async def process_chat(client, room, event, command, store, openai_obj: ModuleTy
 
         logger.debug(f'Generating reply to event {event.event_id}')
 
+        loop = asyncio.get_running_loop()
+
         # I don't think the OpenAI py api has a built-in timeout
         @stopit.threading_timeoutable(default=(None, None))
-        def generate():
-            r = openai_obj.ChatCompletion.create(model=openai_model, messages=messages, temperature=0, timeout=10)
-            return r["choices"][0]["message"]["content"].strip().strip('\n'), r
+        async def generate():
+            return await loop.run_in_executor(None, functools.partial(openai_obj.ChatCompletion.create, model=openai_model, messages=messages, temperature=openai_temperature, timeout=20))
+            # r = openai_obj.ChatCompletion.create(model=openai_model, messages=messages, temperature=openai_temperature, timeout=20)
 
         text_response = response = None
         for i in range(openai_retries):
             try:
-                text_response, response = generate(timeout=20)
-                if text_response is not None and response is not None:
+                task = asyncio.create_task(generate(timeout=20))
+                asyncio.as_completed(task)
+                response = await task
+                if response is not None:
                     break
             except stopit.utils.TimeoutException:
                 time.sleep(2)
                 continue
-        if text_response is None:
+            except Exception as e:
+                logger.critical(f'OpenAI API error: {e}\n{traceback.format_exc()}')
+                await react_to_event(client, room.room_id, event.event_id, '❌')
+                return
+
+        if response is None:
             logger.critical(f'OpenAI API timeout for event {event.event_id} in room {room.room_id}.')
             await react_to_event(client, room.room_id, event.event_id, '❌')
             return
+        text_response = response["choices"][0]["message"]["content"].strip().strip('\n')
 
         # Logging stuff
         if log_full_response: