feat: sort cuda graphs in descending order (#2104)

2024-06-21 14:28:26 -04:00 · 2024-06-21 14:28:26 -04:00 · 811a9381b1
parent 197c47a302
commit 811a9381b1
1 changed files with 7 additions and 0 deletions
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@ -15,6 +15,13 @@ if cuda_graphs is not None:
 else:
    cuda_graphs = None

+
+# sorting the cuda graphs in descending order helps reduce the
+# memory impact and results in less memory usage
+if cuda_graphs is not None:
+    cuda_graphs.sort(reverse=True)
+
+
 CUDA_GRAPHS = cuda_graphs

 # This is overridden at model loading.