Merge pull request #64 from Supahands/develop

Develop to main replacing ollama with hugging face models for supa
Supahands · Dec 13, 2024 · 2140cc9 · 2140cc9
2 parents efe817a + 24d025a
commit 2140cc9
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 9 deletions.
diff --git a/ai_router.py b/ai_router.py
@@ -41,6 +41,7 @@
         "pydantic==2.5.3", 
         "fastapi==0.109.0", 
         "openai", 
+        "langfuse"
     ]
 )
 llm_compare_app = App(
@@ -50,7 +51,8 @@
         Secret.from_name("SUPABASE_SECRETS"),
         Secret.from_name("OLLAMA_API"),
         Secret.from_name("llm_comparison_github"),
-        Secret.from_name("my-huggingface-secret")
+        Secret.from_name("my-huggingface-secret"),
+        Secret.from_name("Langfuse-Secret")
     ],
 )
 
@@ -62,6 +64,8 @@
     import re
 
     litellm.set_verbose=True # 👈 this is the 1-line change you need to make
+    litellm.success_callback = ["langfuse"]
+    litellm.failure_callback = ["langfuse"] # logs errors to langfuse
 
     # Initialize Supabase client
     supabase_url = os.environ["SUPABASE_URL"]
@@ -202,13 +206,19 @@ async def handle_completion(
                 model=model_name,
                 messages=[{"content": message, "role": "user"}],
                 api_base=api_base,
-                timeout=180.00
+                timeout=180.00,
+                metadata = {
+                    "generation_name": model_name, # set langfuse generation name
+                }
             )
         else:
             response_obj = completion(
                 model=model_name,
                 messages=[{"content": message, "role": "user"}],
-                timeout=180.00
+                timeout=180.00,
+                metadata = {
+                    "generation_name": model_name, # set langfuse generation name
+                }
             )
 
         end_time = time.time()

diff --git a/ollama_service.py b/ollama_service.py
@@ -20,17 +20,29 @@
     "llama3.2",
     "llama3.2:1b",
     "llama3.3",
+    "tinyllama:1.1b",
+    "deepseek-coder-v2:16b",
     "mistral",
     "gemma2",
     "qwen2.5",
     "yi",
+    "qwq:32b",
+    "codellama:7b",
+    "codellama:70b",
+    "qwen2.5-coder:7b",
+    "qwen2.5-coder:32b",
+    "medllama2",
+    "meditron:7b",
+    "meditron:70b",
+    "mathstral:7b",
+    "athene-v2:72b",
     "aisingapore/gemma2-9b-cpt-sea-lionv3-instruct",
-    "Supa-AI/gemma2-9b-cpt-sahabatai-v1-instruct:q8_0",
-    "Supa-AI/llama3-8b-cpt-sahabatai-v1-instruct:q8_0",
-    "Supa-AI/gemma2-9b-cpt-sahabatai-v1-base:q8_0",
-    "Supa-AI/ministral-8b-instruct-2410:q8_0",
-    "Supa-AI/mixtral-8x7b-instruct-v0.1:q8_0",
-    "Supa-AI/malaysian-llama-3.2-3b-instruct:q8_0",
+    "hf.co/Supa-AI/llama3-8b-cpt-sahabatai-v1-instruct-gguf:Q8_0",
+    "hf.co/Supa-AI/llama3-8b-cpt-sahabatai-v1-instruct-gguf:Q2_K",
+    "hf.co/Supa-AI/Ministral-8B-Instruct-2410-gguf:Q8_0",
+    "hf.co/Supa-AI/gemma2-9b-cpt-sahabatai-v1-instruct-q8_0-gguf",
+    "hf.co/Supa-AI/Mixtral-8x7B-Instruct-v0.1-gguf:Q8_0",
+    "hf.co/Supa-AI/malaysian-Llama-3.2-3B-Instruct-gguf:Q8_0"
 ]
 
 OLLAMA_PORT: int = 11434