diff --git a/refact_known_models/passthrough.py b/refact_known_models/passthrough.py index 4ed43cb7..d94f8af2 100644 --- a/refact_known_models/passthrough.py +++ b/refact_known_models/passthrough.py @@ -260,5 +260,55 @@ "pp1000t_prompt": 1250, # $1.25 / 1M tokens "pp1000t_generated": 5000, # $5.00 / 1M tokens "filter_caps": ["chat", "tools", "completion", "vision"], + }, + # XAI Models + # WARNING: tokenizer is non-precise as there's no publicly available tokenizer for these models + # XAI says that for exact same model different tokenizers could be used + # therefore, using tokenizer for grok-1 which may or may not provide proximate enough results + # T is decreased not to encounter tokens overflow + + "grok-beta": { + "backend": "litellm", + "provider": "xai", + "tokenizer_path": "Xenova/grok-1-tokenizer", + "resolve_as": "xai/grok-beta", + "T": 128_000, + "T_out": 4096, + "pp1000t_prompt": 5_000, + "pp1000t_generated": 15_000, # $15.00 / 1M tokens + "filter_caps": ["chat", "completion"], + }, + "grok-vision-beta": { + "backend": "litellm", + "provider": "xai", + "tokenizer_path": "Xenova/grok-1-tokenizer", + "resolve_as": "xai/grok-vision-beta", + "T": 8_192, + "T_out": 4096, + "pp1000t_prompt": 5_000, + "pp1000t_generated": 15_000, # $15.00 / 1M tokens + "filter_caps": ["chat", "completion", "vision"], + }, + "grok-2-vision-1212": { + "backend": "litellm", + "provider": "xai", + "tokenizer_path": "Xenova/grok-1-tokenizer", + "resolve_as": "xai/grok-2-vision-1212", + "T": 32_000, + "T_out": 4096, + "pp1000t_prompt": 2_000, + "pp1000t_generated": 10_000, # $10.00 / 1M tokens + "filter_caps": ["chat", "completion", "vision"], + }, + "grok-2-1212": { + "backend": "litellm", + "provider": "xai", + "tokenizer_path": "Xenova/grok-1-tokenizer", + "resolve_as": "xai/grok-2-1212", + "T": 128_000, + "T_out": 4096, + "pp1000t_prompt": 2_000, + "pp1000t_generated": 10_000, # $10.00 / 1M tokens + "filter_caps": ["chat", "completion"], } } diff --git a/refact_utils/finetune/utils.py b/refact_utils/finetune/utils.py index 81b753e0..69558235 100644 --- a/refact_utils/finetune/utils.py +++ b/refact_utils/finetune/utils.py @@ -118,6 +118,9 @@ def _add_results_for_passthrough_provider(provider: str) -> None: if data.get('groq_api_enable'): _add_results_for_passthrough_provider('groq') + if data.get('xai_api_enable'): + _add_results_for_passthrough_provider('xai') + for k, v in data.get("model_assign", {}).items(): if model_dict := [d for d in data['models'] if d['name'] == k]: model_dict = model_dict[0] diff --git a/refact_webgui/webgui/selfhost_fastapi_completions.py b/refact_webgui/webgui/selfhost_fastapi_completions.py index 4f9bb8c4..6fbfa0ec 100644 --- a/refact_webgui/webgui/selfhost_fastapi_completions.py +++ b/refact_webgui/webgui/selfhost_fastapi_completions.py @@ -234,6 +234,7 @@ def _integrations_env_setup(env_var_name: str, api_key_name: str, api_enable_nam _integrations_env_setup("GROQ_API_KEY", "groq_api_key", "groq_api_enable") _integrations_env_setup("CEREBRAS_API_KEY", "cerebras_api_key", "cerebras_api_enable") _integrations_env_setup("GEMINI_API_KEY", "gemini_api_key", "gemini_api_enable") + _integrations_env_setup("XAI_API_KEY", "xai_api_key", "xai_api_enable") def _models_available_dict_rewrite(self, models_available: List[str]) -> Dict[str, Any]: rewrite_dict = {} diff --git a/refact_webgui/webgui/selfhost_model_assigner.py b/refact_webgui/webgui/selfhost_model_assigner.py index 0569c65a..0a6a9fa4 100644 --- a/refact_webgui/webgui/selfhost_model_assigner.py +++ b/refact_webgui/webgui/selfhost_model_assigner.py @@ -187,6 +187,7 @@ def first_run(self): "groq_api_enable": False, "cerebras_api_enable": False, "gemini_api_enable": False, + "xai_api_enable": False, } self.models_to_watchdog_configs(default_config) @@ -261,6 +262,7 @@ def model_assignment(self): j["groq_api_enable"] = j.get("groq_api_enable", False) j["cerebras_api_enable"] = j.get("cerebras_api_enable", False) j["gemini_api_enable"] = j.get("gemini_api_enable", False) + j["xai_api_enable"] = j.get("xai_api_enable", False) else: j = {"model_assign": {}} diff --git a/refact_webgui/webgui/selfhost_queue.py b/refact_webgui/webgui/selfhost_queue.py index 13b4a3b0..e401bfe3 100644 --- a/refact_webgui/webgui/selfhost_queue.py +++ b/refact_webgui/webgui/selfhost_queue.py @@ -70,6 +70,8 @@ def _add_models_for_passthrough_provider(provider): _add_models_for_passthrough_provider('cerebras') if j.get("gemini_api_enable"): _add_models_for_passthrough_provider('gemini') + if j.get("xai_api_enable"): + _add_models_for_passthrough_provider('xai') return self._models_available diff --git a/refact_webgui/webgui/static/tab-model-hosting.html b/refact_webgui/webgui/static/tab-model-hosting.html index ea4a19cc..9d765dda 100644 --- a/refact_webgui/webgui/static/tab-model-hosting.html +++ b/refact_webgui/webgui/static/tab-model-hosting.html @@ -50,6 +50,10 @@