Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

XAI Support #470

Merged
merged 1 commit into from
Dec 20, 2024
Merged

XAI Support #470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions refact_known_models/passthrough.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,5 +260,55 @@
"pp1000t_prompt": 1250, # $1.25 / 1M tokens
"pp1000t_generated": 5000, # $5.00 / 1M tokens
"filter_caps": ["chat", "tools", "completion", "vision"],
},
# XAI Models
# WARNING: tokenizer is non-precise as there's no publicly available tokenizer for these models
# XAI says that for exact same model different tokenizers could be used
# therefore, using tokenizer for grok-1 which may or may not provide proximate enough results
# T is decreased not to encounter tokens overflow

"grok-beta": {
"backend": "litellm",
"provider": "xai",
"tokenizer_path": "Xenova/grok-1-tokenizer",
"resolve_as": "xai/grok-beta",
"T": 128_000,
"T_out": 4096,
"pp1000t_prompt": 5_000,
"pp1000t_generated": 15_000, # $15.00 / 1M tokens
"filter_caps": ["chat", "completion"],
},
"grok-vision-beta": {
"backend": "litellm",
"provider": "xai",
"tokenizer_path": "Xenova/grok-1-tokenizer",
"resolve_as": "xai/grok-vision-beta",
"T": 8_192,
"T_out": 4096,
"pp1000t_prompt": 5_000,
"pp1000t_generated": 15_000, # $15.00 / 1M tokens
"filter_caps": ["chat", "completion", "vision"],
},
"grok-2-vision-1212": {
"backend": "litellm",
"provider": "xai",
"tokenizer_path": "Xenova/grok-1-tokenizer",
"resolve_as": "xai/grok-2-vision-1212",
"T": 32_000,
"T_out": 4096,
"pp1000t_prompt": 2_000,
"pp1000t_generated": 10_000, # $10.00 / 1M tokens
"filter_caps": ["chat", "completion", "vision"],
},
"grok-2-1212": {
"backend": "litellm",
"provider": "xai",
"tokenizer_path": "Xenova/grok-1-tokenizer",
"resolve_as": "xai/grok-2-1212",
"T": 128_000,
"T_out": 4096,
"pp1000t_prompt": 2_000,
"pp1000t_generated": 10_000, # $10.00 / 1M tokens
"filter_caps": ["chat", "completion"],
}
}
3 changes: 3 additions & 0 deletions refact_utils/finetune/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ def _add_results_for_passthrough_provider(provider: str) -> None:
if data.get('groq_api_enable'):
_add_results_for_passthrough_provider('groq')

if data.get('xai_api_enable'):
_add_results_for_passthrough_provider('xai')

for k, v in data.get("model_assign", {}).items():
if model_dict := [d for d in data['models'] if d['name'] == k]:
model_dict = model_dict[0]
Expand Down
1 change: 1 addition & 0 deletions refact_webgui/webgui/selfhost_fastapi_completions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ def _integrations_env_setup(env_var_name: str, api_key_name: str, api_enable_nam
_integrations_env_setup("GROQ_API_KEY", "groq_api_key", "groq_api_enable")
_integrations_env_setup("CEREBRAS_API_KEY", "cerebras_api_key", "cerebras_api_enable")
_integrations_env_setup("GEMINI_API_KEY", "gemini_api_key", "gemini_api_enable")
_integrations_env_setup("XAI_API_KEY", "xai_api_key", "xai_api_enable")

def _models_available_dict_rewrite(self, models_available: List[str]) -> Dict[str, Any]:
rewrite_dict = {}
Expand Down
2 changes: 2 additions & 0 deletions refact_webgui/webgui/selfhost_model_assigner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ def first_run(self):
"groq_api_enable": False,
"cerebras_api_enable": False,
"gemini_api_enable": False,
"xai_api_enable": False,
}
self.models_to_watchdog_configs(default_config)

Expand Down Expand Up @@ -261,6 +262,7 @@ def model_assignment(self):
j["groq_api_enable"] = j.get("groq_api_enable", False)
j["cerebras_api_enable"] = j.get("cerebras_api_enable", False)
j["gemini_api_enable"] = j.get("gemini_api_enable", False)
j["xai_api_enable"] = j.get("xai_api_enable", False)
else:
j = {"model_assign": {}}

Expand Down
2 changes: 2 additions & 0 deletions refact_webgui/webgui/selfhost_queue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ def _add_models_for_passthrough_provider(provider):
_add_models_for_passthrough_provider('cerebras')
if j.get("gemini_api_enable"):
_add_models_for_passthrough_provider('gemini')
if j.get("xai_api_enable"):
_add_models_for_passthrough_provider('xai')

return self._models_available

Expand Down
4 changes: 4 additions & 0 deletions refact_webgui/webgui/static/tab-model-hosting.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ <h3>3rd Party APIs</h3>
<input class="form-check-input" type="checkbox" role="switch" id="enable_gemini">
<label class="form-check-label" for="enable_gemini">Enable Gemini API</label>
</div>
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="enable_xai">
<label class="form-check-label" for="enable_xai">Enable XAI API</label>
</div>

<div class="chat-enabler-status">
To enable Chat GPT add your API key in the <span id="redirect2credentials" class="main-tab-button fake-link" data-tab="settings">API Keys tab</span>.
Expand Down
4 changes: 3 additions & 1 deletion refact_webgui/webgui/static/tab-model-hosting.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ function get_models()
integration_switch_init('enable_groq', models_data['groq_api_enable']);
integration_switch_init('enable_cerebras', models_data['cerebras_api_enable']);
integration_switch_init('enable_gemini', models_data['gemini_api_enable']);

integration_switch_init('enable_xai', models_data['xai_api_enable']);

const more_gpus_notification = document.querySelector('.model-hosting-error');
if(data.hasOwnProperty('more_models_than_gpus') && data.more_models_than_gpus) {
Expand All @@ -147,6 +147,7 @@ function save_model_assigned() {
const groq_enable = document.querySelector('#enable_groq');
const cerebras_enable = document.querySelector('#enable_cerebras');
const gemini_enable = document.querySelector('#enable_gemini');
const xai_enable = document.querySelector('#enable_xai');

const data = {
model_assign: {
Expand All @@ -157,6 +158,7 @@ function save_model_assigned() {
groq_api_enable: groq_enable.checked,
cerebras_api_enable: cerebras_enable.checked,
gemini_api_enable: gemini_enable.checked,
xai_api_enable: xai_enable.checked,
};
console.log(data);
fetch("/tab-host-models-assign", {
Expand Down
2 changes: 2 additions & 0 deletions refact_webgui/webgui/static/tab-settings.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ <h2>API Integrations</h2>
<input type="text" name="cerebras_api_key" value="" class="form-control" id="cerebras_api_key">
<label for="gemini_api_key" class="form-label mt-4">Gemini API Key</label>
<input type="text" name="gemini_api_key" value="" class="form-control" id="gemini_api_key">
<label for="xai_api_key" class="form-label mt-4">XAI API Key</label>
<input type="text" name="xai_api_key" value="" class="form-control" id="xai_api_key">

<!-- <div class="d-flex flex-row-reverse mt-3"><button type="button" class="btn btn-primary" id="integrations-save">Save</button></div>-->
</div>
Expand Down
4 changes: 4 additions & 0 deletions refact_webgui/webgui/static/tab-settings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ function save_integration_api_keys() {
const groq_api_key = document.getElementById('groq_api_key');
const cerebras_api_key = document.getElementById('cerebras_api_key');
const gemini_api_key = document.getElementById("gemini_api_key");
const xai_api_key = document.getElementById('xai_api_key');

const huggingface_api_key = document.getElementById('huggingface_api_key');
fetch("/tab-settings-integrations-save", {
Expand All @@ -188,6 +189,7 @@ function save_integration_api_keys() {
groq_api_key: groq_api_key.getAttribute('data-value'),
cerebras_api_key: cerebras_api_key.getAttribute('data-value'),
gemini_api_key: gemini_api_key.getAttribute("data-value"),
xai_api_key: xai_api_key.getAttribute('data-value'),

huggingface_api_key: huggingface_api_key.getAttribute('data-value'),
})
Expand All @@ -200,6 +202,7 @@ function save_integration_api_keys() {
groq_api_key.setAttribute('data-saved-value', groq_api_key.getAttribute('data-value'))
cerebras_api_key.setAttribute('data-saved-value', cerebras_api_key.getAttribute('data-value'))
gemini_api_key.setAttribute('data-saved-value', gemini_api_key.getAttribute('data-value'))
xai_api_key.setAttribute('data-saved-value', xai_api_key.getAttribute('data-value'))

huggingface_api_key.setAttribute('data-saved-value', huggingface_api_key.getAttribute('data-value'))
});
Expand Down Expand Up @@ -237,6 +240,7 @@ export function tab_settings_integrations_get() {
integrations_input_init(document.getElementById('groq_api_key'), data['groq_api_key']);
integrations_input_init(document.getElementById('cerebras_api_key'), data['cerebras_api_key']);
integrations_input_init(document.getElementById('gemini_api_key'), data['gemini_api_key']);
integrations_input_init(document.getElementById('xai_api_key'), data['xai_api_key']);

integrations_input_init(document.getElementById('huggingface_api_key'), data['huggingface_api_key']);
});
Expand Down
1 change: 1 addition & 0 deletions refact_webgui/webgui/tab_models_host.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class TabHostModelsAssign(BaseModel):
groq_api_enable: bool = False
cerebras_api_enable: bool = False
gemini_api_enable: bool = False
xai_api_enable: bool = False

model_config = ConfigDict(protected_namespaces=()) # avoiding model_ namespace protection

Expand Down
1 change: 1 addition & 0 deletions refact_webgui/webgui/tab_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class Integrations(BaseModel):
groq_api_key: Optional[str] = None
cerebras_api_key: Optional[str] = None
gemini_api_key: Optional[str] = None
xai_api_key: Optional[str] = None

huggingface_api_key: Optional[str] = None

Expand Down
Loading