From 69fd3e6e58f36914de7d6758802b31da1c38f1ab Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 2 Sep 2024 14:17:38 +0800 Subject: [PATCH 001/476] Fix the bug of incorrect request body format for tool use. --- request.py | 38 ++++++++++++++++++++------------------ test/curl.py | 6 +++++- utils.py | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/request.py b/request.py index f1008aad..2c927ef7 100644 --- a/request.py +++ b/request.py @@ -406,15 +406,16 @@ async def get_vertex_claude_payload(request, engine, provider): if tool_calls: tool_calls_list = [] - for tool_call in tool_calls: - tool_calls_list.append({ - "type": "tool_use", - "id": tool_call.id, - "name": tool_call.function.name, - "input": json.loads(tool_call.function.arguments), - }) - messages.append({"role": msg.role, "content": tool_calls_list}) - elif tool_call_id: + tool_call = tool_calls[0] + tool_calls_list.append({ + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": json.loads(tool_call.function.arguments), + }) + messages.append({"role": msg.role, "content": tool_calls_list}) + elif tool_call_id and tool_calls: + tool_call = tool_calls[0] messages.append({"role": "user", "content": [{ "type": "tool_result", "tool_use_id": tool_call.id, @@ -668,15 +669,16 @@ async def get_claude_payload(request, engine, provider): if tool_calls: tool_calls_list = [] - for tool_call in tool_calls: - tool_calls_list.append({ - "type": "tool_use", - "id": tool_call.id, - "name": tool_call.function.name, - "input": json.loads(tool_call.function.arguments), - }) - messages.append({"role": msg.role, "content": tool_calls_list}) - elif tool_call_id: + tool_call = tool_calls[0] + tool_calls_list.append({ + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": json.loads(tool_call.function.arguments), + }) + messages.append({"role": msg.role, "content": tool_calls_list}) + elif tool_call_id and tool_calls: + tool_call = tool_calls[0] messages.append({"role": "user", "content": [{ "type": "tool_result", "tool_use_id": tool_call.id, diff --git a/test/curl.py b/test/curl.py index 4454cb83..3cc910e0 100644 --- a/test/curl.py +++ b/test/curl.py @@ -6,8 +6,12 @@ provider_name = "linuxdoi" model = "claude-3-5-sonnet" +import asyncio +config, api_keys_db, api_list = asyncio.run(load_config()) +import json -config, api_keys_db, api_list = load_config() +print(json.dumps(api_keys_db, indent=2)) +exit(0) providers = config["providers"] provider_config = None for provider in providers: diff --git a/utils.py b/utils.py index 6b18ae51..d07dce34 100644 --- a/utils.py +++ b/utils.py @@ -22,7 +22,7 @@ def update_config(config_data): return config_data, api_keys_db, api_list # 读取YAML配置文件 -async def load_config(app): +async def load_config(app=None): import yaml try: with open('./api.yaml', 'r') as f: From 58a694dc33800475ea3031176c2e4ff5bc0d903b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 2 Sep 2024 14:28:08 +0800 Subject: [PATCH 002/476] Fix the bug where the provider lookup is inaccurate. --- main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 49e6285f..616111e2 100644 --- a/main.py +++ b/main.py @@ -159,13 +159,16 @@ def get_matching_providers(self, model_name, token): # print("provider_rules", provider_rules) for item in provider_rules: for provider in config['providers']: - if provider['provider'] in item: + if provider['provider'] == item: if "/" in item: if item.split("/")[1] == model_name: provider_list.append(provider) else: if model_name in provider['model'].keys(): provider_list.append(provider) + # import json + # for provider in provider_list: + # print(json.dumps(provider, indent=4, ensure_ascii=False)) return provider_list async def request_model(self, request: RequestModel, token: str): From 46ce91064aa21fed2cfe436f1d2311fafabc4b2e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 2 Sep 2024 20:58:50 +0800 Subject: [PATCH 003/476] Fix the bug with the incorrect ending character in OpenAI format responses. --- response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/response.py b/response.py index bf3c3e35..051789fc 100644 --- a/response.py +++ b/response.py @@ -32,7 +32,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f json_data = json.dumps(sample_data, ensure_ascii=False) # 构建SSE响应 - sse_response = f"data: {json_data}\n\n" + sse_response = f"data: {json_data}\n\r" return sse_response @@ -173,7 +173,7 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects line, buffer = buffer.split("\n", 1) # logger.info("line: %s", repr(line)) if line and line != "data: " and line != "data:" and not line.startswith(": "): - yield line + "\n" + yield line + "\n\r" except httpx.RemoteProtocolError as e: yield {"error": f"fetch_gpt_response_stream RemoteProtocolError {e.__class__.__name__}", "details": str(e)} return From 2ec084286ceb537a578b8ea51e6cc56c45905f44 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Sep 2024 03:09:16 +0800 Subject: [PATCH 004/476] Add feature: support OpenAI dall-e-3 image generation --- README.md | 23 ++++++++++------- main.py | 31 +++++++++++++++-------- models.py | 7 ++++++ request.py | 23 ++++++++++++++++- response.py | 72 ++++++++++++++++++++++++----------------------------- utils.py | 28 ++++++++++++++++++++- 6 files changed, 124 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index e419fcd6..bc0d376f 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,23 @@ ## Introduction -这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 ## Features -- 统一管理多个后端服务 -- 支持负载均衡 -- 支持 OpenAI, Anthropic, Gemini, Vertex 函数调用 -- 支持多个模型 -- 支持多个 API Key -- 支持 Vertex 区域负载均衡，支持 Vertex 高并发 +- 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 +- 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 +- 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 +- 支持负载均衡，支持 Vertex 区域负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。除了 Vertex 区域负载均衡，所有 API 均支持渠道级负载均衡，提高沉浸式翻译体验。 +- 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 +- 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 +- 支持多个 API Key。 ## Configuration -使用api.yaml配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： +使用 api.yaml 配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： ```yaml providers: @@ -35,6 +38,7 @@ providers: model: # 至少填一个模型 - gpt-4o # 可以使用的模型名称，必填 - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages @@ -86,7 +90,7 @@ api_keys: model: - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。 preferences: - USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true + USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true ``` @@ -152,6 +156,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` + ## Star History diff --git a/main.py b/main.py index 616111e2..1085f6fd 100644 --- a/main.py +++ b/main.py @@ -5,16 +5,16 @@ from contextlib import asynccontextmanager from fastapi.middleware.cors import CORSMiddleware -from fastapi import FastAPI, HTTPException, Depends, Request +from fastapi import FastAPI, HTTPException, Depends from fastapi.responses import StreamingResponse, JSONResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from models import RequestModel +from models import RequestModel, ImageGenerationRequest from utils import error_handling_wrapper, get_all_models, post_all_models, load_config from request import get_payload from response import fetch_response, fetch_response_stream -from typing import List, Dict +from typing import List, Dict, Union from urllib.parse import urlparse @asynccontextmanager @@ -80,7 +80,7 @@ async def lifespan(app: FastAPI): allow_headers=["*"], # 允许所有头部字段 ) -async def process_request(request: RequestModel, provider: Dict): +async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) # print(parsed_url) @@ -101,6 +101,10 @@ async def process_request(request: RequestModel, provider: Dict): and "gemini" not in provider['model'][request.model]: engine = "openrouter" + if endpoint == "/v1/images/generations": + engine = "dalle" + request.stream = False + if provider.get("engine"): engine = provider["engine"] @@ -122,7 +126,7 @@ async def process_request(request: RequestModel, provider: Dict): wrapped_generator = await error_handling_wrapper(generator, status_code=500) return StreamingResponse(wrapped_generator, media_type="text/event-stream") else: - return await fetch_response(app.state.client, url, headers, payload) + return await anext(fetch_response(app.state.client, url, headers, payload)) import asyncio class ModelRequestHandler: @@ -171,7 +175,7 @@ def get_matching_providers(self, model_name, token): # print(json.dumps(provider, indent=4, ensure_ascii=False)) return provider_list - async def request_model(self, request: RequestModel, token: str): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None): config = app.state.config # api_keys_db = app.state.api_keys_db api_list = app.state.api_list @@ -193,9 +197,9 @@ async def request_model(self, request: RequestModel, token: str): if config['api_keys'][api_index]["preferences"].get("AUTO_RETRY") == False: auto_retry = False - return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry) + return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint) - async def try_all_providers(self, request: RequestModel, providers: List[Dict], use_round_robin: bool, auto_retry: bool): + async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None): num_providers = len(providers) start_index = self.last_provider_index + 1 if use_round_robin else 0 @@ -203,7 +207,7 @@ async def try_all_providers(self, request: RequestModel, providers: List[Dict], self.last_provider_index = (start_index + i) % num_providers provider = providers[self.last_provider_index] try: - response = await process_request(request, provider) + response = await process_request(request, provider, endpoint) return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: logger.error(f"Error with provider {provider['provider']}: {str(e)}") @@ -228,7 +232,7 @@ def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security) return token @app.post("/v1/chat/completions") -async def request_model(request: RequestModel, token: str = Depends(verify_api_key)): +async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): return await model_handler.request_model(request, token) @app.options("/v1/chat/completions") @@ -251,6 +255,13 @@ async def list_models(): "data": models }) +@app.post("/v1/images/generations") +async def images_generations( + request: ImageGenerationRequest, + token: str = Depends(verify_api_key) +): + return await model_handler.request_model(request, token, endpoint="/v1/images/generations") + @app.get("/generate-api-key") def generate_api_key(): api_key = "sk-" + secrets.token_urlsafe(32) diff --git a/models.py b/models.py index b69d3bf5..11eb949a 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,13 @@ from pydantic import BaseModel, Field from typing import List, Dict, Optional, Union +class ImageGenerationRequest(BaseModel): + model: str + prompt: str + n: int + size: str + stream: bool = False + class FunctionParameter(BaseModel): type: str properties: Dict[str, Dict[str, str]] diff --git a/request.py b/request.py index 2c927ef7..3777772f 100644 --- a/request.py +++ b/request.py @@ -1,6 +1,6 @@ import json from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, CircularList +from utils import c35s, c3s, c3o, c3h, gem, BaseAPI async def get_image_message(base64_image, engine = None): if "gpt" == engine: @@ -748,6 +748,25 @@ async def get_claude_payload(request, engine, provider): return url, headers, payload +async def get_dalle_payload(request, engine, provider): + model = provider['model'][request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api']}" + url = provider['base_url'] + url = BaseAPI(url).image_url + + payload = { + "model": model, + "prompt": request.prompt, + "n": request.n, + "size": request.size + } + + return url, headers, payload + async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) @@ -761,5 +780,7 @@ async def get_payload(request: RequestModel, engine, provider): return await get_gpt_payload(request, engine, provider) elif engine == "openrouter": return await get_openrouter_payload(request, engine, provider) + elif engine == "dalle": + return await get_dalle_payload(request, engine, provider) else: raise ValueError("Unknown payload") \ No newline at end of file diff --git a/response.py b/response.py index 051789fc..6a484f73 100644 --- a/response.py +++ b/response.py @@ -36,17 +36,24 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f return sse_response +async def check_response(response, error_log): + if response.status_code != 200: + error_message = await response.aread() + error_str = error_message.decode('utf-8', errors='replace') + try: + error_json = json.loads(error_str) + except json.JSONDecodeError: + error_json = error_str + return {"error": f"{error_log} HTTP Error {response.status_code}", "details": error_json} + return None + async def fetch_gemini_response_stream(client, url, headers, payload, model): timestamp = datetime.timestamp(datetime.now()) async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_gemini_response_stream") + if error_message: + yield error_message + return buffer = "" revicing_function_call = False function_full_response = "{" @@ -87,14 +94,11 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): timestamp = datetime.timestamp(datetime.now()) async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_vertex_claude_response_stream") + if error_message: + yield error_message + return + buffer = "" revicing_function_call = False function_full_response = "{" @@ -138,14 +142,9 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects while redirect_count < max_redirects: # logger.info(f"fetch_gpt_response_stream: {url}") async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_gpt_response_stream") + if error_message: + yield error_message return buffer = "" @@ -185,14 +184,10 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects async def fetch_claude_response_stream(client, url, headers, payload, model): timestamp = datetime.timestamp(datetime.now()) async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_claude_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_claude_response_stream") + if error_message: + yield error_message + return buffer = "" async for chunk in response.aiter_text(): # logger.info(f"chunk: {repr(chunk)}") @@ -241,13 +236,12 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): yield sse_string async def fetch_response(client, url, headers, payload): - try: - response = await client.post(url, headers=headers, json=payload) - return response.json() - except httpx.ConnectError as e: - return {"error": f"500", "details": "fetch_response Connect Error"} - except httpx.ReadTimeout as e: - return {"error": f"500", "details": "fetch_response Read Response Timeout"} + response = await client.post(url, headers=headers, json=payload) + error_message = await check_response(response, "fetch_response") + if error_message: + yield error_message + return + yield response.json() async def fetch_response_stream(client, url, headers, payload, engine, model): try: diff --git a/utils.py b/utils.py index d07dce34..231d61e1 100644 --- a/utils.py +++ b/utils.py @@ -222,4 +222,30 @@ def next(self): c3s = CircularList(["us-east5", "us-central1", "asia-southeast1"]) c3o = CircularList(["us-east5"]) c3h = CircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) -gem = CircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) \ No newline at end of file +gem = CircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) + +class BaseAPI: + def __init__( + self, + api_url: str = "https://api.openai.com/v1/chat/completions", + ): + if api_url == "": + api_url = "https://api.openai.com/v1/chat/completions" + self.source_api_url: str = api_url + from urllib.parse import urlparse, urlunparse + parsed_url = urlparse(self.source_api_url) + if parsed_url.scheme == "": + raise Exception("Error: API_URL is not set") + if parsed_url.path != '/': + before_v1 = parsed_url.path.split("/v1")[0] + else: + before_v1 = "" + self.base_url: str = urlunparse(parsed_url[:2] + (before_v1,) + ("",) * 3) + self.v1_url: str = urlunparse(parsed_url[:2]+ (before_v1 + "/v1",) + ("",) * 3) + self.v1_models: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/models",) + ("",) * 3) + if parsed_url.netloc == "api.deepseek.com": + self.chat_url: str = urlunparse(parsed_url[:2] + ("/chat/completions",) + ("",) * 3) + else: + self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/chat/completions",) + ("",) * 3) + self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) + self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) \ No newline at end of file From 894ff248555e46fac91f18e7dd30c641a6523601 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Sep 2024 03:22:40 +0800 Subject: [PATCH 005/476] Fix the bug where Claude cannot use tool use. --- request.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/request.py b/request.py index 3777772f..4976b9cc 100644 --- a/request.py +++ b/request.py @@ -387,9 +387,10 @@ async def get_vertex_claude_payload(request, engine, provider): messages = [] system_prompt = None + tool_id = None for msg in request.messages: - tool_calls = None tool_call_id = None + tool_calls = None if isinstance(msg.content, list): content = [] for item in msg.content: @@ -402,6 +403,7 @@ async def get_vertex_claude_payload(request, engine, provider): else: content = msg.content tool_calls = msg.tool_calls + tool_id = tool_calls[0].id if tool_calls else None or tool_id tool_call_id = msg.tool_call_id if tool_calls: @@ -414,11 +416,10 @@ async def get_vertex_claude_payload(request, engine, provider): "input": json.loads(tool_call.function.arguments), }) messages.append({"role": msg.role, "content": tool_calls_list}) - elif tool_call_id and tool_calls: - tool_call = tool_calls[0] + elif tool_call_id: messages.append({"role": "user", "content": [{ "type": "tool_result", - "tool_use_id": tool_call.id, + "tool_use_id": tool_id, "content": content }]}) elif msg.role != "system": @@ -650,9 +651,10 @@ async def get_claude_payload(request, engine, provider): messages = [] system_prompt = None + tool_id = None for msg in request.messages: - tool_calls = None tool_call_id = None + tool_calls = None if isinstance(msg.content, list): content = [] for item in msg.content: @@ -665,6 +667,7 @@ async def get_claude_payload(request, engine, provider): else: content = msg.content tool_calls = msg.tool_calls + tool_id = tool_calls[0].id if tool_calls else None or tool_id tool_call_id = msg.tool_call_id if tool_calls: @@ -677,11 +680,10 @@ async def get_claude_payload(request, engine, provider): "input": json.loads(tool_call.function.arguments), }) messages.append({"role": msg.role, "content": tool_calls_list}) - elif tool_call_id and tool_calls: - tool_call = tool_calls[0] + elif tool_call_id: messages.append({"role": "user", "content": [{ "type": "tool_result", - "tool_use_id": tool_call.id, + "tool_use_id": tool_id, "content": content }]}) elif msg.role != "system": From 2a7fbb20be0aa150ca27be668455737032dd75f8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 01:18:37 +0800 Subject: [PATCH 006/476] Fix the bug that does not adapt to the new API for obtaining the model list. --- README.md | 7 +++++++ main.py | 14 ++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bc0d376f..8241d325 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ api_keys: AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true ``` +## 环境变量 + +- CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 +- TIMEOUT: 请求超时时间，默认为 20 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 + ## Docker Local Deployment Start the container @@ -119,6 +124,8 @@ services: - ./api.yaml:/home/api.yaml ``` +CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。 + Run Docker Compose container in the background ```bash diff --git a/main.py b/main.py index 1085f6fd..2822d0ef 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,9 @@ @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 - timeout = httpx.Timeout(connect=15.0, read=20.0, write=30.0, pool=30.0) + import os + TIMEOUT = os.getenv("TIMEOUT", 20) + timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) default_headers = { "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent "Accept": "*/*", # curl 的默认 Accept 头 @@ -239,7 +241,7 @@ async def request_model(request: Union[RequestModel, ImageGenerationRequest], to async def options_handler(): return JSONResponse(status_code=200, content={"detail": "OPTIONS allowed"}) -@app.post("/v1/models") +@app.get("/v1/models") async def list_models(token: str = Depends(verify_api_key)): models = post_all_models(token, app.state.config, app.state.api_list) return JSONResponse(content={ @@ -247,14 +249,6 @@ async def list_models(token: str = Depends(verify_api_key)): "data": models }) -@app.get("/v1/models") -async def list_models(): - models = get_all_models(config=app.state.config) - return JSONResponse(content={ - "object": "list", - "data": models - }) - @app.post("/v1/images/generations") async def images_generations( request: ImageGenerationRequest, From 819abc097a3075dc898d81e1b122773372d0c906 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 04:09:46 +0800 Subject: [PATCH 007/476] Add traffic middleware --- log_config.py | 3 +- main.py | 162 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 116 insertions(+), 49 deletions(-) diff --git a/log_config.py b/log_config.py index 8bf3c4e3..9a1aa037 100644 --- a/log_config.py +++ b/log_config.py @@ -2,4 +2,5 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") logger = logging.getLogger("uni-api") -logging.getLogger("httpx").setLevel(logging.CRITICAL) \ No newline at end of file +logging.getLogger("httpx").setLevel(logging.CRITICAL) +logging.getLogger("watchfiles.main").setLevel(logging.CRITICAL) \ No newline at end of file diff --git a/main.py b/main.py index 2822d0ef..8058a1d2 100644 --- a/main.py +++ b/main.py @@ -5,14 +5,14 @@ from contextlib import asynccontextmanager from fastapi.middleware.cors import CORSMiddleware -from fastapi import FastAPI, HTTPException, Depends +from fastapi import FastAPI, HTTPException, Depends, Request from fastapi.responses import StreamingResponse, JSONResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from models import RequestModel, ImageGenerationRequest -from utils import error_handling_wrapper, get_all_models, post_all_models, load_config from request import get_payload from response import fetch_response, fetch_response_stream +from utils import error_handling_wrapper, post_all_models, load_config from typing import List, Dict, Union from urllib.parse import urlparse @@ -42,36 +42,91 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) -# from time import time -# from collections import defaultdict -# import asyncio - -# class StatsMiddleware: -# def __init__(self): -# self.request_counts = defaultdict(int) -# self.request_times = defaultdict(float) -# self.ip_counts = defaultdict(lambda: defaultdict(int)) -# self.lock = asyncio.Lock() - -# async def __call__(self, request: Request, call_next): -# start_time = time() -# response = await call_next(request) -# process_time = time() - start_time - -# endpoint = f"{request.method} {request.url.path}" -# client_ip = request.client.host - -# async with self.lock: -# self.request_counts[endpoint] += 1 -# self.request_times[endpoint] += process_time -# self.ip_counts[endpoint][client_ip] += 1 - -# return response -# # 创建 StatsMiddleware 实例 -# stats_middleware = StatsMiddleware() - -# # 添加 StatsMiddleware -# app.add_middleware(StatsMiddleware) +import asyncio +from time import time +from collections import defaultdict +from starlette.middleware.base import BaseHTTPMiddleware +from datetime import datetime +from datetime import timedelta +import json +import aiofiles + +class StatsMiddleware(BaseHTTPMiddleware): + def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats.json"): + super().__init__(app) + self.request_counts = defaultdict(int) + self.request_times = defaultdict(float) + self.ip_counts = defaultdict(lambda: defaultdict(int)) + self.request_arrivals = defaultdict(list) + self.lock = asyncio.Lock() + self.exclude_paths = set(exclude_paths or []) + self.save_interval = save_interval + self.filename = filename + self.last_save_time = time() + + # 启动定期保存和清理任务 + asyncio.create_task(self.periodic_save_and_cleanup()) + + async def dispatch(self, request: Request, call_next): + arrival_time = datetime.now() + start_time = time() + response = await call_next(request) + process_time = time() - start_time + + endpoint = f"{request.method} {request.url.path}" + client_ip = request.client.host + + if request.url.path not in self.exclude_paths: + async with self.lock: + self.request_counts[endpoint] += 1 + self.request_times[endpoint] += process_time + self.ip_counts[endpoint][client_ip] += 1 + self.request_arrivals[endpoint].append(arrival_time) + + return response + + async def periodic_save_and_cleanup(self): + while True: + await asyncio.sleep(self.save_interval) + await self.save_stats() + await self.cleanup_old_data() + + async def save_stats(self): + current_time = time() + if current_time - self.last_save_time < self.save_interval: + return + + async with self.lock: + stats = { + "request_counts": dict(self.request_counts), + "request_times": dict(self.request_times), + "ip_counts": {k: dict(v) for k, v in self.ip_counts.items()}, + "request_arrivals": {k: [t.isoformat() for t in v] for k, v in self.request_arrivals.items()} + } + + filename = self.filename + async with aiofiles.open(filename, mode='w') as f: + await f.write(json.dumps(stats, indent=2)) + + self.last_save_time = current_time + # print(f"Stats saved to {filename}") + + async def cleanup_old_data(self): + # cutoff_time = datetime.now() - timedelta(seconds=30) + cutoff_time = datetime.now() - timedelta(hours=24) + async with self.lock: + for endpoint in list(self.request_arrivals.keys()): + self.request_arrivals[endpoint] = [ + t for t in self.request_arrivals[endpoint] if t > cutoff_time + ] + if not self.request_arrivals[endpoint]: + del self.request_arrivals[endpoint] + self.request_counts.pop(endpoint, None) + self.request_times.pop(endpoint, None) + self.ip_counts.pop(endpoint, None) + + async def cleanup(self): + await self.save_stats() # 配置 CORS 中间件 app.add_middleware( @@ -82,6 +137,8 @@ async def lifespan(app: FastAPI): allow_headers=["*"], # 允许所有头部字段 ) +app.add_middleware(StatsMiddleware, exclude_paths=["/stats", "/generate-api-key"]) + async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) @@ -233,6 +290,17 @@ def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security) raise HTTPException(status_code=403, detail="Invalid or missing API Key") return token +def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): + api_list = app.state.api_list + token = credentials.credentials + if token not in api_list: + raise HTTPException(status_code=403, detail="Invalid or missing API Key") + for api_key in app.state.api_keys_db: + if api_key['api'] == token: + if api_key.get('role') != "admin": + raise HTTPException(status_code=403, detail="Permission denied") + return token + @app.post("/v1/chat/completions") async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): return await model_handler.request_model(request, token) @@ -258,24 +326,22 @@ async def images_generations( @app.get("/generate-api-key") def generate_api_key(): - api_key = "sk-" + secrets.token_urlsafe(32) + api_key = "sk-" + secrets.token_urlsafe(36) return JSONResponse(content={"api_key": api_key}) -# @app.get("/stats") -# async def get_stats(token: str = Depends(verify_api_key)): -# async with stats_middleware.lock: -# return { -# "request_counts": dict(stats_middleware.request_counts), -# "average_request_times": { -# endpoint: total_time / count -# for endpoint, total_time in stats_middleware.request_times.items() -# for count in [stats_middleware.request_counts[endpoint]] -# }, -# "ip_counts": { -# endpoint: dict(ips) -# for endpoint, ips in stats_middleware.ip_counts.items() -# } -# } +@app.get("/stats") +async def get_stats(request: Request, token: str = Depends(verify_admin_api_key)): + middleware = app.middleware_stack.app + if isinstance(middleware, StatsMiddleware): + async with middleware.lock: + stats = { + "request_counts": dict(middleware.request_counts), + "request_times": dict(middleware.request_times), + "ip_counts": {k: dict(v) for k, v in middleware.ip_counts.items()}, + "request_arrivals": {k: [t.isoformat() for t in v] for k, v in middleware.request_arrivals.items()} + } + return JSONResponse(content=stats) + return {"error": "StatsMiddleware not found"} # async def on_fetch(request, env): # import asgi From 94c81e3f35e328589aa47747c8a17428dc1b619d Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 04:14:15 +0800 Subject: [PATCH 008/476] fixed bug: ModuleNotFoundError: No module named 'aiofiles' --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 40317098..00f087b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ pyyaml pytest uvicorn fastapi +aiofiles httpx[http2] cryptography \ No newline at end of file From 3fc76ba9f4f5921b8c2f7ebb7c509274508d0892 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 17:05:14 +0800 Subject: [PATCH 009/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20timeout=20was=20not=20converted=20to=20a=20flo?= =?UTF-8?q?at.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 8058a1d2..f3fba4c9 100644 --- a/main.py +++ b/main.py @@ -21,7 +21,7 @@ async def lifespan(app: FastAPI): # 启动时的代码 import os - TIMEOUT = os.getenv("TIMEOUT", 20) + TIMEOUT = float(os.getenv("TIMEOUT", 20)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) default_headers = { "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent From 7b515b4df1cf64ed1ccd7693c3c6a878590100d7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 19:12:37 +0800 Subject: [PATCH 010/476] Fix the bug where the Claude API does not return the DONE SSE message. --- response.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/response.py b/response.py index 6a484f73..357869d2 100644 --- a/response.py +++ b/response.py @@ -90,6 +90,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response = json.dumps(function_call["functionCall"]["args"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id="chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV", function_call_name=None, function_call_content=function_full_response) yield sse_string + yield "data: [DONE]\n\r" async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): timestamp = datetime.timestamp(datetime.now()) @@ -136,6 +137,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod function_full_response = json.dumps(function_call["input"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id=function_call_id, function_call_name=None, function_call_content=function_full_response) yield sse_string + yield "data: [DONE]\n\r" async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): redirect_count = 0 @@ -234,6 +236,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): function_call_content = delta["partial_json"] sse_string = await generate_sse_response(timestamp, model, None, None, None, function_call_content) yield sse_string + yield "data: [DONE]\n\r" async def fetch_response(client, url, headers, payload): response = await client.post(url, headers=headers, json=payload) From 163e912d041a284d0f990d90292bb0659c227f2f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 20:27:02 +0800 Subject: [PATCH 011/476] Fix the bug that cannot adapt to the available model name. --- main.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index f3fba4c9..405c29c7 100644 --- a/main.py +++ b/main.py @@ -222,13 +222,25 @@ def get_matching_providers(self, model_name, token): # print("provider_rules", provider_rules) for item in provider_rules: for provider in config['providers']: - if provider['provider'] == item: - if "/" in item: - if item.split("/")[1] == model_name: - provider_list.append(provider) - else: - if model_name in provider['model'].keys(): + # print("provider", provider, provider['provider'] == item, item) + if "/" in item: + if provider['provider'] == item.split("/")[0]: + if model_name in provider['model'].keys() and item.split("/")[1] == model_name: provider_list.append(provider) + elif provider['provider'] == item: + if model_name in provider['model'].keys(): + provider_list.append(provider) + else: + pass + + # if provider['provider'] == item: + # if "/" in item: + # if item.split("/")[1] == model_name: + # provider_list.append(provider) + # else: + # if model_name in provider['model'].keys(): + # provider_list.append(provider) + # import json # for provider in provider_list: # print(json.dumps(provider, indent=4, ensure_ascii=False)) From f3fce0a76c9cc3c669d4b4e7133947e145b189b8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 20:46:30 +0800 Subject: [PATCH 012/476] Fix the bug where claude stream is not defined. --- request.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/request.py b/request.py index 4976b9cc..8df44e8f 100644 --- a/request.py +++ b/request.py @@ -40,8 +40,7 @@ async def get_gemini_payload(request, engine, provider): 'Content-Type': 'application/json' } model = provider['model'][request.model] - if request.stream: - gemini_stream = "streamGenerateContent" + gemini_stream = "streamGenerateContent" url = provider['base_url'] if url.endswith("v1beta"): url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api']) @@ -232,8 +231,7 @@ async def get_vertex_gemini_payload(request, engine, provider): if provider.get("project_id"): project_id = provider.get("project_id") - if request.stream: - gemini_stream = "streamGenerateContent" + gemini_stream = "streamGenerateContent" model = provider['model'][request.model] location = gem url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) @@ -381,8 +379,7 @@ async def get_vertex_claude_payload(request, engine, provider): elif "claude-3-haiku" in model: location = c3h - if request.stream: - claude_stream = "streamRawPredict" + claude_stream = "streamRawPredict" url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/anthropic/models/{MODEL}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL=model, stream=claude_stream) messages = [] From aeec83c903a464ee3762278a151ede2c7fd782f0 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 4 Sep 2024 23:45:01 +0800 Subject: [PATCH 013/476] Add feature: support load balancing for multiple keys in a single channel, enabled by default. --- README.md | 7 +-- request.py | 12 ++--- test/test_nostream.py | 121 ++++++++++++++++++++++++++++++++++++++++++ utils.py | 8 +++ 4 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 test/test_nostream.py diff --git a/README.md b/README.md index 8241d325..63d8b11f 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,9 @@ - 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 -- 支持负载均衡，支持 Vertex 区域负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。除了 Vertex 区域负载均衡，所有 API 均支持渠道级负载均衡，提高沉浸式翻译体验。 +- 支持三种负载均衡，默认同时开启。1. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级负载均衡，提高沉浸式翻译体验。 - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 -- 支持多个 API Key。 ## Configuration @@ -42,7 +41,9 @@ providers: - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: sk-ant-api03-bNnAOJyA-xQw_twAA + api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + - sk-ant-api03-bNnAOJyA-xQw_twAA + - sk-ant-api02-bNnxxxx model: - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 tools: true # 是否支持工具，如生成代码、生成文档等，默认是 true，选填 diff --git a/request.py b/request.py index 8df44e8f..89602315 100644 --- a/request.py +++ b/request.py @@ -43,9 +43,9 @@ async def get_gemini_payload(request, engine, provider): gemini_stream = "streamGenerateContent" url = provider['base_url'] if url.endswith("v1beta"): - url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api']) + url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api'].next()) if url.endswith("v1"): - url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api']) + url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api'].next()) messages = [] systemInstruction = None @@ -492,7 +492,7 @@ async def get_gpt_payload(request, engine, provider): 'Content-Type': 'application/json', } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api']}" + headers['Authorization'] = f"Bearer {provider['api'].next()}" url = provider['base_url'] messages = [] @@ -556,7 +556,7 @@ async def get_openrouter_payload(request, engine, provider): 'Content-Type': 'application/json' } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api']}" + headers['Authorization'] = f"Bearer {provider['api'].next()}" url = provider['base_url'] @@ -640,7 +640,7 @@ async def get_claude_payload(request, engine, provider): model = provider['model'][request.model] headers = { "content-type": "application/json", - "x-api-key": f"{provider['api']}", + "x-api-key": f"{provider['api'].next()}", "anthropic-version": "2023-06-01", "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" if "claude-3-5-sonnet" in model else "tools-2024-05-16", } @@ -753,7 +753,7 @@ async def get_dalle_payload(request, engine, provider): "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api']}" + headers['Authorization'] = f"Bearer {provider['api'].next()}" url = provider['base_url'] url = BaseAPI(url).image_url diff --git a/test/test_nostream.py b/test/test_nostream.py new file mode 100644 index 00000000..7378d7c7 --- /dev/null +++ b/test/test_nostream.py @@ -0,0 +1,121 @@ +import requests +import base64 +import json +import os +from datetime import datetime + +# 設置API密鑰和自定義base URL +API_KEY = '' +BASE_URL = 'http://localhost:8000/v1' +SAVE_DIR = 'safe_output' # 保存 JSON 輸出的目錄 + +def ensure_save_directory(): + if not os.path.exists(SAVE_DIR): + os.makedirs(SAVE_DIR) + +def image_to_base64(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + +def get_model_response(image_base64): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {API_KEY}" + } + + tools = [ + { + "type": "function", + "function": { + "name": "extract_underlined_text", + "description": "從圖片中提取紅色下劃線的文字", + "parameters": { + "type": "object", + "properties": { + "underlined_text": { + "type": "array", + "items": {"type": "string"}, + "description": "紅色下劃線的文字列表" + } + }, + "required": ["underlined_text"] + } + } + } + ] + + payload = { + + "model": "claude-3-5-sonnet", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "請仔細分析圖片，並提取所有使用紅色筆在單字、單詞或句子下方畫有橫線的文字。只提取有紅色下劃線的文字，忽略其他未標記的文字。將結果以 JSON 格式輸出，格式為 {\"underlined_text\": [\"文字1\", \"文字2\", ...]}。" + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64}" + } + } + ] + } + ], + "stream": True, + "tools": tools, + "tool_choice": {"type": "function", "function": {"name": "extract_underlined_text"}}, + "max_tokens": 300 + } + + try: + response = requests.post(f"{BASE_URL}/chat/completions", headers=headers, json=payload, timeout=30) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + return f"Error: {e}" + +def save_json_output(data): + ensure_save_directory() + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + filename = f"{SAVE_DIR}/output_{timestamp}.json" + with open(filename, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + return filename + +def main(image_path): + image_base64 = image_to_base64(image_path) + + response = get_model_response(image_base64) + + print("模型回應:") + print(json.dumps(response, indent=2, ensure_ascii=False)) + + if isinstance(response, str) and response.startswith("Error"): + print(response) + return + + if 'choices' in response and response['choices']: + message = response['choices'][0]['message'] + if 'tool_calls' in message: + tool_call = message['tool_calls'][0] + if tool_call['function']['name'] == 'extract_underlined_text': + function_args = json.loads(tool_call['function']['arguments']) + print("\n提取的紅色下劃線文字:") + print(json.dumps(function_args, indent=2, ensure_ascii=False)) + + # 保存 JSON 輸出 + saved_file = save_json_output(function_args) + print(f"\nJSON 輸出已保存至: {saved_file}") + else: + print("\n模型調用了未預期的函數。") + else: + print("\n模型沒有調用工具。") + else: + print("\n無法解析回應。") + +if __name__ == "__main__": + image_path = "00001 (8).jpg" # 替換為您的圖像路徑 + main(image_path) \ No newline at end of file diff --git a/utils.py b/utils.py index 231d61e1..f27154ed 100644 --- a/utils.py +++ b/utils.py @@ -15,7 +15,15 @@ def update_config(config_data): provider['model'] = model_dict if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' + + if provider.get('api'): + if isinstance(provider.get('api'), str): + provider['api'] = CircularList([provider.get('api')]) + if isinstance(provider.get('api'), list): + provider['api'] = CircularList(provider.get('api')) + config_data['providers'][index] = provider + api_keys_db = config_data['api_keys'] api_list = [item["api"] for item in api_keys_db] # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False)) From 7f8be8fe08df31fb0c605a5cdf9f79ddd5d5ee0f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 5 Sep 2024 00:17:22 +0800 Subject: [PATCH 014/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20the=20max=5Ftokens=20is=20missing=20in=20the=20Cl?= =?UTF-8?q?aude=20request=20body?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where claude tool_choice does not support "tool_choice": {"type": "function", "function": {"name": "extract_underlined_text"}} --- request.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/request.py b/request.py index 89602315..d68c8108 100644 --- a/request.py +++ b/request.py @@ -447,14 +447,11 @@ async def get_vertex_claude_payload(request, engine, provider): "anthropic_version": "vertex-2023-10-16", "messages": messages, "system": system_prompt or "You are Claude, a large language model trained by Anthropic.", + "max_tokens": 8192 if "claude-3-5-sonnet" in model else 4096, } - # 檢查是否需要添加 max_tokens - if 'max_tokens' not in payload: - if "claude-3-5-sonnet" in model: - payload['max_tokens'] = 8192 - elif "claude-3" in model: # 處理其他 Claude 3 模型 - payload['max_tokens'] = 4096 + if request.max_tokens: + payload["max_tokens"] = int(request.max_tokens) miss_fields = [ 'model', @@ -477,9 +474,15 @@ async def get_vertex_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - payload["tool_choice"] = { - "type": "auto" - } + if payload["tool_choice"]["type"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } if provider.get("tools") == False: payload.pop("tools", None) @@ -711,8 +714,12 @@ async def get_claude_payload(request, engine, provider): "model": model, "messages": messages, "system": system_prompt or "You are Claude, a large language model trained by Anthropic.", + "max_tokens": 8192 if "claude-3-5-sonnet" in model else 4096, } + if request.max_tokens: + payload["max_tokens"] = int(request.max_tokens) + miss_fields = [ 'model', 'messages', @@ -735,9 +742,15 @@ async def get_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - payload["tool_choice"] = { - "type": "auto" - } + if payload["tool_choice"]["type"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } if provider.get("tools") == False: payload.pop("tools", None) From 0ce2715a821933136748da05b2ae0311c27166e1 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 5 Sep 2024 00:19:33 +0800 Subject: [PATCH 015/476] Fix the bug where claude tool_choice does not support "tool_choice": {"type": "any"} --- request.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/request.py b/request.py index d68c8108..800a3fee 100644 --- a/request.py +++ b/request.py @@ -478,6 +478,10 @@ async def get_vertex_claude_payload(request, engine, provider): payload["tool_choice"] = { "type": "auto" } + if payload["tool_choice"]["type"] == "any": + payload["tool_choice"] = { + "type": "any" + } if payload["tool_choice"]["type"] == "function": payload["tool_choice"] = { "type": "tool", @@ -746,6 +750,10 @@ async def get_claude_payload(request, engine, provider): payload["tool_choice"] = { "type": "auto" } + if payload["tool_choice"]["type"] == "any": + payload["tool_choice"] = { + "type": "any" + } if payload["tool_choice"]["type"] == "function": payload["tool_choice"] = { "type": "tool", From 44caf41402b7082a02285cff93f52948f4fdeb32 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 5 Sep 2024 01:41:48 +0800 Subject: [PATCH 016/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20tool=20use=20request=20body=20format=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- models.py | 13 ++++++---- request.py | 56 +++++++++++++++++++++++-------------------- test/test_nostream.py | 5 ++-- 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 83edc3dd..b471fc8c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ __pycache__ .vscode node_modules .wrangler -.pytest_cache \ No newline at end of file +.pytest_cache +*.jpg +*.json \ No newline at end of file diff --git a/models.py b/models.py index 11eb949a..44887d90 100644 --- a/models.py +++ b/models.py @@ -10,16 +10,14 @@ class ImageGenerationRequest(BaseModel): class FunctionParameter(BaseModel): type: str - properties: Dict[str, Dict[str, str]] + properties: Dict[str, Dict[str, Union[str, Dict[str, str]]]] required: List[str] -# 定义 Function 模型 class Function(BaseModel): name: str description: str parameters: Optional[FunctionParameter] = Field(default=None, exclude=None) -# 定义 Tool 模型 class Tool(BaseModel): type: str function: Function @@ -58,6 +56,13 @@ class Message(BaseModel): class Config: extra = "allow" # 允许额外的字段 +class FunctionChoice(BaseModel): + name: str + +class ToolChoice(BaseModel): + type: str + function: Optional[FunctionChoice] = None + class RequestModel(BaseModel): model: str messages: List[Message] @@ -72,5 +77,5 @@ class RequestModel(BaseModel): frequency_penalty: Optional[float] = 0.0 n: Optional[int] = 1 user: Optional[str] = None - tool_choice: Optional[str] = None + tool_choice: Optional[Union[str, ToolChoice]] = None tools: Optional[List[Tool]] = None \ No newline at end of file diff --git a/request.py b/request.py index 800a3fee..765c66fb 100644 --- a/request.py +++ b/request.py @@ -474,19 +474,21 @@ async def get_vertex_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - if payload["tool_choice"]["type"] == "auto": - payload["tool_choice"] = { - "type": "auto" - } - if payload["tool_choice"]["type"] == "any": - payload["tool_choice"] = { - "type": "any" - } - if payload["tool_choice"]["type"] == "function": - payload["tool_choice"] = { - "type": "tool", - "name": payload["tool_choice"]["function"]["name"] - } + if isinstance(payload["tool_choice"], dict): + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } + if isinstance(payload["tool_choice"], str): + if payload["tool_choice"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"] == "none": + payload["tool_choice"] = { + "type": "any" + } if provider.get("tools") == False: payload.pop("tools", None) @@ -746,19 +748,21 @@ async def get_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - if payload["tool_choice"]["type"] == "auto": - payload["tool_choice"] = { - "type": "auto" - } - if payload["tool_choice"]["type"] == "any": - payload["tool_choice"] = { - "type": "any" - } - if payload["tool_choice"]["type"] == "function": - payload["tool_choice"] = { - "type": "tool", - "name": payload["tool_choice"]["function"]["name"] - } + if isinstance(payload["tool_choice"], dict): + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } + if isinstance(payload["tool_choice"], str): + if payload["tool_choice"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"] == "none": + payload["tool_choice"] = { + "type": "any" + } if provider.get("tools") == False: payload.pop("tools", None) diff --git a/test/test_nostream.py b/test/test_nostream.py index 7378d7c7..0ae7642f 100644 --- a/test/test_nostream.py +++ b/test/test_nostream.py @@ -45,7 +45,6 @@ def get_model_response(image_base64): ] payload = { - "model": "claude-3-5-sonnet", "messages": [ { @@ -64,7 +63,7 @@ def get_model_response(image_base64): ] } ], - "stream": True, + # "stream": True, "tools": tools, "tool_choice": {"type": "function", "function": {"name": "extract_underlined_text"}}, "max_tokens": 300 @@ -117,5 +116,5 @@ def main(image_path): print("\n無法解析回應。") if __name__ == "__main__": - image_path = "00001 (8).jpg" # 替換為您的圖像路徑 + image_path = "1.jpg" # 替換為您的圖像路徑 main(image_path) \ No newline at end of file From 73a667ff651f3cd2dba4b03100bb35ca4e7169c8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 5 Sep 2024 16:16:13 +0800 Subject: [PATCH 017/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20features:?= =?UTF-8?q?=20Add=20API=20channel=20success=20rate=20statistics,=20channel?= =?UTF-8?q?=20status=20records.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 85 ++++++++++++++++++++++++++++++++----------- request.py | 12 +++--- response.py | 4 +- test/test_nostream.py | 2 +- 4 files changed, 72 insertions(+), 31 deletions(-) diff --git a/main.py b/main.py index 405c29c7..1b2b73a2 100644 --- a/main.py +++ b/main.py @@ -58,6 +58,8 @@ def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats. self.request_times = defaultdict(float) self.ip_counts = defaultdict(lambda: defaultdict(int)) self.request_arrivals = defaultdict(list) + self.channel_success_counts = defaultdict(int) + self.channel_failure_counts = defaultdict(int) self.lock = asyncio.Lock() self.exclude_paths = set(exclude_paths or []) self.save_interval = save_interval @@ -101,7 +103,11 @@ async def save_stats(self): "request_counts": dict(self.request_counts), "request_times": dict(self.request_times), "ip_counts": {k: dict(v) for k, v in self.ip_counts.items()}, - "request_arrivals": {k: [t.isoformat() for t in v] for k, v in self.request_arrivals.items()} + "request_arrivals": {k: [t.isoformat() for t in v] for k, v in self.request_arrivals.items()}, + "channel_success_counts": dict(self.channel_success_counts), + "channel_failure_counts": dict(self.channel_failure_counts), + "channel_success_percentages": self.calculate_success_percentages(), + "channel_failure_percentages": self.calculate_failure_percentages() } filename = self.filename @@ -109,10 +115,28 @@ async def save_stats(self): await f.write(json.dumps(stats, indent=2)) self.last_save_time = current_time - # print(f"Stats saved to {filename}") + + def calculate_success_percentages(self): + percentages = {} + for channel, success_count in self.channel_success_counts.items(): + total_count = success_count + self.channel_failure_counts[channel] + if total_count > 0: + percentages[channel] = success_count / total_count * 100 + else: + percentages[channel] = 0 + return percentages + + def calculate_failure_percentages(self): + percentages = {} + for channel, failure_count in self.channel_failure_counts.items(): + total_count = failure_count + self.channel_success_counts[channel] + if total_count > 0: + percentages[channel] = failure_count / total_count * 100 + else: + percentages[channel] = 0 + return percentages async def cleanup_old_data(self): - # cutoff_time = datetime.now() - timedelta(seconds=30) cutoff_time = datetime.now() - timedelta(hours=24) async with self.lock: for endpoint in list(self.request_arrivals.keys()): @@ -139,10 +163,10 @@ async def cleanup(self): app.add_middleware(StatsMiddleware, exclude_paths=["/stats", "/generate-api-key"]) +# 在 process_request 函数中更新成功和失败计数 async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) - # print(parsed_url) engine = None if parsed_url.netloc == 'generativelanguage.googleapis.com': engine = "gemini" @@ -160,6 +184,12 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], and "gemini" not in provider['model'][request.model]: engine = "openrouter" + if "claude" in provider['model'][request.model] and engine == "vertex": + engine = "vertex-claude" + + if "gemini" in provider['model'][request.model] and engine == "vertex": + engine = "vertex-gemini" + if endpoint == "/v1/images/generations": engine = "dalle" request.stream = False @@ -171,21 +201,28 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], url, headers, payload = await get_payload(request, engine, provider) - # request_info = { - # "url": url, - # "headers": headers, - # "payload": payload - # } - # import json - # logger.info(f"Request details: {json.dumps(request_info, indent=4, ensure_ascii=False)}") - - if request.stream: - model = provider['model'][request.model] - generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) - wrapped_generator = await error_handling_wrapper(generator, status_code=500) - return StreamingResponse(wrapped_generator, media_type="text/event-stream") - else: - return await anext(fetch_response(app.state.client, url, headers, payload)) + try: + if request.stream: + model = provider['model'][request.model] + generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) + wrapped_generator = await error_handling_wrapper(generator, status_code=500) + response = StreamingResponse(wrapped_generator, media_type="text/event-stream") + else: + response = await anext(fetch_response(app.state.client, url, headers, payload)) + + # 更新成功计数 + async with app.middleware_stack.app.lock: + app.middleware_stack.app.channel_success_counts[provider['provider']] += 1 + + return response + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: + logger.error(f"Error with provider {provider['provider']}: {str(e)}") + + # 更新失败计数 + async with app.middleware_stack.app.lock: + app.middleware_stack.app.channel_failure_counts[provider['provider']] += 1 + + raise e import asyncio class ModelRequestHandler: @@ -270,10 +307,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint) + # 在 try_all_providers 函数中处理失败的情况 async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None): num_providers = len(providers) start_index = self.last_provider_index + 1 if use_round_robin else 0 - for i in range(num_providers + 1): self.last_provider_index = (start_index + i) % num_providers provider = providers[self.last_provider_index] @@ -287,7 +324,6 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe else: raise HTTPException(status_code=500, detail="Error: Current provider response failed!") - raise HTTPException(status_code=500, detail=f"All providers failed: {request.model}") model_handler = ModelRequestHandler() @@ -341,6 +377,7 @@ def generate_api_key(): api_key = "sk-" + secrets.token_urlsafe(36) return JSONResponse(content={"api_key": api_key}) +# 在 /stats 路由中返回成功和失败百分比 @app.get("/stats") async def get_stats(request: Request, token: str = Depends(verify_admin_api_key)): middleware = app.middleware_stack.app @@ -350,7 +387,11 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) "request_counts": dict(middleware.request_counts), "request_times": dict(middleware.request_times), "ip_counts": {k: dict(v) for k, v in middleware.ip_counts.items()}, - "request_arrivals": {k: [t.isoformat() for t in v] for k, v in middleware.request_arrivals.items()} + "request_arrivals": {k: [t.isoformat() for t in v] for k, v in middleware.request_arrivals.items()}, + "channel_success_counts": dict(middleware.channel_success_counts), + "channel_failure_counts": dict(middleware.channel_failure_counts), + "channel_success_percentages": middleware.calculate_success_percentages(), + "channel_failure_percentages": middleware.calculate_failure_percentages() } return JSONResponse(content=stats) return {"error": "StatsMiddleware not found"} diff --git a/request.py b/request.py index 765c66fb..cc20f790 100644 --- a/request.py +++ b/request.py @@ -10,7 +10,7 @@ async def get_image_message(base64_image, engine = None): "url": base64_image, } } - if "claude" == engine: + if "claude" == engine or "vertex-claude" == engine: return { "type": "image", "source": { @@ -19,7 +19,7 @@ async def get_image_message(base64_image, engine = None): "data": base64_image.split(",")[1], } } - if "gemini" == engine: + if "gemini" == engine or "vertex-gemini" == engine: return { "inlineData": { "mimeType": "image/jpeg", @@ -29,9 +29,9 @@ async def get_image_message(base64_image, engine = None): raise ValueError("Unknown engine") async def get_text_message(role, message, engine = None): - if "gpt" == engine or "claude" == engine or "openrouter" == engine: + if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine: return {"type": "text", "text": message} - if "gemini" == engine: + if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} raise ValueError("Unknown engine") @@ -794,9 +794,9 @@ async def get_dalle_payload(request, engine, provider): async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) - elif engine == "vertex" and "gemini" in provider['model'][request.model]: + elif engine == "vertex-gemini": return await get_vertex_gemini_payload(request, engine, provider) - elif engine == "vertex" and "claude" in provider['model'][request.model]: + elif engine == "vertex-claude": return await get_vertex_claude_payload(request, engine, provider) elif engine == "claude": return await get_claude_payload(request, engine, provider) diff --git a/response.py b/response.py index 357869d2..6d92e818 100644 --- a/response.py +++ b/response.py @@ -248,10 +248,10 @@ async def fetch_response(client, url, headers, payload): async def fetch_response_stream(client, url, headers, payload, engine, model): try: - if engine == "gemini" or (engine == "vertex" and "gemini" in model): + if engine == "gemini" or engine == "vertex-gemini": async for chunk in fetch_gemini_response_stream(client, url, headers, payload, model): yield chunk - elif engine == "claude" or (engine == "vertex" and "claude" in model): + elif engine == "claude" or engine == "vertex-claude": async for chunk in fetch_claude_response_stream(client, url, headers, payload, model): yield chunk elif engine == "gpt": diff --git a/test/test_nostream.py b/test/test_nostream.py index 0ae7642f..febb248f 100644 --- a/test/test_nostream.py +++ b/test/test_nostream.py @@ -66,7 +66,7 @@ def get_model_response(image_base64): # "stream": True, "tools": tools, "tool_choice": {"type": "function", "function": {"name": "extract_underlined_text"}}, - "max_tokens": 300 + "max_tokens": 1000 } try: From 3ec7a0bce06ab8e9a87bf90f329221d2b410b712 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 6 Sep 2024 03:09:16 +0800 Subject: [PATCH 018/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Add=20support=20for=20weighted=20load=20balancing.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++- README.md | 13 +++++++++- main.py | 53 ++++++++++++++++++++++++++++++++++++----- test/test_matplotlib.py | 49 +++++++++++++++++++++++++++++++++++++ test/test_weights.py | 33 +++++++++++++++++++++++++ utils.py | 42 ++++++++++++++++++++++++++++++-- 6 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 test/test_matplotlib.py create mode 100644 test/test_weights.py diff --git a/.gitignore b/.gitignore index b471fc8c..c026a68b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ node_modules .wrangler .pytest_cache *.jpg -*.json \ No newline at end of file +*.json +*.png \ No newline at end of file diff --git a/README.md b/README.md index 63d8b11f..2d488285 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ - 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 -- 支持三种负载均衡，默认同时开启。1. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级负载均衡，提高沉浸式翻译体验。 +- 支持四种负载均衡。1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 @@ -93,6 +93,17 @@ api_keys: preferences: USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true + + # 渠道级加权负载均衡配置示例 + - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + model: + - gcp1/*: 5 # 冒号后面就是权重，权重仅支持正整数。 + - gcp2/*: 3 # 数字的大小代表权重，数字越大，请求的概率越大。 + - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 + + preferences: + USE_ROUND_ROBIN: true # 当 USE_ROUND_ROBIN 必须为 true 并且上面的渠道后面没有权重时，会按照原始的渠道顺序请求，如果有权重，会按照加权后的顺序请求。 + AUTO_RETRY: true ``` ## 环境变量 diff --git a/main.py b/main.py index 1b2b73a2..42805064 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from models import RequestModel, ImageGenerationRequest from request import get_payload from response import fetch_response, fetch_response_stream -from utils import error_handling_wrapper, post_all_models, load_config +from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder from typing import List, Dict, Union from urllib.parse import urlparse @@ -224,6 +224,29 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], raise e +def weighted_round_robin(weights): + provider_names = list(weights.keys()) + current_weights = {name: 0 for name in provider_names} + num_selections = total_weight = sum(weights.values()) + weighted_provider_list = [] + + for _ in range(num_selections): + max_ratio = -1 + selected_letter = None + + for name in provider_names: + current_weights[name] += weights[name] + ratio = current_weights[name] / weights[name] + + if ratio > max_ratio: + max_ratio = ratio + selected_letter = name + + weighted_provider_list.append(selected_letter) + current_weights[selected_letter] -= total_weight + + return weighted_provider_list + import asyncio class ModelRequestHandler: def __init__(self): @@ -297,13 +320,31 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # 检查是否启用轮询 api_index = api_list.index(token) + weights = safe_get(config, 'api_keys', api_index, "weights") + if weights: + # 步骤 1: 提取 matching_providers 中的所有 provider 值 + providers = set(provider['provider'] for provider in matching_providers) + weight_keys = set(weights.keys()) + + # 步骤 3: 计算交集 + intersection = providers.intersection(weight_keys) + weights = dict(filter(lambda item: item[0] in intersection, weights.items())) + weighted_provider_name_list = weighted_round_robin(weights) + new_matching_providers = [] + for provider_name in weighted_provider_name_list: + for provider in matching_providers: + if provider['provider'] == provider_name: + new_matching_providers.append(provider) + matching_providers = new_matching_providers + # import json + # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False, default=circular_list_encoder)) + use_round_robin = True auto_retry = True - if config['api_keys'][api_index].get("preferences"): - if config['api_keys'][api_index]["preferences"].get("USE_ROUND_ROBIN") == False: - use_round_robin = False - if config['api_keys'][api_index]["preferences"].get("AUTO_RETRY") == False: - auto_retry = False + if safe_get(config, 'api_keys', api_index, "preferences", "USE_ROUND_ROBIN") == False: + use_round_robin = False + if safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY") == False: + auto_retry = False return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint) diff --git a/test/test_matplotlib.py b/test/test_matplotlib.py new file mode 100644 index 00000000..aea4c9e4 --- /dev/null +++ b/test/test_matplotlib.py @@ -0,0 +1,49 @@ +import json +import matplotlib.pyplot as plt +from datetime import datetime, timedelta +from collections import defaultdict + +import matplotlib.font_manager as fm +font_path = '/System/Library/Fonts/PingFang.ttc' +prop = fm.FontProperties(fname=font_path) +plt.rcParams['font.family'] = prop.get_name() + +with open('./test/states.json') as f: + data = json.load(f) + request_arrivals = data["request_arrivals"] + +def create_pic(request_arrivals, key): + request_arrivals = request_arrivals[key] + # 将字符串转换为datetime对象 + datetimes = [datetime.fromisoformat(t) for t in request_arrivals] + # 获取最新的时间 + latest_time = max(datetimes) + + # 创建24小时的时间范围 + time_range = [latest_time - timedelta(hours=i) for i in range(24, 0, -1)] + # 统计每小时的请求数 + hourly_counts = defaultdict(int) + for dt in datetimes: + for t in time_range[::-1]: + if dt >= t: + hourly_counts[t] += 1 + break + + # 准备绘图数据 + hours = [t.strftime('%Y-%m-%d %H:00') for t in time_range] + counts = [hourly_counts[t] for t in time_range] + + # 创建柱状图 + plt.figure(figsize=(15, 6)) + plt.bar(hours, counts) + plt.title(f'{key} 端点请求量 (过去24小时)') + plt.xlabel('时间') + plt.ylabel('请求数') + plt.xticks(rotation=45, ha='right') + plt.tight_layout() + + # 保存图片 + plt.savefig(f'{key.replace("/", "")}.png') + +if __name__ == '__main__': + create_pic(request_arrivals, 'POST /v1/chat/completions') \ No newline at end of file diff --git a/test/test_weights.py b/test/test_weights.py new file mode 100644 index 00000000..e1732cb2 --- /dev/null +++ b/test/test_weights.py @@ -0,0 +1,33 @@ +def weighted_round_robin(weights): + provider_names = list(weights.keys()) + current_weights = {name: 0 for name in provider_names} + num_selections = total_weight = sum(weights.values()) + weighted_provider_list = [] + + for _ in range(num_selections): + max_ratio = -1 + selected_letter = None + + for name in provider_names: + current_weights[name] += weights[name] + ratio = current_weights[name] / weights[name] + + if ratio > max_ratio: + max_ratio = ratio + selected_letter = name + + weighted_provider_list.append(selected_letter) + current_weights[selected_letter] -= total_weight + + return weighted_provider_list + +# 权重和选择次数 +weights = {'a': 5, 'b': 3, 'c': 2} +index = {'a', 'c'} + +result = dict(filter(lambda item: item[0] in index, weights.items())) +print(result) +# result = {k: weights[k] for k in index if k in weights} +# print(result) +weighted_provider_list = weighted_round_robin(weights) +print(weighted_provider_list) diff --git a/utils.py b/utils.py index f27154ed..9c2fa42b 100644 --- a/utils.py +++ b/utils.py @@ -25,8 +25,25 @@ def update_config(config_data): config_data['providers'][index] = provider api_keys_db = config_data['api_keys'] + + for index, api_key in enumerate(config_data['api_keys']): + weights_dict = {} + models = [] + for model in api_key.get('model'): + if isinstance(model, dict): + key, value = list(model.items())[0] + provider_name = key.split("/")[0] + if "/" in key: + weights_dict.update({provider_name: int(value)}) + models.append(key) + if isinstance(model, str): + models.append(model) + config_data['api_keys'][index]['weights'] = weights_dict + config_data['api_keys'][index]['model'] = models + api_keys_db[index]['model'] = models + api_list = [item["api"] for item in api_keys_db] - # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False)) + # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False, default=circular_list_encoder)) return config_data, api_keys_db, api_list # 读取YAML配置文件 @@ -214,6 +231,12 @@ def get_all_models(config): # us-central1 # europe-west1 # europe-west4 + +def circular_list_encoder(obj): + if isinstance(obj, CircularList): + return obj.to_dict() + raise TypeError(f'Object of type {obj.__class__.__name__} is not JSON serializable') + from collections import deque class CircularList: def __init__(self, items): @@ -226,6 +249,13 @@ def next(self): self.queue.append(item) return item + def to_dict(self): + return { + 'queue': list(self.queue) + } + + + c35s = CircularList(["us-east5", "europe-west1"]) c3s = CircularList(["us-east5", "us-central1", "asia-southeast1"]) c3o = CircularList(["us-east5"]) @@ -256,4 +286,12 @@ def __init__( else: self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/chat/completions",) + ("",) * 3) self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) - self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) \ No newline at end of file + self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) + +def safe_get(data, *keys): + for key in keys: + try: + data = data[key] if isinstance(data, (dict, list)) else data.get(key) + except (KeyError, IndexError, AttributeError, TypeError): + return None + return data \ No newline at end of file From dac9c70771b3f5b80609736be98faf4f953dc351 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 6 Sep 2024 03:22:48 +0800 Subject: [PATCH 019/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20channel=20status=20is=20not=20sorted=20by=20su?= =?UTF-8?q?ccess=20rate.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 8 ++++++-- test/test_matplotlib.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 42805064..726d7d5e 100644 --- a/main.py +++ b/main.py @@ -124,7 +124,9 @@ def calculate_success_percentages(self): percentages[channel] = success_count / total_count * 100 else: percentages[channel] = 0 - return percentages + + sorted_percentages = dict(sorted(percentages.items(), key=lambda item: item[1], reverse=True)) + return sorted_percentages def calculate_failure_percentages(self): percentages = {} @@ -134,7 +136,9 @@ def calculate_failure_percentages(self): percentages[channel] = failure_count / total_count * 100 else: percentages[channel] = 0 - return percentages + + sorted_percentages = dict(sorted(percentages.items(), key=lambda item: item[1], reverse=True)) + return sorted_percentages async def cleanup_old_data(self): cutoff_time = datetime.now() - timedelta(hours=24) diff --git a/test/test_matplotlib.py b/test/test_matplotlib.py index aea4c9e4..41601bc3 100644 --- a/test/test_matplotlib.py +++ b/test/test_matplotlib.py @@ -20,7 +20,7 @@ def create_pic(request_arrivals, key): latest_time = max(datetimes) # 创建24小时的时间范围 - time_range = [latest_time - timedelta(hours=i) for i in range(24, 0, -1)] + time_range = [latest_time - timedelta(hours=i) for i in range(32, 0, -1)] # 统计每小时的请求数 hourly_counts = defaultdict(int) for dt in datetimes: From 60e7a94ae60bc2ac9240d608a1baa3febc592431 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 6 Sep 2024 04:22:46 +0800 Subject: [PATCH 020/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20the=20OpenAI=20API=20key=20from=20being=20?= =?UTF-8?q?used=20normally=20in=20zed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/response.py b/response.py index 6d92e818..5d3f831c 100644 --- a/response.py +++ b/response.py @@ -32,7 +32,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f json_data = json.dumps(sample_data, ensure_ascii=False) # 构建SSE响应 - sse_response = f"data: {json_data}\n\r" + sse_response = f"data: {json_data}\n\r\n" return sse_response @@ -90,7 +90,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response = json.dumps(function_call["functionCall"]["args"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id="chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV", function_call_name=None, function_call_content=function_full_response) yield sse_string - yield "data: [DONE]\n\r" + yield "data: [DONE]\n\r\n" async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): timestamp = datetime.timestamp(datetime.now()) @@ -137,7 +137,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod function_full_response = json.dumps(function_call["input"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id=function_call_id, function_call_name=None, function_call_content=function_full_response) yield sse_string - yield "data: [DONE]\n\r" + yield "data: [DONE]\n\r\n" async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): redirect_count = 0 @@ -174,7 +174,7 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects line, buffer = buffer.split("\n", 1) # logger.info("line: %s", repr(line)) if line and line != "data: " and line != "data:" and not line.startswith(": "): - yield line + "\n\r" + yield line.strip() + "\n\r\n" except httpx.RemoteProtocolError as e: yield {"error": f"fetch_gpt_response_stream RemoteProtocolError {e.__class__.__name__}", "details": str(e)} return @@ -236,7 +236,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): function_call_content = delta["partial_json"] sse_string = await generate_sse_response(timestamp, model, None, None, None, function_call_content) yield sse_string - yield "data: [DONE]\n\r" + yield "data: [DONE]\n\r\n" async def fetch_response(client, url, headers, payload): response = await client.post(url, headers=headers, json=payload) From 8eca72ece8a055daa2869657477e9e91975721d6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 7 Sep 2024 00:38:25 +0800 Subject: [PATCH 021/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20of=20log=20errors=20being=20repeatedly=20displayed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug when the model list is empty. 💻 Code: Remove the redundant redirection code from the GPT model interface --- main.py | 6 ++---- response.py | 52 ++++++++++++---------------------------------------- utils.py | 25 +++++++++++++------------ 3 files changed, 27 insertions(+), 56 deletions(-) diff --git a/main.py b/main.py index 726d7d5e..df4107bc 100644 --- a/main.py +++ b/main.py @@ -220,8 +220,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: - logger.error(f"Error with provider {provider['provider']}: {str(e)}") - # 更新失败计数 async with app.middleware_stack.app.lock: app.middleware_stack.app.channel_failure_counts[provider['provider']] += 1 @@ -340,9 +338,9 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if provider['provider'] == provider_name: new_matching_providers.append(provider) matching_providers = new_matching_providers - # import json - # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False, default=circular_list_encoder)) + # import json + # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False, default=circular_list_encoder)) use_round_robin = True auto_retry = True if safe_get(config, 'api_keys', api_index, "preferences", "USE_ROUND_ROBIN") == False: diff --git a/response.py b/response.py index 5d3f831c..f4cc1270 100644 --- a/response.py +++ b/response.py @@ -140,48 +140,20 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod yield "data: [DONE]\n\r\n" async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): - redirect_count = 0 - while redirect_count < max_redirects: - # logger.info(f"fetch_gpt_response_stream: {url}") - async with client.stream('POST', url, headers=headers, json=payload) as response: - error_message = await check_response(response, "fetch_gpt_response_stream") - if error_message: - yield error_message - return - - buffer = "" - try: - async for chunk in response.aiter_text(): - # logger.info(f"chunk: {repr(chunk)}") - buffer += chunk - if chunk.startswith(" Date: Sat, 7 Sep 2024 00:55:48 +0800 Subject: [PATCH 022/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20tool=20use=20is=20still=20included=20in=20the=20requ?= =?UTF-8?q?est=20body=20when=20tools=20are=20closed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 ++ request.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index df4107bc..5172a0b2 100644 --- a/main.py +++ b/main.py @@ -205,6 +205,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], url, headers, payload = await get_payload(request, engine, provider) + # logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) + # logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) try: if request.stream: model = provider['model'][request.model] diff --git a/request.py b/request.py index cc20f790..88d44728 100644 --- a/request.py +++ b/request.py @@ -533,9 +533,11 @@ async def get_gpt_payload(request, engine, provider): "arguments": tool_call.function.arguments } }) - messages.append({"role": msg.role, "tool_calls": tool_calls_list}) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_calls": tool_calls_list}) elif tool_call_id: - messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) else: messages.append({"role": msg.role, "content": content}) From 7477ff749353a0ac29f96a2ff2c812f8a6256f1a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 7 Sep 2024 01:04:30 +0800 Subject: [PATCH 023/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20when=20the=20model=20list=20is=20empty.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 5172a0b2..4a183ff6 100644 --- a/main.py +++ b/main.py @@ -260,8 +260,9 @@ def get_matching_providers(self, model_name, token): config = app.state.config # api_keys_db = app.state.api_keys_db api_list = app.state.api_list - api_index = api_list.index(token) + if not safe_get(config, 'api_keys', api_index, 'model'): + raise HTTPException(status_code=404, detail="No matching model found") provider_rules = [] for model in config['api_keys'][api_index]['model']: From b812da1f10247daad64ead1564545678d942f1eb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 7 Sep 2024 05:29:08 +0800 Subject: [PATCH 024/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Add=20support=20for=20rate=20limiting.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++- main.py | 82 +++++++++++++++++++++++++++++++++++++---- test/test_rate_limit.py | 41 +++++++++++++++++++++ 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 test/test_rate_limit.py diff --git a/README.md b/README.md index 2d488285..3176e55d 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,14 @@ - 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 -- 支持四种负载均衡。1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 +- 支持四种负载均衡。 + 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 + 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 + 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。 + 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 +- 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 ## Configuration @@ -93,6 +98,7 @@ api_keys: preferences: USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true + RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 # 渠道级加权负载均衡配置示例 - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo diff --git a/main.py b/main.py index 4a183ff6..1450b7c2 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ from log_config import logger +import re import httpx import secrets +import time as time_module from contextlib import asynccontextmanager from fastapi.middleware.cors import CORSMiddleware @@ -14,6 +16,7 @@ from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder +from collections import defaultdict from typing import List, Dict, Union from urllib.parse import urlparse @@ -374,8 +377,73 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe model_handler = ModelRequestHandler() -# 安全性依赖 +def parse_rate_limit(limit_string): + # 定义时间单位到秒的映射 + time_units = { + 's': 1, 'sec': 1, 'second': 1, + 'm': 60, 'min': 60, 'minute': 60, + 'h': 3600, 'hr': 3600, 'hour': 3600, + 'd': 86400, 'day': 86400, + 'mo': 2592000, 'month': 2592000, + 'y': 31536000, 'year': 31536000 + } + + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit_string) + if not match: + raise ValueError(f"Invalid rate limit format: {limit_string}") + + count, unit = match.groups() + count = int(count) + + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") + + seconds = time_units[unit] + + return (count, seconds) + +class InMemoryRateLimiter: + def __init__(self): + self.requests = defaultdict(list) + + async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: + now = time_module.time() + self.requests[key] = [req for req in self.requests[key] if req > now - period] + if len(self.requests[key]) >= limit: + return True + self.requests[key].append(now) + return False + +rate_limiter = InMemoryRateLimiter() + +async def get_user_rate_limit(token: str = None): + # 这里应该实现根据 token 获取用户速率限制的逻辑 + # 示例： 返回 (次数， 秒数) + config = app.state.config + api_list = app.state.api_list + api_index = api_list.index(token) + raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") + + if not token or not raw_rate_limit: + return (60, 60) + + rate_limit = parse_rate_limit(raw_rate_limit) + return rate_limit + security = HTTPBearer() +async def rate_limit_dependency(request: Request, credentials: HTTPAuthorizationCredentials = Depends(security)): + token = credentials.credentials if credentials else None + # print("token", token) + limit, period = await get_user_rate_limit(token) + + # 使用 IP 地址和 token（如果有）作为限制键 + client_ip = request.client.host + rate_limit_key = f"{client_ip}:{token}" if token else client_ip + + if await rate_limiter.is_rate_limited(rate_limit_key, limit, period): + raise HTTPException(status_code=429, detail="Too many requests") def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): api_list = app.state.api_list @@ -395,15 +463,15 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec raise HTTPException(status_code=403, detail="Permission denied") return token -@app.post("/v1/chat/completions") +@app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): return await model_handler.request_model(request, token) -@app.options("/v1/chat/completions") +@app.options("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def options_handler(): return JSONResponse(status_code=200, content={"detail": "OPTIONS allowed"}) -@app.get("/v1/models") +@app.get("/v1/models", dependencies=[Depends(rate_limit_dependency)]) async def list_models(token: str = Depends(verify_api_key)): models = post_all_models(token, app.state.config, app.state.api_list) return JSONResponse(content={ @@ -411,20 +479,20 @@ async def list_models(token: str = Depends(verify_api_key)): "data": models }) -@app.post("/v1/images/generations") +@app.post("/v1/images/generations", dependencies=[Depends(rate_limit_dependency)]) async def images_generations( request: ImageGenerationRequest, token: str = Depends(verify_api_key) ): return await model_handler.request_model(request, token, endpoint="/v1/images/generations") -@app.get("/generate-api-key") +@app.get("/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) def generate_api_key(): api_key = "sk-" + secrets.token_urlsafe(36) return JSONResponse(content={"api_key": api_key}) # 在 /stats 路由中返回成功和失败百分比 -@app.get("/stats") +@app.get("/stats", dependencies=[Depends(rate_limit_dependency)]) async def get_stats(request: Request, token: str = Depends(verify_admin_api_key)): middleware = app.middleware_stack.app if isinstance(middleware, StatsMiddleware): diff --git a/test/test_rate_limit.py b/test/test_rate_limit.py new file mode 100644 index 00000000..48803203 --- /dev/null +++ b/test/test_rate_limit.py @@ -0,0 +1,41 @@ +import re + +def parse_rate_limit(limit_string): + # 定义时间单位到秒的映射 + time_units = { + 's': 1, 'sec': 1, 'second': 1, + 'm': 60, 'min': 60, 'minute': 60, + 'h': 3600, 'hr': 3600, 'hour': 3600, + 'd': 86400, 'day': 86400, + 'mo': 2592000, 'month': 2592000, + 'y': 31536000, 'year': 31536000 + } + + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit_string) + if not match: + raise ValueError(f"Invalid rate limit format: {limit_string}") + + count, unit = match.groups() + count = int(count) + + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") + + seconds = time_units[unit] + + return (count, seconds) + +# 测试函数 +test_cases = [ + "2/min", "5/hour", "10/day", "1/second", "3/mo", "1/year", + "20/s", "15/m", "8/h", "100/d", "50/mo", "2/y" +] + +for case in test_cases: + try: + result = parse_rate_limit(case) + print(f"{case} => {result}") + except ValueError as e: + print(f"Error parsing {case}: {str(e)}") \ No newline at end of file From fc8f1eeaecf0aa227a67befd771f2472a5128757 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 9 Sep 2024 02:11:13 +0800 Subject: [PATCH 025/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20the=20d?= =?UTF-8?q?ocumentation,=20add=20an=20English=20README.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 105 ++++++++++++----------- README_CN.md | 190 +++++++++++++++++++++++++++++++++++++++++ test/parse_markdown.py | 159 ++++++++++++++++++++++++++++++++++ test/translate_md.py | 50 +++++++++++ 4 files changed, 452 insertions(+), 52 deletions(-) create mode 100644 README_CN.md create mode 100644 test/parse_markdown.py create mode 100644 test/translate_md.py diff --git a/README.md b/README.md index 3176e55d..7f3bef61 100644 --- a/README.md +++ b/README.md @@ -1,71 +1,72 @@ # uni-api

- + - + docker pull

+[English](./README.md) | [Chinese](./README_CN.md) ## Introduction -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 +If used personally, one/new-api is too complex and has many commercial features that individuals do not need. If you don't want a complicated frontend interface and want to support more models, you can try uni-api. This is a project that manages large model APIs uniformly. It allows you to call multiple backend services through a unified API interface, converting them uniformly to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, DeepBricks, OpenRouter, etc. ## Features -- 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 -- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 -- 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 -- 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 -- 支持四种负载均衡。 - 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 - 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 - 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。 - 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 -- 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 -- 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 -- 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 +- No front-end, purely configuration file to set up API channels. You can run your own API site just by writing a file. The documentation has detailed configuration guidelines, friendly for beginners. +- Unified management of multiple backend services, supporting OpenAI, Deepseek, DeepBricks, OpenRouter, and other API providers in the OpenAI format. Supports OpenAI Dalle-3 image generation. +- Supports Anthropic, Gemini, Vertex API simultaneously. Vertex supports both Claude and Gemini API. +- Support native tool use function calls for OpenAI, Anthropic, Gemini, Vertex. +- Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. +- Supports four types of load balancing. + 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. By default, it is not enabled and requires channel weight configuration. + 2. Supports Vertex region-level load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. + 4. Support automatic API key-level polling load balancing for multiple API Keys in a single channel. +- Supports automatic retry. When an API channel response fails, automatically retry the next API channel. +- Supports fine-grained access control. Supports using wildcards to set specific models available for API key channels. +- Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. ## Configuration -使用 api.yaml 配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： +Using the api.yaml configuration file, you can configure multiple models, and each model can configure multiple backend services, supporting load balancing. Below is an example of the api.yaml configuration file: ```yaml providers: - - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 - base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 - api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 - model: # 至少填一个模型 - - gpt-4o # 可以使用的模型名称，必填 - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, arbitrary name, required + base_url: https://api.your.com/v1/chat/completions # Backend service API address, required + api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required + model: # At least one model + - gpt-4o # Usable model name, required + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 - tools: true # 是否支持工具，如生成代码、生成文档等，默认是 true，选填 + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional + tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini - base_url: https://generativelanguage.googleapis.com/v1beta # base_url 支持 v1beta/v1, 仅供 Gemini 模型使用，必填 + base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # 重命名后，原来的模型名字 gemini-1.5-flash-exp-0827 无法使用，如果要使用原来的名字，可以在 model 中添加原来的名字，只要加上下面一行就可以使用原来的名字了 - - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the following line to use the original name + - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个JSON格式的字符串，包含服务账号的私钥信息。获取方式： 在Google Cloud Console中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在Google Cloud Console的"IAM与管理"部分查看服务账号详情获得。 + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, you can also view the service account details in the "IAM & Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -74,7 +75,7 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + notes: https://xxxxx.com/ # Can put the provider's website, notes, official documentation, optional - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -82,40 +83,40 @@ providers: model: - causallm-35b-beta2ep-q6k: causallm-35b tools: false - engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 + engine: openrouter # Force the use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 - model: # 该 API Key 可以使用的模型，必填 - - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 - - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 - - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 + - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service + model: # Models that this API Key can use, required + - gpt-4o # Usable model name, can use all gpt-4o models provided by providers + - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers + - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models role: admin - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。 + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models of other providers' claude-3-5-sonnet cannot be used. preferences: - USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 - AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true - RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 + USE_ROUND_ROBIN: true # Whether to use polling load balancing, true to use, false to not use, default is true. When polling is enabled, each request model is requested in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request sequences for each API key. + AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true + RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional - # 渠道级加权负载均衡配置示例 + # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # 冒号后面就是权重，权重仅支持正整数。 - - gcp2/*: 3 # 数字的大小代表权重，数字越大，请求的概率越大。 - - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 + - gcp1/*: 5 # The number after the colon is the weight, weights only support positive integers. + - gcp2/*: 3 # The larger the number, the greater the probability of being requested. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # 当 USE_ROUND_ROBIN 必须为 true 并且上面的渠道后面没有权重时，会按照原始的渠道顺序请求，如果有权重，会按照加权后的顺序请求。 + USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the channels above, it will request in the original channel order, if there is weight, it will request in the weighted order. AUTO_RETRY: true ``` -## 环境变量 +## Environment variables -- CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 -- TIMEOUT: 请求超时时间，默认为 20 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +- CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional +- TIMEOUT: Request timeout, default is 20 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional ## Docker Local Deployment @@ -142,7 +143,7 @@ services: - ./api.yaml:/home/api.yaml ``` -CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。 +CONFIG_URL is a direct link that can automatically download remote configuration files. For example, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. Run Docker Compose container in the background diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 00000000..01bfc3e5 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,190 @@ +# uni-api + +

+ + + + + docker pull + +

+ +[英文](./README.md) | [中文](./README_CN.md) + +## Introduction + +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 + +## Features + +- 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 +- 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 +- 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 +- 支持四种负载均衡。 + 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 + 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 + 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。 + 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 +- 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 +- 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 +- 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 + +## Configuration + +使用 api.yaml 配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： + +```yaml +providers: + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 + base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 + api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 + model: # 至少填一个模型 + - gpt-4o # 可以使用的模型名称，必填 + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - dall-e-3 + + - provider: anthropic + base_url: https://api.anthropic.com/v1/messages + api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + - sk-ant-api03-bNnAOJyA-xQw_twAA + - sk-ant-api02-bNnxxxx + model: + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + tools: true # 是否支持工具，如生成代码、生成文档等，默认是 true，选填 + + - provider: gemini + base_url: https://generativelanguage.googleapis.com/v1beta # base_url 支持 v1beta/v1, 仅供 Gemini 模型使用，必填 + api: AIzaSyAN2k6IRdgw + model: + - gemini-1.5-pro + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # 重命名后，原来的模型名字 gemini-1.5-flash-exp-0827 无法使用，如果要使用原来的名字，可以在 model 中添加原来的名字，只要加上下面一行就可以使用原来的名字了 + - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 + tools: true + + - provider: vertex + project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个JSON格式的字符串，包含服务账号的私钥信息。获取方式： 在Google Cloud Console中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在Google Cloud Console的"IAM与管理"部分查看服务账号详情获得。 + model: + - gemini-1.5-pro + - gemini-1.5-flash + - claude-3-5-sonnet@20240620: claude-3-5-sonnet + - claude-3-opus@20240229: claude-3-opus + - claude-3-sonnet@20240229: claude-3-sonnet + - claude-3-haiku@20240307: claude-3-haiku + tools: true + notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + + - provider: other-provider + base_url: https://api.xxx.com/v1/messages + api: sk-bNnAOJyA-xQw_twAA + model: + - causallm-35b-beta2ep-q6k: causallm-35b + tools: false + engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 + +api_keys: + - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 + model: # 该 API Key 可以使用的模型，必填 + - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 + - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 + - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 + role: admin + + - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy + model: + - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。 + preferences: + USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 + AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true + RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 + + # 渠道级加权负载均衡配置示例 + - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + model: + - gcp1/*: 5 # 冒号后面就是权重，权重仅支持正整数。 + - gcp2/*: 3 # 数字的大小代表权重，数字越大，请求的概率越大。 + - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 + + preferences: + USE_ROUND_ROBIN: true # 当 USE_ROUND_ROBIN 必须为 true 并且上面的渠道后面没有权重时，会按照原始的渠道顺序请求，如果有权重，会按照加权后的顺序请求。 + AUTO_RETRY: true +``` + +## 环境变量 + +- CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 +- TIMEOUT: 请求超时时间，默认为 20 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 + +## Docker Local Deployment + +Start the container + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +``` + +Or if you want to use Docker Compose, here is a docker-compose.yml example: + +```yaml +services: + uni-api: + container_name: uni-api + image: yym68686/uni-api:latest + environment: + - CONFIG_URL=http://file_url/api.yaml + ports: + - 8001:8000 + volumes: + - ./api.yaml:/home/api.yaml +``` + +CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。 + +Run Docker Compose container in the background + +```bash +docker-compose pull +docker-compose up -d +``` + +Docker build + +```bash +docker build --no-cache -t uni-api:latest -f Dockerfile --platform linux/amd64 . +docker tag uni-api:latest yym68686/uni-api:latest +docker push yym68686/uni-api:latest +``` + +One-Click Restart Docker Image + +```bash +set -eu +docker pull yym68686/uni-api:latest +docker rm -f uni-api +docker run --user root -p 8001:8000 -dit --name uni-api \ +-e CONFIG_URL=http://file_url/api.yaml \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +docker logs -f uni-api +``` + +RESTful curl test + +```bash +curl -X POST http://127.0.0.1:8000/v1/chat/completions \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer ${API}" \ +-d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' +``` + + +## Star History + + + Star History Chart + \ No newline at end of file diff --git a/test/parse_markdown.py b/test/parse_markdown.py new file mode 100644 index 00000000..b8524a65 --- /dev/null +++ b/test/parse_markdown.py @@ -0,0 +1,159 @@ +class MarkdownEntity: + def __init__(self, content: str, entity_type: str): + self.content = content + self.entity_type = entity_type + + def __repr__(self): + return f'<{self.entity_type}: {self.content}>' + +class Title(MarkdownEntity): + def __init__(self, content: str, level: int): + super().__init__(content, 'Title') + self.level = level + +class CodeBlock(MarkdownEntity): + def __init__(self, content: str, language: str = 'python'): + super().__init__(content, 'CodeBlock') + self.language = language + +class ListItem(MarkdownEntity): + def __init__(self, content: str): + super().__init__(content, 'ListItem') + +class Link(MarkdownEntity): + def __init__(self, content: str, url: str): + super().__init__(content, 'Link') + self.url = url + +class EmptyLine(MarkdownEntity): + def __init__(self, content: str): + super().__init__(content, 'EmptyLine') + +class Paragraph(MarkdownEntity): + def __init__(self, content: str): + super().__init__(content, 'Paragraph') + +def parse_markdown(lines, delimiter='\n\n'): + entities = [] + current_code_block = [] + in_code_block = False + language = None + + for line in lines: + # line = line.strip() + + if line.startswith('#'): + level = line.count('#') + title_content = line[level:].strip() + entities.append(Title(title_content, level)) + + elif line.startswith('```'): + if in_code_block and language: + entities.append(CodeBlock(''.join(current_code_block), language)) + current_code_block = [] + in_code_block = False + language = None + else: + in_code_block = True + language = line.lstrip('`').strip() + + elif in_code_block: + current_code_block.append(line) + + elif '[' in line and ']' in line and '(' in line and ')' in line and line.count('[') == 1: + start = line.index('[') + 1 + end = line.index(']') + url_start = line.index('(') + 1 + url_end = line.index(')') + link_text = line[start:end].strip() + link_url = line[url_start:url_end].strip() + entities.append(Link(link_text, link_url)) + + elif line == delimiter: + entities.append(EmptyLine(line)) + + elif line: + entities.append(Paragraph(line)) + + return entities + +def convert_entities_to_text(entities): + result = [] + for entity in entities: + if isinstance(entity, Title): + result.append(f"{'#' * entity.level} {entity.content}") + elif isinstance(entity, CodeBlock): + code = entity.content.lstrip('\n').rstrip('\n') + result.append(f"```{entity.language}\n{code}\n```") + elif isinstance(entity, ListItem): + result.append(f"- {entity.content}") + elif isinstance(entity, Link): + result.append(f"[{entity.content}]({entity.url})") + elif isinstance(entity, EmptyLine): + result.append(f"{entity.content}") + elif isinstance(entity, Paragraph): + result.append(f"{entity.content}") + return ''.join(result) + +def save_text_to_file(text: str, file_path: str): + with open(file_path, 'w', encoding='utf-8') as file: + file.write(text) + +def process_markdown_entities_and_save(entities, file_path, raw_text=None): + # Step 1: Convert entities to text + text_output = convert_entities_to_text(entities) + if raw_text and raw_text != text_output: + raise ValueError("The text output does not match the raw text input.") + # Step 2: Save to file + save_text_to_file(text_output, file_path) + +def read_markdown_file(file_path): + with open(file_path, 'r', encoding='utf-8') as file: + return file.read() + +def split_markdown(text, delimiter='\n\n'): + # 使用两个换行符作为分割标记，分割段落 + # 创建一个新的列表来存储结果 + paragraphs = text.split(delimiter) + result = [] + + # 遍历分割后的段落，在它们之间插入空行实体 + for i, paragraph in enumerate(paragraphs): + if i > 0: + # 在非第一段之前插入空行实体 + result.append(delimiter) + + # 添加当前段落 + result.append(paragraph) + + return result + +def get_entities_from_markdown_file(file_path, delimiter='\n\n'): + # 读取 Markdown 文件 + markdown_text = read_markdown_file(file_path) + + # 分割 Markdown 文档 + paragraphs = split_markdown(markdown_text, delimiter=delimiter) + + # 解析 Markdown 文档 + return parse_markdown(paragraphs, delimiter=delimiter) + +if __name__ == '__main__': + markdown_file_path = "README_CN.md" # 替换为你的 Markdown 文件路径 + + # 读取 Markdown 文件 + delimiter = '\n' + markdown_text = read_markdown_file(markdown_file_path) + paragraphs = split_markdown(markdown_text, delimiter=delimiter) + parsed_entities = parse_markdown(paragraphs, delimiter=delimiter) + + # # 显示解析结果 + # result = [str(entity) for entity in parsed_entities] + # for idx, entity in enumerate(result): + # print(f"段落 {idx + 1} 解析：{entity}\n") + + # 保存到文件 + output_file_path = "output.md" + process_markdown_entities_and_save(parsed_entities, output_file_path, raw_text=markdown_text) + + print(f"Markdown 文档已保存到 {output_file_path}") \ No newline at end of file diff --git a/test/translate_md.py b/test/translate_md.py new file mode 100644 index 00000000..3a958143 --- /dev/null +++ b/test/translate_md.py @@ -0,0 +1,50 @@ +import os +from ModelMerge import chatgpt +from parse_markdown import get_entities_from_markdown_file, process_markdown_entities_and_save + +def translate_text(text, agent): + result = agent.ask(text) + return result + +def translate(input_file_path, output_file_path="output.md", language="English", api_key=None, api_url="https://api.openai.com/v1/chat/completions", engine="gpt-4o"): + if not api_key: + raise ValueError("API key is required for translation.") + translator_prompt = ( + "You are a translation engine, you can only translate text and cannot interpret it, and do not explain. " + "Translate the text to {}, please do not explain any sentences, just translate or leave them as they are. " + "Retain all spaces and line breaks in the original text. " + "Please do not wrap the code in code blocks, I will handle it myself. " + "If the code has comments, you should translate the comments as well. " + "This is the content you need to translate: " + ).format(language) + + agent = chatgpt( + api_key=api_key, + api_url=api_url, + engine=engine, + system_prompt=translator_prompt, + use_plugins=False + ) + + # 读取 Markdown 文件 + raw_paragraphs = get_entities_from_markdown_file(input_file_path, delimiter='\n') + target_paragraphs = raw_paragraphs + + # 逐段翻译 + for index, paragraph in enumerate(raw_paragraphs): + if paragraph.content and paragraph.content.strip() != "": + translated_text = translate_text(paragraph.content, agent) + if translated_text: + target_paragraphs[index].content = translated_text + + # 输出翻译结果 + process_markdown_entities_and_save(target_paragraphs, output_file_path) + +if __name__ == "__main__": + input_file_path = "README_CN.md" + output_file_path = "README.md" + language = "English" + api_key = os.getenv("API") + api_url = os.getenv("API_URL") + engine = "gpt-4o" + translate(input_file_path, output_file_path, language, api_key, api_url, engine) \ No newline at end of file From 237c936ad1331cf0ef644abc43345b681b64a90e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 9 Sep 2024 19:24:00 +0800 Subject: [PATCH 026/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20timestamp=20is=20a=20decimal.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 6 +- test/parse_markdown.py | 159 ------------------- test/{test_vertex copy.py => test_vertex.py} | 0 test/translate_md.py | 50 ------ 4 files changed, 3 insertions(+), 212 deletions(-) delete mode 100644 test/parse_markdown.py rename test/{test_vertex copy.py => test_vertex.py} (100%) delete mode 100644 test/translate_md.py diff --git a/response.py b/response.py index f4cc1270..962c7311 100644 --- a/response.py +++ b/response.py @@ -48,7 +48,7 @@ async def check_response(response, error_log): return None async def fetch_gemini_response_stream(client, url, headers, payload, model): - timestamp = datetime.timestamp(datetime.now()) + timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_gemini_response_stream") if error_message: @@ -93,7 +93,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): yield "data: [DONE]\n\r\n" async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): - timestamp = datetime.timestamp(datetime.now()) + timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_vertex_claude_response_stream") if error_message: @@ -156,7 +156,7 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects yield line.strip() + "\n\r\n" async def fetch_claude_response_stream(client, url, headers, payload, model): - timestamp = datetime.timestamp(datetime.now()) + timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_claude_response_stream") if error_message: diff --git a/test/parse_markdown.py b/test/parse_markdown.py deleted file mode 100644 index b8524a65..00000000 --- a/test/parse_markdown.py +++ /dev/null @@ -1,159 +0,0 @@ -class MarkdownEntity: - def __init__(self, content: str, entity_type: str): - self.content = content - self.entity_type = entity_type - - def __repr__(self): - return f'<{self.entity_type}: {self.content}>' - -class Title(MarkdownEntity): - def __init__(self, content: str, level: int): - super().__init__(content, 'Title') - self.level = level - -class CodeBlock(MarkdownEntity): - def __init__(self, content: str, language: str = 'python'): - super().__init__(content, 'CodeBlock') - self.language = language - -class ListItem(MarkdownEntity): - def __init__(self, content: str): - super().__init__(content, 'ListItem') - -class Link(MarkdownEntity): - def __init__(self, content: str, url: str): - super().__init__(content, 'Link') - self.url = url - -class EmptyLine(MarkdownEntity): - def __init__(self, content: str): - super().__init__(content, 'EmptyLine') - -class Paragraph(MarkdownEntity): - def __init__(self, content: str): - super().__init__(content, 'Paragraph') - -def parse_markdown(lines, delimiter='\n\n'): - entities = [] - current_code_block = [] - in_code_block = False - language = None - - for line in lines: - # line = line.strip() - - if line.startswith('#'): - level = line.count('#') - title_content = line[level:].strip() - entities.append(Title(title_content, level)) - - elif line.startswith('```'): - if in_code_block and language: - entities.append(CodeBlock(''.join(current_code_block), language)) - current_code_block = [] - in_code_block = False - language = None - else: - in_code_block = True - language = line.lstrip('`').strip() - - elif in_code_block: - current_code_block.append(line) - - elif '[' in line and ']' in line and '(' in line and ')' in line and line.count('[') == 1: - start = line.index('[') + 1 - end = line.index(']') - url_start = line.index('(') + 1 - url_end = line.index(')') - link_text = line[start:end].strip() - link_url = line[url_start:url_end].strip() - entities.append(Link(link_text, link_url)) - - elif line == delimiter: - entities.append(EmptyLine(line)) - - elif line: - entities.append(Paragraph(line)) - - return entities - -def convert_entities_to_text(entities): - result = [] - for entity in entities: - if isinstance(entity, Title): - result.append(f"{'#' * entity.level} {entity.content}") - elif isinstance(entity, CodeBlock): - code = entity.content.lstrip('\n').rstrip('\n') - result.append(f"```{entity.language}\n{code}\n```") - elif isinstance(entity, ListItem): - result.append(f"- {entity.content}") - elif isinstance(entity, Link): - result.append(f"[{entity.content}]({entity.url})") - elif isinstance(entity, EmptyLine): - result.append(f"{entity.content}") - elif isinstance(entity, Paragraph): - result.append(f"{entity.content}") - return ''.join(result) - -def save_text_to_file(text: str, file_path: str): - with open(file_path, 'w', encoding='utf-8') as file: - file.write(text) - -def process_markdown_entities_and_save(entities, file_path, raw_text=None): - # Step 1: Convert entities to text - text_output = convert_entities_to_text(entities) - if raw_text and raw_text != text_output: - raise ValueError("The text output does not match the raw text input.") - # Step 2: Save to file - save_text_to_file(text_output, file_path) - -def read_markdown_file(file_path): - with open(file_path, 'r', encoding='utf-8') as file: - return file.read() - -def split_markdown(text, delimiter='\n\n'): - # 使用两个换行符作为分割标记，分割段落 - # 创建一个新的列表来存储结果 - paragraphs = text.split(delimiter) - result = [] - - # 遍历分割后的段落，在它们之间插入空行实体 - for i, paragraph in enumerate(paragraphs): - if i > 0: - # 在非第一段之前插入空行实体 - result.append(delimiter) - - # 添加当前段落 - result.append(paragraph) - - return result - -def get_entities_from_markdown_file(file_path, delimiter='\n\n'): - # 读取 Markdown 文件 - markdown_text = read_markdown_file(file_path) - - # 分割 Markdown 文档 - paragraphs = split_markdown(markdown_text, delimiter=delimiter) - - # 解析 Markdown 文档 - return parse_markdown(paragraphs, delimiter=delimiter) - -if __name__ == '__main__': - markdown_file_path = "README_CN.md" # 替换为你的 Markdown 文件路径 - - # 读取 Markdown 文件 - delimiter = '\n' - markdown_text = read_markdown_file(markdown_file_path) - paragraphs = split_markdown(markdown_text, delimiter=delimiter) - parsed_entities = parse_markdown(paragraphs, delimiter=delimiter) - - # # 显示解析结果 - # result = [str(entity) for entity in parsed_entities] - # for idx, entity in enumerate(result): - # print(f"段落 {idx + 1} 解析：{entity}\n") - - # 保存到文件 - output_file_path = "output.md" - process_markdown_entities_and_save(parsed_entities, output_file_path, raw_text=markdown_text) - - print(f"Markdown 文档已保存到 {output_file_path}") \ No newline at end of file diff --git a/test/test_vertex copy.py b/test/test_vertex.py similarity index 100% rename from test/test_vertex copy.py rename to test/test_vertex.py diff --git a/test/translate_md.py b/test/translate_md.py deleted file mode 100644 index 3a958143..00000000 --- a/test/translate_md.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -from ModelMerge import chatgpt -from parse_markdown import get_entities_from_markdown_file, process_markdown_entities_and_save - -def translate_text(text, agent): - result = agent.ask(text) - return result - -def translate(input_file_path, output_file_path="output.md", language="English", api_key=None, api_url="https://api.openai.com/v1/chat/completions", engine="gpt-4o"): - if not api_key: - raise ValueError("API key is required for translation.") - translator_prompt = ( - "You are a translation engine, you can only translate text and cannot interpret it, and do not explain. " - "Translate the text to {}, please do not explain any sentences, just translate or leave them as they are. " - "Retain all spaces and line breaks in the original text. " - "Please do not wrap the code in code blocks, I will handle it myself. " - "If the code has comments, you should translate the comments as well. " - "This is the content you need to translate: " - ).format(language) - - agent = chatgpt( - api_key=api_key, - api_url=api_url, - engine=engine, - system_prompt=translator_prompt, - use_plugins=False - ) - - # 读取 Markdown 文件 - raw_paragraphs = get_entities_from_markdown_file(input_file_path, delimiter='\n') - target_paragraphs = raw_paragraphs - - # 逐段翻译 - for index, paragraph in enumerate(raw_paragraphs): - if paragraph.content and paragraph.content.strip() != "": - translated_text = translate_text(paragraph.content, agent) - if translated_text: - target_paragraphs[index].content = translated_text - - # 输出翻译结果 - process_markdown_entities_and_save(target_paragraphs, output_file_path) - -if __name__ == "__main__": - input_file_path = "README_CN.md" - output_file_path = "README.md" - language = "English" - api_key = os.getenv("API") - api_url = os.getenv("API_URL") - engine = "gpt-4o" - translate(input_file_path, output_file_path, language, api_key, api_url, engine) \ No newline at end of file From 9874f60c7c2a1e9f0badab6903137e39350c0a52 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 10 Sep 2024 04:21:24 +0800 Subject: [PATCH 027/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Adapt=20to=20the?= =?UTF-8?q?=20deprecated=20function=20call=20request=20body=20format=20of?= =?UTF-8?q?=20OpenAI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/request.py b/request.py index 88d44728..2068d82d 100644 --- a/request.py +++ b/request.py @@ -419,6 +419,18 @@ async def get_vertex_claude_payload(request, engine, provider): "tool_use_id": tool_id, "content": content }]}) + elif msg.role == "function": + messages.append({"role": "assistant", "content": [{ + "type": "tool_use", + "id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "name": msg.name, + "input": {"prompt": "..."} + }]}) + messages.append({"role": "user", "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "content": msg.content + }]}) elif msg.role != "system": messages.append({"role": msg.role, "content": content}) elif msg.role == "system": @@ -694,6 +706,18 @@ async def get_claude_payload(request, engine, provider): "tool_use_id": tool_id, "content": content }]}) + elif msg.role == "function": + messages.append({"role": "assistant", "content": [{ + "type": "tool_use", + "id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "name": msg.name, + "input": {"prompt": "..."} + }]}) + messages.append({"role": "user", "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "content": msg.content + }]}) elif msg.role != "system": messages.append({"role": msg.role, "content": content}) elif msg.role == "system": From 1de140d4081d7bbd4d2819e20140c52e88a2b21f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 10 Sep 2024 04:54:13 +0800 Subject: [PATCH 028/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20official=20Claude=20API=20does=20not=20correct?= =?UTF-8?q?ly=20pass=20the=20token=20count.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/response.py b/response.py index 962c7311..8ee5a987 100644 --- a/response.py +++ b/response.py @@ -5,7 +5,7 @@ from log_config import logger -async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, tokens_use=None, total_tokens=None): +async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): sample_data = { "id": "chatcmpl-9ijPeRHa0wtyA2G8wq5z8FC3wGMzc", "object": "chat.completion.chunk", @@ -29,6 +29,10 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]} if role: sample_data["choices"][0]["delta"] = {"role": role, "content": ""} + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens,"total_tokens": total_tokens} + sample_data["choices"] = [] json_data = json.dumps(sample_data, ensure_ascii=False) # 构建SSE响应 @@ -68,7 +72,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) - sse_string = await generate_sse_response(timestamp, model, content) + sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") @@ -114,7 +118,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) - sse_string = await generate_sse_response(timestamp, model, content) + sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") @@ -163,6 +167,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): yield error_message return buffer = "" + input_tokens = 0 async for chunk in response.aiter_text(): # logger.info(f"chunk: {repr(chunk)}") buffer += chunk @@ -171,20 +176,25 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): # logger.info(line) if line.startswith("data:"): - line = line[5:] - if line.startswith(" "): - line = line[1:] + line = line.lstrip("data: ") resp: dict = json.loads(line) message = resp.get("message") if message: - tokens_use = resp.get("usage") role = message.get("role") if role: sse_string = await generate_sse_response(timestamp, model, None, None, None, None, role) yield sse_string + tokens_use = message.get("usage") if tokens_use: - total_tokens = tokens_use["input_tokens"] + tokens_use["output_tokens"] - # print("\n\rtotal_tokens", total_tokens) + input_tokens = tokens_use.get("input_tokens", 0) + usage = resp.get("usage") + if usage: + output_tokens = usage.get("output_tokens", 0) + total_tokens = input_tokens + output_tokens + sse_string = await generate_sse_response(timestamp, model, None, None, None, None, None, total_tokens, input_tokens, output_tokens) + yield sse_string + # print("\n\rtotal_tokens", total_tokens) + tool_use = resp.get("content_block") tools_id = None function_call_name = None From 95ca783064efc23095a0827fe5885555351fad75 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 10 Sep 2024 05:31:06 +0800 Subject: [PATCH 029/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20Cloudflare=20API=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 69 +++++++++++++++++++++++++++++----------------------- README_CN.md | 13 +++++++--- main.py | 8 ++++-- request.py | 53 ++++++++++++++++++++++++++++++++++++++++ response.py | 32 ++++++++++++++++++++++-- utils.py | 2 ++ 6 files changed, 139 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 7f3bef61..f5dd5835 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ - + docker pull

@@ -13,23 +13,23 @@ ## Introduction -If used personally, one/new-api is too complex and has many commercial features that individuals do not need. If you don't want a complicated frontend interface and want to support more models, you can try uni-api. This is a project that manages large model APIs uniformly. It allows you to call multiple backend services through a unified API interface, converting them uniformly to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, DeepBricks, OpenRouter, etc. +If used personally, one/new-api is overly complex, with many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project for unified management of large model APIs, allowing you to call multiple backend services through a unified API interface, uniformly converting them to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cloudflare, DeepBricks, OpenRouter, etc. ## Features -- No front-end, purely configuration file to set up API channels. You can run your own API site just by writing a file. The documentation has detailed configuration guidelines, friendly for beginners. -- Unified management of multiple backend services, supporting OpenAI, Deepseek, DeepBricks, OpenRouter, and other API providers in the OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex API simultaneously. Vertex supports both Claude and Gemini API. -- Support native tool use function calls for OpenAI, Anthropic, Gemini, Vertex. +- No front-end, pure configuration file setup for API channels. You can run your own API site just by writing a single file, and the documentation includes a detailed configuration guide, beginner-friendly. +- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. +- Supports Anthropic, Gemini, Vertex API, and Cloudflare simultaneously. Vertex supports both Claude and Gemini API. +- Support for OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Supports four types of load balancing. - 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. By default, it is not enabled and requires channel weight configuration. - 2. Supports Vertex region-level load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. Disabled by default, channel weights need to be configured. + 2. Supports Vertex regional load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency up to (API quantity * regional quantity) times. Automatically enabled without additional configuration. 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. - 4. Support automatic API key-level polling load balancing for multiple API Keys in a single channel. -- Supports automatic retry. When an API channel response fails, automatically retry the next API channel. -- Supports fine-grained access control. Supports using wildcards to set specific models available for API key channels. -- Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. + 4. Support automatic API key-level round-robin load balancing for multiple API keys in a single channel. +- Supports automatic retry, when an API channel response fails, automatically retry the next API channel. +- Supports fine-grained permission control. Supports using wildcards to set specific models available for API key channels. +- Supports rate limiting, allowing you to set the maximum number of requests per minute. It can be set as an integer, such as 2/min (2 times per minute), 5/hour (5 times per hour), 10/day (10 times per day), 10/month (10 times per month), 10/year (10 times per year). The default is 60/min. ## Configuration @@ -37,21 +37,21 @@ Using the api.yaml configuration file, you can configure multiple models, and ea ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, arbitrary name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # At least one model + model: # At least one model is required - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini @@ -59,14 +59,14 @@ providers: api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the following line to use the original name + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the line below to use the original name. - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, you can also view the service account details in the "IAM & Admin" section of the Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually consists of lowercase letters, numbers, and hyphens. How to get: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to get: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to get: Generated when creating the service account, or you can view the service account details in the "IAM & Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -75,7 +75,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # Can put the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes otherwise YAML syntax error, llama-3.1-8b is the renamed name, you can use a simpler name instead of the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes otherwise YAML syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -95,28 +102,28 @@ api_keys: - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models of other providers' claude-3-5-sonnet cannot be used. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. preferences: - USE_ROUND_ROBIN: true # Whether to use polling load balancing, true to use, false to not use, default is true. When polling is enabled, each request model is requested in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request sequences for each API key. + USE_ROUND_ROBIN: true # Whether to use round-robin load balancing, true to use, false to not use, default is true. When enabled, each request to the model is made in the order configured in the model. This is independent of the original channel order in providers. Therefore, you can set different request orders for each API key. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, weights only support positive integers. - - gcp2/*: 3 # The larger the number, the greater the probability of being requested. - - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp1/*: 5 # The number after the colon is the weight, only positive integers are supported. + - gcp2/*: 3 # The larger the number, the higher the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and out of 10 requests, 5 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the channels above, it will request in the original channel order, if there is weight, it will request in the weighted order. + USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, requests will be made in the original channel order. If there are weights, requests will be made in the weighted order. AUTO_RETRY: true ``` -## Environment variables +## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 20 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional +- TIMEOUT: Request timeout, default is 20 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional ## Docker Local Deployment @@ -143,7 +150,7 @@ services: - ./api.yaml:/home/api.yaml ``` -CONFIG_URL is a direct link that can automatically download remote configuration files. For example, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. +CONFIG_URL is a direct link that can automatically download remote configuration files. For instance, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration files to a hosting service that provides a direct link for uni-api to download. CONFIG_URL is this direct link. Run Docker Compose container in the background diff --git a/README_CN.md b/README_CN.md index 01bfc3e5..a813bf5c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,13 +13,13 @@ ## Introduction -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、cloudflare、DeepBricks、OpenRouter 等。 ## Features - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 -- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 同时支持 Anthropic、Gemini、Vertex API、cloudflare。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 - 支持四种负载均衡。 @@ -77,6 +77,13 @@ providers: tools: true notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key，必填 + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID，必填 + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # 重命名模型，@cf/meta/llama-3.1-8b-instruct 是服务商的原始的模型名称，必须使用引号包裹模型名，否则yaml语法错误，llama-3.1-8b 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - '@cf/meta/llama-3.1-8b-instruct' # 必须使用引号包裹模型名，否则yaml语法错误 + - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA diff --git a/main.py b/main.py index 1450b7c2..e84998f3 100644 --- a/main.py +++ b/main.py @@ -174,11 +174,14 @@ async def cleanup(self): async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) + # print("parsed_url", parsed_url) engine = None if parsed_url.netloc == 'generativelanguage.googleapis.com': engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" + elif parsed_url.netloc == 'api.cloudflare.com': + engine = "cloudflare" elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): engine = "claude" elif parsed_url.netloc == 'openrouter.ai': @@ -188,7 +191,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], if "claude" not in provider['model'][request.model] \ and "gpt" not in provider['model'][request.model] \ - and "gemini" not in provider['model'][request.model]: + and "gemini" not in provider['model'][request.model] \ + and parsed_url.netloc != 'api.cloudflare.com': engine = "openrouter" if "claude" in provider['model'][request.model] and engine == "vertex": @@ -311,7 +315,7 @@ def get_matching_providers(self, model_name, token): # import json # for provider in provider_list: - # print(json.dumps(provider, indent=4, ensure_ascii=False)) + # print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None): diff --git a/request.py b/request.py index 2068d82d..da51b9df 100644 --- a/request.py +++ b/request.py @@ -33,6 +33,8 @@ async def get_text_message(role, message, engine = None): return {"type": "text", "text": message} if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} + if engine == "cloudflare": + return message raise ValueError("Unknown engine") async def get_gemini_payload(request, engine, provider): @@ -640,6 +642,55 @@ async def get_openrouter_payload(request, engine, provider): return url, headers, payload +async def get_cloudflare_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + + model = provider['model'][request.model] + url = "https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model_id}".format(cf_account_id=provider['cf_account_id'], cf_model_id=model) + + msg = request.messages[-1] + messages = [] + content = None + if isinstance(msg.content, list): + for item in msg.content: + if item.type == "text": + content = await get_text_message(msg.role, item.text, engine) + else: + content = msg.content + name = msg.name + + model = provider['model'][request.model] + payload = { + "prompt": content, + } + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + async def gpt2claude_tools_json(json_dict): import copy json_dict = copy.deepcopy(json_dict) @@ -830,6 +881,8 @@ async def get_payload(request: RequestModel, engine, provider): return await get_gpt_payload(request, engine, provider) elif engine == "openrouter": return await get_openrouter_payload(request, engine, provider) + elif engine == "cloudflare": + return await get_cloudflare_payload(request, engine, provider) elif engine == "dalle": return await get_dalle_payload(request, engine, provider) else: diff --git a/response.py b/response.py index 8ee5a987..d33d2fc8 100644 --- a/response.py +++ b/response.py @@ -112,7 +112,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod buffer += chunk while "\n" in buffer: line, buffer = buffer.split("\n", 1) - logger.info(f"{line}") + # logger.info(f"{line}") if line and '\"text\": \"' in line: try: json_data = json.loads( "{" + line + "}") @@ -143,7 +143,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod yield sse_string yield "data: [DONE]\n\r\n" -async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): +async def fetch_gpt_response_stream(client, url, headers, payload): async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_gpt_response_stream") if error_message: @@ -159,6 +159,31 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects if line and line != "data: " and line != "data:" and not line.startswith(": "): yield line.strip() + "\n\r\n" +async def fetch_cloudflare_response_stream(client, url, headers, payload, model): + timestamp = int(datetime.timestamp(datetime.now())) + async with client.stream('POST', url, headers=headers, json=payload) as response: + error_message = await check_response(response, "fetch_gpt_response_stream") + if error_message: + yield error_message + return + + buffer = "" + async for chunk in response.aiter_text(): + buffer += chunk + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + # logger.info("line: %s", repr(line)) + if line.startswith("data:"): + line = line.lstrip("data: ") + if line == "[DONE]": + yield "data: [DONE]\n\r\n" + return + resp: dict = json.loads(line) + message = resp.get("response") + if message: + sse_string = await generate_sse_response(timestamp, model, content=message) + yield sse_string + async def fetch_claude_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: @@ -242,6 +267,9 @@ async def fetch_response_stream(client, url, headers, payload, engine, model): elif engine == "openrouter": async for chunk in fetch_gpt_response_stream(client, url, headers, payload): yield chunk + elif engine == "cloudflare": + async for chunk in fetch_cloudflare_response_stream(client, url, headers, payload, model): + yield chunk else: raise ValueError("Unknown response") except httpx.ConnectError as e: diff --git a/utils.py b/utils.py index e376ede2..366a3a2b 100644 --- a/utils.py +++ b/utils.py @@ -15,6 +15,8 @@ def update_config(config_data): provider['model'] = model_dict if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' + if provider.get('cf_account_id'): + provider['base_url'] = 'https://api.cloudflare.com/' if provider.get('api'): if isinstance(provider.get('api'), str): From cfd3f47b341724469e3525e4a8bfcc55f2964bfb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 11 Sep 2024 05:14:08 +0800 Subject: [PATCH 030/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20the=20API=20key=20is=20not=20found=20when=20rate?= =?UTF-8?q?=20limiting.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where the characters before the slash in the model name with a slash are parsed as the channel name. --- main.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index e84998f3..7c22d03b 100644 --- a/main.py +++ b/main.py @@ -275,20 +275,26 @@ def get_matching_providers(self, model_name, token): for model in config['api_keys'][api_index]['model']: if "/" in model: provider_name = model.split("/")[0] - model = model.split("/")[1] + model_name_split = "/".join(model.split("/")[1:]) models_list = [] for provider in config['providers']: if provider['provider'] == provider_name: models_list.extend(list(provider['model'].keys())) # print("models_list", models_list) # print("model_name", model_name) + + # 处理带斜杠的模型名 + for provider in config['providers']: + if model in provider['model'].keys(): + provider_rules.append(provider['provider'] + "/" + model) + # print("model", model) - if (model and model_name in models_list) or (model == "*" and model_name in models_list): + if (model_name_split and model_name in models_list) or (model_name_split == "*" and model_name in models_list): provider_rules.append(provider_name) else: for provider in config['providers']: if model in provider['model'].keys(): - provider_rules.append(provider['provider'] + "/" + model) + provider_rules.append(provider['provider'] + "/" + model_name_split) provider_list = [] # print("provider_rules", provider_rules) @@ -297,7 +303,7 @@ def get_matching_providers(self, model_name, token): # print("provider", provider, provider['provider'] == item, item) if "/" in item: if provider['provider'] == item.split("/")[0]: - if model_name in provider['model'].keys() and item.split("/")[1] == model_name: + if model_name in provider['model'].keys() and "/".join(item.split("/")[1:]) == model_name: provider_list.append(provider) elif provider['provider'] == item: if model_name in provider['model'].keys(): @@ -422,15 +428,13 @@ async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: rate_limiter = InMemoryRateLimiter() -async def get_user_rate_limit(token: str = None): +async def get_user_rate_limit(api_index: str = None): # 这里应该实现根据 token 获取用户速率限制的逻辑 # 示例： 返回 (次数， 秒数) config = app.state.config - api_list = app.state.api_list - api_index = api_list.index(token) raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") - if not token or not raw_rate_limit: + if not api_index or not raw_rate_limit: return (60, 60) rate_limit = parse_rate_limit(raw_rate_limit) @@ -439,8 +443,14 @@ async def get_user_rate_limit(token: str = None): security = HTTPBearer() async def rate_limit_dependency(request: Request, credentials: HTTPAuthorizationCredentials = Depends(security)): token = credentials.credentials if credentials else None - # print("token", token) - limit, period = await get_user_rate_limit(token) + api_list = app.state.api_list + try: + api_index = api_list.index(token) + except ValueError: + print("error: Invalid or missing API Key:", token) + api_index = None + token = None + limit, period = await get_user_rate_limit(api_index) # 使用 IP 地址和 token（如果有）作为限制键 client_ip = request.client.host From 6038b371621c48ac186895002258a6314026efb3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 11 Sep 2024 05:46:03 +0800 Subject: [PATCH 031/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Add=20support=20for=20using=20angle=20brackets=20to=20enclos?= =?UTF-8?q?e=20strings=20to=20set=20the=20string=20before=20the=20slash=20?= =?UTF-8?q?to=20the=20channel=20name.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 68 +++++++++++++++++++++++++++------------------------- README_CN.md | 4 +++- main.py | 37 ++++++++++++++-------------- 3 files changed, 57 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index f5dd5835..3a3216d7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # uni-api

- + @@ -13,36 +13,36 @@ ## Introduction -If used personally, one/new-api is overly complex, with many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project for unified management of large model APIs, allowing you to call multiple backend services through a unified API interface, uniformly converting them to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cloudflare, DeepBricks, OpenRouter, etc. +If used personally, one/new-api is too complex and has many commercial functions that individuals do not need. If you do not want a complicated front-end interface and want to support more models, you can try uni-api. This is a project for unified management of large model APIs, allowing you to call multiple backend services through a unified API interface, converting them uniformly to OpenAI format and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cloudflare, DeepBricks, OpenRouter, etc. ## Features -- No front-end, pure configuration file setup for API channels. You can run your own API site just by writing a single file, and the documentation includes a detailed configuration guide, beginner-friendly. -- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex API, and Cloudflare simultaneously. Vertex supports both Claude and Gemini API. -- Support for OpenAI, Anthropic, Gemini, Vertex native tool use function calls. +- No frontend, pure configuration file setup for API channels. You can run your own API site by just writing one file, with detailed configuration guides in the documentation, beginner-friendly. +- Unified management of multiple backend services, supporting providers like OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in the OpenAI format. Supports OpenAI Dalle-3 image generation. +- Supports Anthropic, Gemini, Vertex API, and Cloudflare. Vertex supports both Claude and Gemini API. +- Supports OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Supports four types of load balancing. - 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. Disabled by default, channel weights need to be configured. - 2. Supports Vertex regional load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency up to (API quantity * regional quantity) times. Automatically enabled without additional configuration. - 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. - 4. Support automatic API key-level round-robin load balancing for multiple API keys in a single channel. + 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. Disabled by default, requires channel weight configuration. + 2. Supports Vertex regional load balancing, supports Vertex high concurrency, and can increase Gemini, Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 3. In addition to Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. + 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. - Supports automatic retry, when an API channel response fails, automatically retry the next API channel. -- Supports fine-grained permission control. Supports using wildcards to set specific models available for API key channels. -- Supports rate limiting, allowing you to set the maximum number of requests per minute. It can be set as an integer, such as 2/min (2 times per minute), 5/hour (5 times per hour), 10/day (10 times per day), 10/month (10 times per month), 10/year (10 times per year). The default is 60/min. +- Supports fine-grained access control. Supports using wildcards to set specific models for API key available channels. +- Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. ## Configuration -Using the api.yaml configuration file, you can configure multiple models, and each model can configure multiple backend services, supporting load balancing. Below is an example of the api.yaml configuration file: +Using the api.yaml configuration file, multiple models can be configured, and each model can be configured with multiple backend services, supporting load balancing. Below is an example of the api.yaml configuration file: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required model: # At least one model is required - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional - dall-e-3 - provider: anthropic @@ -51,22 +51,22 @@ providers: - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional - tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional + tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the line below to use the original name. + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually consists of lowercase letters, numbers, and hyphens. How to get: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to get: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to get: Generated when creating the service account, or you can view the service account details in the "IAM & Admin" section of the Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud Project ID. Format: String, usually consists of lowercase letters, numbers, and hyphens. How to get it: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to get it: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to get it: Generated when creating the service account, can also be found in the "IAM & admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -75,25 +75,26 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # You can put the service provider's website, notes, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes otherwise YAML syntax error, llama-3.1-8b is the renamed name, you can use a simpler name instead of the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes otherwise YAML syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes, otherwise a yaml syntax error, llama-3.1-8b is the renamed name, you can use a concise name instead of the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes, otherwise a yaml syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA model: - causallm-35b-beta2ep-q6k: causallm-35b + - anthropic/claude-3-5-sonnet tools: false engine: openrouter # Force the use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service + - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required model: # Models that this API Key can use, required - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers @@ -102,28 +103,29 @@ api_keys: - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. This method will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will take the entire anthropic/claude-3-5-sonnet as the model name. This method can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. preferences: - USE_ROUND_ROBIN: true # Whether to use round-robin load balancing, true to use, false to not use, default is true. When enabled, each request to the model is made in the order configured in the model. This is independent of the original channel order in providers. Therefore, you can set different request orders for each API key. + USE_ROUND_ROBIN: true # Whether to use round-robin load balancing, true to use, false to not use, default is true. When enabled, each request to the model will be made in sequence according to the model configuration. It has nothing to do with the original channel order in providers. Therefore, you can set a different request order for each API key. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, only positive integers are supported. - - gcp2/*: 3 # The larger the number, the higher the probability of the request. - - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and out of 10 requests, 5 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp1/*: 5 # The number after the colon is the weight, the weight only supports positive integers. + - gcp2/*: 3 # The larger the number, the greater the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, requests will be made in the original channel order. If there are weights, requests will be made in the weighted order. + USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, it will request according to the original channel order, if there is weight, it will request according to the weighted order. AUTO_RETRY: true ``` ## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 20 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional +- TIMEOUT: Request timeout, default is 20 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional. ## Docker Local Deployment @@ -150,7 +152,7 @@ services: - ./api.yaml:/home/api.yaml ``` -CONFIG_URL is a direct link that can automatically download remote configuration files. For instance, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration files to a hosting service that provides a direct link for uni-api to download. CONFIG_URL is this direct link. +CONFIG_URL is a link that can automatically download a remote configuration file. For example, if you find it inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service that provides a direct link for uni-api to download. CONFIG_URL is this direct link. Run Docker Compose container in the background diff --git a/README_CN.md b/README_CN.md index a813bf5c..ec74e91b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -89,6 +89,7 @@ providers: api: sk-bNnAOJyA-xQw_twAA model: - causallm-35b-beta2ep-q6k: causallm-35b + - anthropic/claude-3-5-sonnet tools: false engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 @@ -102,7 +103,8 @@ api_keys: - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。 + - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。这种写法不会匹配到other-provider提供的名为anthropic/claude-3-5-sonnet的模型。 + - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 preferences: USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true diff --git a/main.py b/main.py index 7c22d03b..64dc24cf 100644 --- a/main.py +++ b/main.py @@ -274,27 +274,28 @@ def get_matching_providers(self, model_name, token): for model in config['api_keys'][api_index]['model']: if "/" in model: - provider_name = model.split("/")[0] - model_name_split = "/".join(model.split("/")[1:]) - models_list = [] - for provider in config['providers']: - if provider['provider'] == provider_name: - models_list.extend(list(provider['model'].keys())) - # print("models_list", models_list) - # print("model_name", model_name) - - # 处理带斜杠的模型名 - for provider in config['providers']: - if model in provider['model'].keys(): - provider_rules.append(provider['provider'] + "/" + model) - - # print("model", model) - if (model_name_split and model_name in models_list) or (model_name_split == "*" and model_name in models_list): - provider_rules.append(provider_name) + if model.startswith("<") and model.endswith(">"): + model = model[1:-1] + # 处理带斜杠的模型名 + for provider in config['providers']: + if model in provider['model'].keys(): + provider_rules.append(provider['provider'] + "/" + model) + else: + provider_name = model.split("/")[0] + model_name_split = "/".join(model.split("/")[1:]) + models_list = [] + for provider in config['providers']: + if provider['provider'] == provider_name: + models_list.extend(list(provider['model'].keys())) + # print("models_list", models_list) + # print("model_name", model_name) + # print("model", model) + if (model_name_split and model_name in models_list) or (model_name_split == "*" and model_name in models_list): + provider_rules.append(provider_name) else: for provider in config['providers']: if model in provider['model'].keys(): - provider_rules.append(provider['provider'] + "/" + model_name_split) + provider_rules.append(provider['provider'] + "/" + model) provider_list = [] # print("provider_rules", provider_rules) From 14428d9c40a5b71b171f54ecb0ca2313525492f1 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Sep 2024 03:27:46 +0800 Subject: [PATCH 032/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20the=20official=20Claude=20API=20cannot=20use=20PNG=20?= =?UTF-8?q?format=20images.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/request.py b/request.py index da51b9df..7a8d2ecd 100644 --- a/request.py +++ b/request.py @@ -3,6 +3,10 @@ from utils import c35s, c3s, c3o, c3h, gem, BaseAPI async def get_image_message(base64_image, engine = None): + colon_index = base64_image.index(":") + semicolon_index = base64_image.index(";") + image_type = base64_image[colon_index + 1:semicolon_index] + # print("image_type", image_type) if "gpt" == engine: return { "type": "image_url", @@ -15,14 +19,14 @@ async def get_image_message(base64_image, engine = None): "type": "image", "source": { "type": "base64", - "media_type": "image/jpeg", + "media_type": image_type, "data": base64_image.split(",")[1], } } if "gemini" == engine or "vertex-gemini" == engine: return { "inlineData": { - "mimeType": "image/jpeg", + "mimeType": image_type, "data": base64_image.split(",")[1], } } From 3972d748d7ac6cebb8e2fdb878a74927ec9a9fd9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Sep 2024 04:06:33 +0800 Subject: [PATCH 033/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20error=20codes=20are=20not=20accurately=20returned=20?= =?UTF-8?q?to=20the=20client.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 20 ++++++++++++++++---- response.py | 2 +- utils.py | 8 +++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 64dc24cf..e49ea1d6 100644 --- a/main.py +++ b/main.py @@ -218,7 +218,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], if request.stream: model = provider['model'][request.model] generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) - wrapped_generator = await error_handling_wrapper(generator, status_code=500) + wrapped_generator = await error_handling_wrapper(generator) response = StreamingResponse(wrapped_generator, media_type="text/event-stream") else: response = await anext(fetch_response(app.state.client, url, headers, payload)) @@ -369,6 +369,8 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # 在 try_all_providers 函数中处理失败的情况 async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None): + status_code = 500 + error_message = None num_providers = len(providers) start_index = self.last_provider_index + 1 if use_round_robin else 0 for i in range(num_providers + 1): @@ -377,14 +379,24 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe try: response = await process_request(request, provider, endpoint) return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: + except HTTPException as e: logger.error(f"Error with provider {provider['provider']}: {str(e)}") + status_code = e.status_code + error_message = e.detail + + if auto_retry: + continue + else: + raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") + except (Exception, asyncio.CancelledError, httpx.ReadError) as e: + logger.error(f"Error with provider {provider['provider']}: {str(e)}") + error_message = str(e) if auto_retry: continue else: - raise HTTPException(status_code=500, detail="Error: Current provider response failed!") + raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") - raise HTTPException(status_code=500, detail=f"All providers failed: {request.model}") + raise HTTPException(status_code=status_code, detail=f"All {request.model} error: {error_message}") model_handler = ModelRequestHandler() diff --git a/response.py b/response.py index d33d2fc8..be2f1c3c 100644 --- a/response.py +++ b/response.py @@ -48,7 +48,7 @@ async def check_response(response, error_log): error_json = json.loads(error_str) except json.JSONDecodeError: error_json = error_str - return {"error": f"{error_log} HTTP Error {response.status_code}", "details": error_json} + return {"error": f"{error_log} HTTP Error", "status_code": response.status_code, "details": error_json} return None async def fetch_gemini_response_stream(client, url, headers, payload, model): diff --git a/utils.py b/utils.py index 366a3a2b..2882f9e2 100644 --- a/utils.py +++ b/utils.py @@ -104,7 +104,7 @@ def ensure_string(item): return str(item) import asyncio -async def error_handling_wrapper(generator, status_code=200): +async def error_handling_wrapper(generator): try: first_item = await generator.__anext__() first_item_str = first_item @@ -126,7 +126,9 @@ async def error_handling_wrapper(generator, status_code=200): raise StopAsyncIteration if isinstance(first_item_str, dict) and 'error' in first_item_str: # 如果第一个 yield 的项是错误信息，抛出 HTTPException - raise HTTPException(status_code=status_code, detail=f"{first_item_str}"[:300]) + status_code = first_item_str.get('status_code', 500) + detail = first_item_str.get('details', f"{first_item_str}") + raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) # 如果不是错误，创建一个新的生成器，首先yield第一个项，然后yield剩余的项 async def new_generator(): @@ -141,7 +143,7 @@ async def new_generator(): return new_generator() except StopAsyncIteration: - raise HTTPException(status_code=status_code, detail="data: {'error': 'No data returned'}") + raise HTTPException(status_code=400, detail="data: {'error': 'No data returned'}") def post_all_models(token, config, api_list): all_models = [] From 5c2c7404c49af5d5365e991706bff1fadb9f525a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Sep 2024 14:19:47 +0800 Subject: [PATCH 034/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20client=20uploads=20the=20image=20URL=20instead?= =?UTF-8?q?=20of=20converting=20it=20to=20base64=20encoding=20during=20ima?= =?UTF-8?q?ge=20recognition.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + request.py | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index e49ea1d6..ddbb223e 100644 --- a/main.py +++ b/main.py @@ -492,6 +492,7 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): + # logger.info(f"Request received: {request}") return await model_handler.request_model(request, token) @app.options("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) diff --git a/request.py b/request.py index 7a8d2ecd..92541ad7 100644 --- a/request.py +++ b/request.py @@ -1,12 +1,54 @@ +import os import json from models import RequestModel from utils import c35s, c3s, c3o, c3h, gem, BaseAPI +import base64 +import urllib.parse + +def encode_image(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + +async def get_doc_from_url(url): + filename = urllib.parse.unquote(url.split("/")[-1]) + transport = httpx.AsyncHTTPTransport( + http2=True, + verify=False, + retries=1 + ) + async with httpx.AsyncClient(transport=transport) as client: + try: + response = await client.get( + url, + timeout=30.0 + ) + with open(filename, 'wb') as f: + f.write(response.content) + + except httpx.RequestError as e: + print(f"An error occurred while requesting {e.request.url!r}.") + + return filename + +async def get_encode_image(image_url): + filename = await get_doc_from_url(image_url) + image_path = os.getcwd() + "/" + filename + base64_image = encode_image(image_path) + if filename.endswith(".png"): + prompt = f"data:image/png;base64,{base64_image}" + else: + prompt = f"data:image/jpeg;base64,{base64_image}" + os.remove(image_path) + return prompt + async def get_image_message(base64_image, engine = None): + if base64_image.startswith("http"): + base64_image = await get_encode_image(base64_image) colon_index = base64_image.index(":") semicolon_index = base64_image.index(";") image_type = base64_image[colon_index + 1:semicolon_index] - # print("image_type", image_type) + if "gpt" == engine: return { "type": "image_url", From f4d6dda66918acd0614d4a974f2fa395dc0b36ba Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Sep 2024 17:54:32 +0800 Subject: [PATCH 035/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20httpx.RemoteProtocolError=20was=20not=20caught?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index ddbb223e..31cf49e2 100644 --- a/main.py +++ b/main.py @@ -228,7 +228,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], app.middleware_stack.app.channel_success_counts[provider['provider']] += 1 return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: # 更新失败计数 async with app.middleware_stack.app.lock: app.middleware_stack.app.channel_failure_counts[provider['provider']] += 1 @@ -388,7 +388,7 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe continue else: raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") - except (Exception, asyncio.CancelledError, httpx.ReadError) as e: + except (Exception, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: logger.error(f"Error with provider {provider['provider']}: {str(e)}") error_message = str(e) if auto_retry: From 1126d73a9de977a548594717fbf25dec7aa7fc26 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 13 Sep 2024 14:56:37 +0800 Subject: [PATCH 036/476] =?UTF-8?q?=F0=9F=A4=96=20Models:=20Add=20support?= =?UTF-8?q?=20for=20o1-mini=20o1-preview=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 4 ++++ request.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ utils.py | 9 +++++++++ 3 files changed, 71 insertions(+) diff --git a/main.py b/main.py index 31cf49e2..f801002b 100644 --- a/main.py +++ b/main.py @@ -201,6 +201,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], if "gemini" in provider['model'][request.model] and engine == "vertex": engine = "vertex-gemini" + if "o1-preview" in provider['model'][request.model] or "o1-mini" in provider['model'][request.model]: + engine = "o1" + request.stream = False + if endpoint == "/v1/images/generations": engine = "dalle" request.stream = False diff --git a/request.py b/request.py index 92541ad7..788fd50e 100644 --- a/request.py +++ b/request.py @@ -737,6 +737,62 @@ async def get_cloudflare_payload(request, engine, provider): return url, headers, payload +async def get_o1_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + + url = provider['base_url'] + + messages = [] + for msg in request.messages: + if isinstance(msg.content, list): + content = [] + for item in msg.content: + if item.type == "text": + text_message = await get_text_message(msg.role, item.text, engine) + content.append(text_message) + else: + content = msg.content + + if isinstance(content, list): + for item in content: + if item["type"] == "text": + messages.append({"role": msg.role, "content": item["text"]}) + else: + messages.append({"role": msg.role, "content": content}) + + model = provider['model'][request.model] + payload = { + "model": model, + "messages": messages, + } + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + async def gpt2claude_tools_json(json_dict): import copy json_dict = copy.deepcopy(json_dict) @@ -929,6 +985,8 @@ async def get_payload(request: RequestModel, engine, provider): return await get_openrouter_payload(request, engine, provider) elif engine == "cloudflare": return await get_cloudflare_payload(request, engine, provider) + elif engine == "o1": + return await get_o1_payload(request, engine, provider) elif engine == "dalle": return await get_dalle_payload(request, engine, provider) else: diff --git a/utils.py b/utils.py index 2882f9e2..a62d9bcc 100644 --- a/utils.py +++ b/utils.py @@ -53,6 +53,15 @@ def update_config(config_data): async def load_config(app=None): import yaml try: + # with open('./api.yaml', 'r') as f: + # tokens = yaml.scan(f) + # for token in tokens: + # if isinstance(token, yaml.ScalarToken): + # value = token.value + # # 如果plain为False，表示字符串被引号包裹 + # is_quoted = not token.plain + # print(f"值: {value}, 是否被引号包裹: {is_quoted}") + with open('./api.yaml', 'r') as f: # 判断是否为空文件 conf = yaml.safe_load(f) From aea01d0eb61557bc57927ea2af07b7fa32303a3c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 13 Sep 2024 15:30:35 +0800 Subject: [PATCH 037/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20system=20prompt=20is=20not=20deleted=20in=20th?= =?UTF-8?q?e=20o1-mini=20model=20message.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request.py b/request.py index 788fd50e..b7da10c1 100644 --- a/request.py +++ b/request.py @@ -757,11 +757,11 @@ async def get_o1_payload(request, engine, provider): else: content = msg.content - if isinstance(content, list): + if isinstance(content, list) and msg.role != "system": for item in content: if item["type"] == "text": messages.append({"role": msg.role, "content": item["text"]}) - else: + elif msg.role != "system": messages.append({"role": msg.role, "content": content}) model = provider['model'][request.model] From 78ff70d3bd961f4342eb42ecaaa5ca6f8812e856 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 13 Sep 2024 15:32:39 +0800 Subject: [PATCH 038/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20o1-mi?= =?UTF-8?q?ni=20timeout=20bug,=20increase=20the=20default=20timeout=20to?= =?UTF-8?q?=2040=20seconds.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index f801002b..7e749e52 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,7 @@ async def lifespan(app: FastAPI): # 启动时的代码 import os - TIMEOUT = float(os.getenv("TIMEOUT", 20)) + TIMEOUT = float(os.getenv("TIMEOUT", 40)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) default_headers = { "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent From 49f1818a98597ad7ed0f8fdddeef03a5077ce6a6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 13 Sep 2024 15:35:50 +0800 Subject: [PATCH 039/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3a3216d7..6581db7a 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ api_keys: ## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 20 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional. +- TIMEOUT: Request timeout, default is 40 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional. ## Docker Local Deployment diff --git a/README_CN.md b/README_CN.md index ec74e91b..278a5402 100644 --- a/README_CN.md +++ b/README_CN.md @@ -125,7 +125,7 @@ api_keys: ## 环境变量 - CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 -- TIMEOUT: 请求超时时间，默认为 20 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +- TIMEOUT: 请求超时时间，默认为 40 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 ## Docker Local Deployment From ab734b8fd49c7f990804fa4f11bd13f876affb92 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Sep 2024 00:53:41 +0800 Subject: [PATCH 040/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Increase=20the?= =?UTF-8?q?=20default=20timeout=20to=20100=20seconds=20to=20improve=20the?= =?UTF-8?q?=20success=20rate=20of=20o1-preview=20responses.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 7e749e52..2bc60cb5 100644 --- a/main.py +++ b/main.py @@ -20,11 +20,13 @@ from typing import List, Dict, Union from urllib.parse import urlparse +import os +is_debug = os.getenv("DEBUG", False) + @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 - import os - TIMEOUT = float(os.getenv("TIMEOUT", 40)) + TIMEOUT = float(os.getenv("TIMEOUT", 100)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) default_headers = { "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent @@ -215,9 +217,9 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], logger.info(f"provider: {provider['provider']:<10} model: {request.model:<10} engine: {engine}") url, headers, payload = await get_payload(request, engine, provider) - - # logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) - # logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) + if is_debug: + logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) + logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) try: if request.stream: model = provider['model'][request.model] @@ -323,10 +325,10 @@ def get_matching_providers(self, model_name, token): # else: # if model_name in provider['model'].keys(): # provider_list.append(provider) - - # import json - # for provider in provider_list: - # print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) + if is_debug: + import json + for provider in provider_list: + print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None): @@ -530,14 +532,14 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) if isinstance(middleware, StatsMiddleware): async with middleware.lock: stats = { + "channel_success_percentages": middleware.calculate_success_percentages(), + "channel_failure_percentages": middleware.calculate_failure_percentages(), "request_counts": dict(middleware.request_counts), "request_times": dict(middleware.request_times), "ip_counts": {k: dict(v) for k, v in middleware.ip_counts.items()}, "request_arrivals": {k: [t.isoformat() for t in v] for k, v in middleware.request_arrivals.items()}, "channel_success_counts": dict(middleware.channel_success_counts), "channel_failure_counts": dict(middleware.channel_failure_counts), - "channel_success_percentages": middleware.calculate_success_percentages(), - "channel_failure_percentages": middleware.calculate_failure_percentages() } return JSONResponse(content=stats) return {"error": "StatsMiddleware not found"} From c09ef14ef1428983babbc07eb92a114309c46e95 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Sep 2024 02:00:33 +0800 Subject: [PATCH 041/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20connectivity=20check=20error=20at=20lobechat=20endpoint?= =?UTF-8?q?=20/v1/chat/completions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 2bc60cb5..f7423046 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,11 @@ @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 + + # # 启动事件 + # routes = [{"path": route.path, "name": route.name} for route in app.routes] + # logger.info(f"Registered routes: {routes}") + TIMEOUT = float(os.getenv("TIMEOUT", 100)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) default_headers = { @@ -45,7 +50,7 @@ async def lifespan(app: FastAPI): # 关闭时的代码 await app.state.client.aclose() -app = FastAPI(lifespan=lifespan) +app = FastAPI(lifespan=lifespan, debug=is_debug) import asyncio from time import time @@ -454,7 +459,7 @@ async def get_user_rate_limit(api_index: str = None): raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") if not api_index or not raw_rate_limit: - return (60, 60) + return (30, 60) rate_limit = parse_rate_limit(raw_rate_limit) return rate_limit @@ -496,6 +501,10 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec raise HTTPException(status_code=403, detail="Permission denied") return token +@app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) +async def request_model(request: Union[RequestModel, ImageGenerationRequest]): + logger.info(f"Request received: {request}") + @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): # logger.info(f"Request received: {request}") From 061e8aa6afc56cf1aae4684b0dd73c59aa1814dc Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Sep 2024 02:46:52 +0800 Subject: [PATCH 042/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Optimize=20log?= =?UTF-8?q?=20display:=20When=20the=20model=20does=20not=20exist,=20displa?= =?UTF-8?q?y=20it=20in=20the=20log.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index f7423046..bb3eee43 100644 --- a/main.py +++ b/main.py @@ -52,6 +52,15 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan, debug=is_debug) +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 404: + logger.error(f"404 Error: {exc.detail}") + return JSONResponse( + status_code=exc.status_code, + content={"message": exc.detail}, + ) + import asyncio from time import time from collections import defaultdict @@ -501,10 +510,6 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec raise HTTPException(status_code=403, detail="Permission denied") return token -@app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) -async def request_model(request: Union[RequestModel, ImageGenerationRequest]): - logger.info(f"Request received: {request}") - @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): # logger.info(f"Request received: {request}") From 5eb8cba3d2a17480e0550c16dfbfec93787d4062 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Sep 2024 03:44:18 +0800 Subject: [PATCH 043/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20lobechat=20gemini=20official=20API=20cannot=20?= =?UTF-8?q?use=20the=20tool.=20The=20reason=20is=20that=20lobechat=20autom?= =?UTF-8?q?atically=20uses=20multiple=20consecutive=20underscores=20in=20t?= =?UTF-8?q?he=20system=20prompt=20to=20name=20function=20names,=20which=20?= =?UTF-8?q?gemini=20does=20not=20support.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/request.py b/request.py index b7da10c1..6c65c7e3 100644 --- a/request.py +++ b/request.py @@ -1,4 +1,5 @@ import os +import re import json from models import RequestModel from utils import c35s, c3s, c3o, c3h, gem, BaseAPI @@ -150,6 +151,7 @@ async def get_gemini_payload(request, engine, provider): elif msg.role != "system": messages.append({"role": msg.role, "parts": content}) elif msg.role == "system": + content[0]["text"] = re.sub(r"_+", "_", content[0]["text"]) systemInstruction = {"parts": content} From 983694e8597e63eef47a9d0d57261716fa47bd88 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Sep 2024 17:20:38 +0800 Subject: [PATCH 044/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20fails=20to=20properly=20catch=20httpx.RemoteProtocolE?= =?UTF-8?q?rror.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index a62d9bcc..e92cbcb9 100644 --- a/utils.py +++ b/utils.py @@ -145,7 +145,7 @@ async def new_generator(): try: async for item in generator: yield ensure_string(item) - except (httpx.ReadError, asyncio.CancelledError) as e: + except (httpx.ReadError, asyncio.CancelledError, httpx.RemoteProtocolError) as e: logger.error(f"Network error in new_generator: {e}") raise From bdfeca1815a5097ae2d3d7f4e8e7e176ae730277 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 15 Sep 2024 01:37:40 +0800 Subject: [PATCH 045/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20next=20channel=20cannot=20be=20automatically?= =?UTF-8?q?=20switched=20when=20an=20error=20occurs=20in=20non-streaming?= =?UTF-8?q?=20output=20for=20o1-mini=20and=20o1-preview.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 7 ++++++- utils.py | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index bb3eee43..407110a7 100644 --- a/main.py +++ b/main.py @@ -241,7 +241,12 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], wrapped_generator = await error_handling_wrapper(generator) response = StreamingResponse(wrapped_generator, media_type="text/event-stream") else: - response = await anext(fetch_response(app.state.client, url, headers, payload)) + generator = fetch_response(app.state.client, url, headers, payload) + wrapped_generator = await error_handling_wrapper(generator) + first_element = await anext(wrapped_generator) + first_element = first_element.lstrip("data: ") + first_element = json.loads(first_element) + response = JSONResponse(first_element) # 更新成功计数 async with app.middleware_stack.app.lock: diff --git a/utils.py b/utils.py index e92cbcb9..f55c2a66 100644 --- a/utils.py +++ b/utils.py @@ -121,10 +121,8 @@ async def error_handling_wrapper(generator): if isinstance(first_item_str, (bytes, bytearray)): first_item_str = first_item_str.decode("utf-8") if isinstance(first_item_str, str): - if first_item_str.startswith("data: "): - first_item_str = first_item_str[6:] - elif first_item_str.startswith("data:"): - first_item_str = first_item_str[5:] + if first_item_str.startswith("data:"): + first_item_str = first_item_str.lstrip("data: ") if first_item_str.startswith("[DONE]"): logger.error("error_handling_wrapper [DONE]!") raise StopAsyncIteration From 5eb10aa655aec99737648c1fbcda1758ad4a3be4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 15 Sep 2024 02:29:44 +0800 Subject: [PATCH 046/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20for=20counting=20model=20usage=20in=20the=20stats?= =?UTF-8?q?=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/main.py b/main.py index 407110a7..adeafd36 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ from fastapi import FastAPI, HTTPException, Depends, Request from fastapi.responses import StreamingResponse, JSONResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi.exceptions import RequestValidationError from models import RequestModel, ImageGenerationRequest from request import get_payload @@ -70,6 +71,14 @@ async def http_exception_handler(request: Request, exc: HTTPException): import json import aiofiles +async def parse_request_body(request: Request): + if request.method == "POST" and "application/json" in request.headers.get("content-type", ""): + try: + return await request.json() + except json.JSONDecodeError: + return None + return None + class StatsMiddleware(BaseHTTPMiddleware): def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats.json"): super().__init__(app) @@ -78,6 +87,7 @@ def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats. self.ip_counts = defaultdict(lambda: defaultdict(int)) self.request_arrivals = defaultdict(list) self.channel_success_counts = defaultdict(int) + self.model_counts = defaultdict(int) self.channel_failure_counts = defaultdict(int) self.lock = asyncio.Lock() self.exclude_paths = set(exclude_paths or []) @@ -91,6 +101,20 @@ def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats. async def dispatch(self, request: Request, call_next): arrival_time = datetime.now() start_time = time() + + # 使用依赖注入获取预解析的请求体 + request.state.parsed_body = await parse_request_body(request) + + model = "unknown" + if request.state.parsed_body: + try: + request_model = RequestModel(**request.state.parsed_body) + model = request_model.model + except RequestValidationError: + pass + except Exception as e: + logger.error(f"Error processing request: {str(e)}") + response = await call_next(request) process_time = time() - start_time @@ -103,6 +127,7 @@ async def dispatch(self, request: Request, call_next): self.request_times[endpoint] += process_time self.ip_counts[endpoint][client_ip] += 1 self.request_arrivals[endpoint].append(arrival_time) + self.model_counts[model] += 1 return response @@ -121,6 +146,7 @@ async def save_stats(self): stats = { "request_counts": dict(self.request_counts), "request_times": dict(self.request_times), + "model_counts": dict(self.model_counts), "ip_counts": {k: dict(v) for k, v in self.ip_counts.items()}, "request_arrivals": {k: [t.isoformat() for t in v] for k, v in self.request_arrivals.items()}, "channel_success_counts": dict(self.channel_success_counts), @@ -553,6 +579,7 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) stats = { "channel_success_percentages": middleware.calculate_success_percentages(), "channel_failure_percentages": middleware.calculate_failure_percentages(), + "model_counts": dict(middleware.model_counts), "request_counts": dict(middleware.request_counts), "request_times": dict(middleware.request_times), "ip_counts": {k: dict(v) for k, v in middleware.ip_counts.items()}, From ed9524453f99ff5f5d2e7ad206d9c189d21ab054 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 15 Sep 2024 18:53:37 +0800 Subject: [PATCH 047/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20middleware=20statistics=20for=20unknown=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index adeafd36..290b5dd3 100644 --- a/main.py +++ b/main.py @@ -127,7 +127,8 @@ async def dispatch(self, request: Request, call_next): self.request_times[endpoint] += process_time self.ip_counts[endpoint][client_ip] += 1 self.request_arrivals[endpoint].append(arrival_time) - self.model_counts[model] += 1 + if model != "unknown": + self.model_counts[model] += 1 return response From a0bdc85ff452aa1eac6c806754a6592389da92f9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 15 Sep 2024 22:51:01 +0800 Subject: [PATCH 048/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20automatic=20restart=20after=20detecting=20changes?= =?UTF-8?q?=20to=20api.yaml,=20no=20need=20to=20manually=20restart=20the?= =?UTF-8?q?=20container.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 290b5dd3..8d1a2bfd 100644 --- a/main.py +++ b/main.py @@ -602,6 +602,8 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) host="0.0.0.0", port=8000, reload=True, + reload_dirs=["./"], + reload_includes=["*.py", "api.yaml"], ws="none", # log_level="warning" ) \ No newline at end of file From 52fb3318acedddd22cd321ab33781f2ce7d2dc9b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 16 Sep 2024 02:16:18 +0800 Subject: [PATCH 049/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20uvicorn=20cannot=20monitor=20changes=20to=20the=20ap?= =?UTF-8?q?i.yaml=20file=20inside=20docker.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 ++- requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95fc0423..9c0c76c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,5 @@ EXPOSE 8000 WORKDIR /home COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages COPY . /home -ENTRYPOINT ["python", "-u", "/home/main.py"] \ No newline at end of file +ENV WATCHFILES_FORCE_POLLING=true +ENTRYPOINT ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--reload", "--reload-include", "*.yaml"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 00f087b2..2a6c0992 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ pytest uvicorn fastapi aiofiles +watchfiles httpx[http2] cryptography \ No newline at end of file From 252357cdaf82e41bba744567b587be03056ce616 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 16 Sep 2024 16:06:26 +0800 Subject: [PATCH 050/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20treats=20api.yaml=20as=20a=20folder.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index f55c2a66..621b5941 100644 --- a/utils.py +++ b/utils.py @@ -72,11 +72,14 @@ async def load_config(app=None): # logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") config, api_keys_db, api_list = [], [], [] except FileNotFoundError: - logger.error("配置文件 'api.yaml' 未找到。请确保文件存在于正确的位置。") + logger.error("'api.yaml' not found. Please check the file path.") config, api_keys_db, api_list = [], [], [] except yaml.YAMLError: logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。") config, api_keys_db, api_list = [], [], [] + except OSError as e: + logger.error(f"open 'api.yaml' failed: {e}") + config, api_keys_db, api_list = [], [], [] if config != []: return config, api_keys_db, api_list From eb02b52d091109caa3eecc69f6255628be48dd40 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Sep 2024 01:33:45 +0800 Subject: [PATCH 051/476] =?UTF-8?q?=F0=9F=A4=96=20Models:=20Add=20support?= =?UTF-8?q?=20for=20the=20cohere=20series=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation --- README.md | 6 ++-- README_CN.md | 6 ++-- main.py | 6 +++- request.py | 80 ++++++++++++++++++++++++++++++++++++++--- response.py | 26 ++++++++++++++ test/test_matplotlib.py | 73 ++++++++++++++++++++++++++++++++++++- 6 files changed, 184 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6581db7a..9f133d3a 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ ## Introduction -If used personally, one/new-api is too complex and has many commercial functions that individuals do not need. If you do not want a complicated front-end interface and want to support more models, you can try uni-api. This is a project for unified management of large model APIs, allowing you to call multiple backend services through a unified API interface, converting them uniformly to OpenAI format and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cloudflare, DeepBricks, OpenRouter, etc. +If used for personal purposes, one/new-api is too complex and has many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project that manages large model APIs uniformly and allows you to call multiple backend services through a unified API interface, converting them uniformly to the OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Cloudflare, DeepBricks, OpenRouter, etc. ## Features - No frontend, pure configuration file setup for API channels. You can run your own API site by just writing one file, with detailed configuration guides in the documentation, beginner-friendly. - Unified management of multiple backend services, supporting providers like OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in the OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex API, and Cloudflare. Vertex supports both Claude and Gemini API. +- Supports Anthropic, Gemini, Vertex AI, Cohere, Cloudflare. Vertex supports both Claude and Gemini API. - Supports OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Supports four types of load balancing. @@ -125,7 +125,7 @@ api_keys: ## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 40 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional. +- TIMEOUT: Request timeout, default is 100 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional ## Docker Local Deployment diff --git a/README_CN.md b/README_CN.md index 278a5402..8b73ff28 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,13 +13,13 @@ ## Introduction -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、cloudflare、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Cloudflare、DeepBricks、OpenRouter 等。 ## Features - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 - 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex API、cloudflare。Vertex 同时支持 Claude 和 Gemini API。 +- 同时支持 Anthropic、Gemini、Vertex AI、Cohere、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 - 支持四种负载均衡。 @@ -125,7 +125,7 @@ api_keys: ## 环境变量 - CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 -- TIMEOUT: 请求超时时间，默认为 40 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +- TIMEOUT: 请求超时时间，默认为 100 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 ## Docker Local Deployment diff --git a/main.py b/main.py index 8d1a2bfd..2de9d9cf 100644 --- a/main.py +++ b/main.py @@ -229,13 +229,17 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], engine = "claude" elif parsed_url.netloc == 'openrouter.ai': engine = "openrouter" + elif parsed_url.netloc == 'api.cohere.com': + engine = "cohere" + request.stream = True else: engine = "gpt" if "claude" not in provider['model'][request.model] \ and "gpt" not in provider['model'][request.model] \ and "gemini" not in provider['model'][request.model] \ - and parsed_url.netloc != 'api.cloudflare.com': + and parsed_url.netloc != 'api.cloudflare.com' \ + and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" if "claude" in provider['model'][request.model] and engine == "vertex": diff --git a/request.py b/request.py index 6c65c7e3..5dbcc8cb 100644 --- a/request.py +++ b/request.py @@ -1,12 +1,13 @@ import os import re import json -from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, BaseAPI - +import httpx import base64 import urllib.parse +from models import RequestModel +from utils import c35s, c3s, c3o, c3h, gem, BaseAPI + def encode_image(image_path): with open(image_path, "rb") as image_file: return base64.b64encode(image_file.read()).decode('utf-8') @@ -82,6 +83,8 @@ async def get_text_message(role, message, engine = None): return {"text": message} if engine == "cloudflare": return message + if engine == "cohere": + return message raise ValueError("Unknown engine") async def get_gemini_payload(request, engine, provider): @@ -215,8 +218,6 @@ async def get_gemini_payload(request, engine, provider): return url, headers, payload import time -import httpx -import base64 from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.serialization import load_pem_private_key @@ -690,6 +691,73 @@ async def get_openrouter_payload(request, engine, provider): return url, headers, payload +async def get_cohere_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + + url = provider['base_url'] + + role_map = { + "user": "USER", + "assistant" : "CHATBOT", + "system": "SYSTEM" + } + + messages = [] + for msg in request.messages: + if isinstance(msg.content, list): + content = [] + for item in msg.content: + if item.type == "text": + text_message = await get_text_message(msg.role, item.text, engine) + content.append(text_message) + else: + content = msg.content + + if isinstance(content, list): + for item in content: + if item["type"] == "text": + messages.append({"role": role_map[msg.role], "message": item["text"]}) + else: + messages.append({"role": role_map[msg.role], "message": content}) + + model = provider['model'][request.model] + chat_history = messages[:-1] + query = messages[-1].get("message") + payload = { + "model": model, + "message": query, + } + + if chat_history: + payload["chat_history"] = chat_history + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + async def get_cloudflare_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' @@ -989,6 +1057,8 @@ async def get_payload(request: RequestModel, engine, provider): return await get_cloudflare_payload(request, engine, provider) elif engine == "o1": return await get_o1_payload(request, engine, provider) + elif engine == "cohere": + return await get_cohere_payload(request, engine, provider) elif engine == "dalle": return await get_dalle_payload(request, engine, provider) else: diff --git a/response.py b/response.py index be2f1c3c..b67eb6ae 100644 --- a/response.py +++ b/response.py @@ -184,6 +184,29 @@ async def fetch_cloudflare_response_stream(client, url, headers, payload, model) sse_string = await generate_sse_response(timestamp, model, content=message) yield sse_string +async def fetch_cohere_response_stream(client, url, headers, payload, model): + timestamp = int(datetime.timestamp(datetime.now())) + async with client.stream('POST', url, headers=headers, json=payload) as response: + error_message = await check_response(response, "fetch_gpt_response_stream") + if error_message: + yield error_message + return + + buffer = "" + async for chunk in response.aiter_text(): + buffer += chunk + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + # logger.info("line: %s", repr(line)) + resp: dict = json.loads(line) + if resp.get("is_finished") == True: + yield "data: [DONE]\n\r\n" + return + if resp.get("event_type") == "text-generation": + message = resp.get("text") + sse_string = await generate_sse_response(timestamp, model, content=message) + yield sse_string + async def fetch_claude_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: @@ -270,6 +293,9 @@ async def fetch_response_stream(client, url, headers, payload, engine, model): elif engine == "cloudflare": async for chunk in fetch_cloudflare_response_stream(client, url, headers, payload, model): yield chunk + elif engine == "cohere": + async for chunk in fetch_cohere_response_stream(client, url, headers, payload, model): + yield chunk else: raise ValueError("Unknown response") except httpx.ConnectError as e: diff --git a/test/test_matplotlib.py b/test/test_matplotlib.py index 41601bc3..881a1206 100644 --- a/test/test_matplotlib.py +++ b/test/test_matplotlib.py @@ -2,6 +2,7 @@ import matplotlib.pyplot as plt from datetime import datetime, timedelta from collections import defaultdict +import numpy as np import matplotlib.font_manager as fm font_path = '/System/Library/Fonts/PingFang.ttc' @@ -45,5 +46,75 @@ def create_pic(request_arrivals, key): # 保存图片 plt.savefig(f'{key.replace("/", "")}.png') +def create_pie_chart(model_counts): + models = list(model_counts.keys()) + counts = list(model_counts.values()) + + # 设置颜色和排列顺序 + colors = plt.cm.Set3(np.linspace(0, 1, len(models))) + sorted_data = sorted(zip(counts, models, colors), reverse=True) + counts, models, colors = zip(*sorted_data) + + # 创建饼图 + fig, ax = plt.subplots(figsize=(16, 10)) + wedges, _ = ax.pie(counts, colors=colors, startangle=90, wedgeprops=dict(width=0.5)) + + # 添加圆环效果 + centre_circle = plt.Circle((0, 0), 0.35, fc='white') + fig.gca().add_artist(centre_circle) + + # 计算总数 + total = sum(counts) + + # 准备标注 + bbox_props = dict(boxstyle="round,pad=0.3", fc="w", ec="k", lw=0.72) + kw = dict(xycoords='data', textcoords='data', arrowprops=dict(arrowstyle="-"), bbox=bbox_props, zorder=0) + + left_labels = [] + right_labels = [] + + for i, p in enumerate(wedges): + ang = (p.theta2 - p.theta1) / 2. + p.theta1 + y = np.sin(np.deg2rad(ang)) + x = np.cos(np.deg2rad(ang)) + + percentage = counts[i] / total * 100 + label = f"{models[i]}: {percentage:.1f}%" + + if x > 0: + right_labels.append((x, y, label)) + else: + left_labels.append((x, y, label)) + + # 绘制左侧标注 + for i, (x, y, label) in enumerate(left_labels): + ax.annotate(label, xy=(x, y), xytext=(-1.2, 0.9 - i * 0.15), **kw) + + # 绘制右侧标注 + for i, (x, y, label) in enumerate(right_labels): + ax.annotate(label, xy=(x, y), xytext=(1.2, 0.9 - i * 0.15), **kw) + + plt.title("各模型使用次数对比", size=16) + ax.set_xlim(-1.5, 1.5) + ax.set_ylim(-1.2, 1.2) + ax.axis('off') + plt.tight_layout() + plt.savefig('model_usage_pie_chart.png', bbox_inches='tight', pad_inches=0.5) + if __name__ == '__main__': - create_pic(request_arrivals, 'POST /v1/chat/completions') \ No newline at end of file + model_counts = { + "model_counts": { + "claude-3-5-sonnet": 94, + "o1-preview": 71, + "gpt-4o": 512, + "gpt-4o-mini": 5, + "gemini-1.5-pro": 5, + "deepseek-chat": 7, + "grok-2-mini": 1, + "grok-2": 9, + "o1-mini": 8 + } + } + # create_pic(request_arrivals, 'POST /v1/chat/completions') + + create_pie_chart(model_counts["model_counts"]) From 864c752ea43af62d1e4248e1d3905f3bbfe953bd Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Sep 2024 02:16:34 +0800 Subject: [PATCH 052/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20fixed=20message=20for=20pass-through=20did=20n?= =?UTF-8?q?ot=20check=20for=20generation=20failure.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.py b/utils.py index 621b5941..dc7a8119 100644 --- a/utils.py +++ b/utils.py @@ -129,6 +129,9 @@ async def error_handling_wrapper(generator): if first_item_str.startswith("[DONE]"): logger.error("error_handling_wrapper [DONE]!") raise StopAsyncIteration + if "The bot's usage is covered by the developer" in first_item_str: + logger.error("error const string!") + raise StopAsyncIteration try: first_item_str = json.loads(first_item_str) except json.JSONDecodeError: From cfee5f1de5cb6780143342978380df84705eca30 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Sep 2024 15:44:22 +0800 Subject: [PATCH 053/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- README_CN.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9f133d3a..64ff8833 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ ## Introduction -If used for personal purposes, one/new-api is too complex and has many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project that manages large model APIs uniformly and allows you to call multiple backend services through a unified API interface, converting them uniformly to the OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Cloudflare, DeepBricks, OpenRouter, etc. +If used personally, one/new-api is too complex and has many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project that unifies the management of large model APIs, allowing multiple backend services to be called through a unified API interface and uniformly converted to the OpenAI format, supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, DeepBricks, OpenRouter, etc. ## Features - No frontend, pure configuration file setup for API channels. You can run your own API site by just writing one file, with detailed configuration guides in the documentation, beginner-friendly. - Unified management of multiple backend services, supporting providers like OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in the OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex AI, Cohere, Cloudflare. Vertex supports both Claude and Gemini API. +- Supports Anthropic, Gemini, Vertex AI, Cohere, Groq, Cloudflare. Vertex supports both Claude and Gemini APIs. - Supports OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Supports four types of load balancing. diff --git a/README_CN.md b/README_CN.md index 8b73ff28..46e4fe1a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,13 +13,13 @@ ## Introduction -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Cloudflare、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Groq、Cloudflare、DeepBricks、OpenRouter 等。 ## Features - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 - 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex AI、Cohere、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 +- 同时支持 Anthropic、Gemini、Vertex AI、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 - 支持四种负载均衡。 From e5b8220deb76ebe6b3d230477682eb6fc7ba9e93 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Sep 2024 22:19:27 +0800 Subject: [PATCH 054/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20o1=20model=20cannot=20obtain=20text=20format.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index 5dbcc8cb..75d0e3a5 100644 --- a/request.py +++ b/request.py @@ -77,7 +77,7 @@ async def get_image_message(base64_image, engine = None): raise ValueError("Unknown engine") async def get_text_message(role, message, engine = None): - if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine: + if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine or "o1" == engine: return {"type": "text", "text": message} if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} From 1d1b0f17924d7a5aa3f58160c7571129446d459e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 25 Sep 2024 17:06:30 +0800 Subject: [PATCH 055/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20database,?= =?UTF-8?q?=20count=20the=20first=20character=20time.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation --- .dockerignore | 6 +- .gitignore | 3 +- README.md | 12 +- README_CN.md | 12 +- docker-compose.yml | 3 +- main.py | 305 ++++++++++++++++++++++++------------------ requirements.txt | 3 + test/provider_test.py | 3 +- utils.py | 5 +- 9 files changed, 207 insertions(+), 145 deletions(-) diff --git a/.dockerignore b/.dockerignore index b5f14095..23478050 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ api.yaml test -json_str \ No newline at end of file +json_str +*.jpg +*.json +*.png +*.db \ No newline at end of file diff --git a/.gitignore b/.gitignore index c026a68b..fd12d49c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ node_modules .pytest_cache *.jpg *.json -*.png \ No newline at end of file +*.png +*.db \ No newline at end of file diff --git a/README.md b/README.md index 64ff8833..f71cc40f 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,9 @@ Start the container ```bash docker run --user root -p 8001:8000 --name uni-api -dit \ --v ./api.yaml:/home/api.yaml \ +-e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file is already mounted, you do not need to set CONFIG_URL +-v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, you do not need to mount the configuration file +-v ./stats.db:/home/stats.db \ # If you do not want to save statistical data, you do not need to mount the stats.db file yym68686/uni-api:latest ``` @@ -145,14 +147,15 @@ services: container_name: uni-api image: yym68686/uni-api:latest environment: - - CONFIG_URL=http://file_url/api.yaml + - CONFIG_URL=http://file_url/api.yaml # If the local configuration file is already mounted, there is no need to set CONFIG_URL ports: - 8001:8000 volumes: - - ./api.yaml:/home/api.yaml + - ./api.yaml:/home/api.yaml # If CONFIG_URL is already set, there is no need to mount the configuration file + - ./stats.db:/home/stats.db # If you do not want to save statistical data, there is no need to mount the stats.db file ``` -CONFIG_URL is a link that can automatically download a remote configuration file. For example, if you find it inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service that provides a direct link for uni-api to download. CONFIG_URL is this direct link. +CONFIG_URL is used to automatically download remote configuration files. For example, if it is inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. If you are using a locally mounted configuration file, you do not need to set CONFIG_URL. CONFIG_URL is used in situations where it is inconvenient to mount the configuration file. Run Docker Compose container in the background @@ -178,6 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ +-v ./stats.db:/home/stats.db \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/README_CN.md b/README_CN.md index 46e4fe1a..a46340ad 100644 --- a/README_CN.md +++ b/README_CN.md @@ -133,7 +133,9 @@ Start the container ```bash docker run --user root -p 8001:8000 --name uni-api -dit \ --v ./api.yaml:/home/api.yaml \ +-e CONFIG_URL=http://file_url/api.yaml \ # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL +-v ./api.yaml:/home/api.yaml \ # 如果已经设置 CONFIG_URL，不需要挂载配置文件 +-v ./stats.db:/home/stats.db \ # 如果不想保存统计数据，不需要挂载 stats.db 文件 yym68686/uni-api:latest ``` @@ -145,14 +147,15 @@ services: container_name: uni-api image: yym68686/uni-api:latest environment: - - CONFIG_URL=http://file_url/api.yaml + - CONFIG_URL=http://file_url/api.yaml # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL ports: - 8001:8000 volumes: - - ./api.yaml:/home/api.yaml + - ./api.yaml:/home/api.yaml # 如果已经设置 CONFIG_URL，不需要挂载配置文件 + - ./stats.db:/home/stats.db # 如果不想保存统计数据，不需要挂载 stats.db 文件 ``` -CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。 +CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。如果使用本地挂载的配置文件，不需要设置 CONFIG_URL。CONFIG_URL 是在不方便挂载配置文件的情况下使用。 Run Docker Compose container in the background @@ -178,6 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ +-v ./stats.db:/home/stats.db \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/docker-compose.yml b/docker-compose.yml index 644fe2b3..61e2bb89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,4 +7,5 @@ services: ports: - 8001:8000 volumes: - - ./api.yaml:/home/api.yaml \ No newline at end of file + - ./api.yaml:/home/api.yaml + - ./stats.db:/home/stats.db \ No newline at end of file diff --git a/main.py b/main.py index 2de9d9cf..004693b4 100644 --- a/main.py +++ b/main.py @@ -22,15 +22,16 @@ from urllib.parse import urlparse import os -is_debug = os.getenv("DEBUG", False) +is_debug = bool(os.getenv("DEBUG", False)) + +async def create_tables(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 - - # # 启动事件 - # routes = [{"path": route.path, "name": route.name} for route in app.routes] - # logger.info(f"Registered routes: {routes}") + await create_tables() TIMEOUT = float(os.getenv("TIMEOUT", 100)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) @@ -66,10 +67,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): from time import time from collections import defaultdict from starlette.middleware.base import BaseHTTPMiddleware -from datetime import datetime -from datetime import timedelta import json -import aiofiles async def parse_request_body(request: Request): if request.method == "POST" and "application/json" in request.headers.get("content-type", ""): @@ -79,30 +77,53 @@ async def parse_request_body(request: Request): return None return None +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import declarative_base, sessionmaker +from sqlalchemy import Column, Integer, String, Float, DateTime, select, Boolean +from sqlalchemy.sql import func + +# 定义数据库模型 +Base = declarative_base() + +class RequestStat(Base): + __tablename__ = 'request_stats' + id = Column(Integer, primary_key=True) + endpoint = Column(String) + ip = Column(String) + token = Column(String) + total_time = Column(Float) + model = Column(String) + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + +class ChannelStat(Base): + __tablename__ = 'channel_stats' + id = Column(Integer, primary_key=True) + provider = Column(String) + model = Column(String) + api_key = Column(String) + success = Column(Boolean) + first_response_time = Column(Float) # 新增: 记录首次响应时间 + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + +# 创建异步引擎和会话 +engine = create_async_engine('sqlite+aiosqlite:///stats.db', echo=is_debug) +async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + class StatsMiddleware(BaseHTTPMiddleware): - def __init__(self, app, exclude_paths=None, save_interval=3600, filename="stats.json"): + def __init__(self, app): super().__init__(app) - self.request_counts = defaultdict(int) - self.request_times = defaultdict(float) - self.ip_counts = defaultdict(lambda: defaultdict(int)) - self.request_arrivals = defaultdict(list) - self.channel_success_counts = defaultdict(int) - self.model_counts = defaultdict(int) - self.channel_failure_counts = defaultdict(int) - self.lock = asyncio.Lock() - self.exclude_paths = set(exclude_paths or []) - self.save_interval = save_interval - self.filename = filename - self.last_save_time = time() - - # 启动定期保存和清理任务 - asyncio.create_task(self.periodic_save_and_cleanup()) + self.db = async_session() async def dispatch(self, request: Request, call_next): - arrival_time = datetime.now() + if request.headers.get("x-api-key"): + token = request.headers.get("x-api-key") + elif request.headers.get("Authorization"): + token = request.headers.get("Authorization").split(" ")[1] + else: + token = None + start_time = time() - # 使用依赖注入获取预解析的请求体 request.state.parsed_body = await parse_request_body(request) model = "unknown" @@ -121,86 +142,35 @@ async def dispatch(self, request: Request, call_next): endpoint = f"{request.method} {request.url.path}" client_ip = request.client.host - if request.url.path not in self.exclude_paths: - async with self.lock: - self.request_counts[endpoint] += 1 - self.request_times[endpoint] += process_time - self.ip_counts[endpoint][client_ip] += 1 - self.request_arrivals[endpoint].append(arrival_time) - if model != "unknown": - self.model_counts[model] += 1 + # 异步更新数据库 + await self.update_stats(endpoint, process_time, client_ip, model, token) return response - async def periodic_save_and_cleanup(self): - while True: - await asyncio.sleep(self.save_interval) - await self.save_stats() - await self.cleanup_old_data() - - async def save_stats(self): - current_time = time() - if current_time - self.last_save_time < self.save_interval: - return - - async with self.lock: - stats = { - "request_counts": dict(self.request_counts), - "request_times": dict(self.request_times), - "model_counts": dict(self.model_counts), - "ip_counts": {k: dict(v) for k, v in self.ip_counts.items()}, - "request_arrivals": {k: [t.isoformat() for t in v] for k, v in self.request_arrivals.items()}, - "channel_success_counts": dict(self.channel_success_counts), - "channel_failure_counts": dict(self.channel_failure_counts), - "channel_success_percentages": self.calculate_success_percentages(), - "channel_failure_percentages": self.calculate_failure_percentages() - } - - filename = self.filename - async with aiofiles.open(filename, mode='w') as f: - await f.write(json.dumps(stats, indent=2)) - - self.last_save_time = current_time - - def calculate_success_percentages(self): - percentages = {} - for channel, success_count in self.channel_success_counts.items(): - total_count = success_count + self.channel_failure_counts[channel] - if total_count > 0: - percentages[channel] = success_count / total_count * 100 - else: - percentages[channel] = 0 - - sorted_percentages = dict(sorted(percentages.items(), key=lambda item: item[1], reverse=True)) - return sorted_percentages - - def calculate_failure_percentages(self): - percentages = {} - for channel, failure_count in self.channel_failure_counts.items(): - total_count = failure_count + self.channel_success_counts[channel] - if total_count > 0: - percentages[channel] = failure_count / total_count * 100 - else: - percentages[channel] = 0 - - sorted_percentages = dict(sorted(percentages.items(), key=lambda item: item[1], reverse=True)) - return sorted_percentages - - async def cleanup_old_data(self): - cutoff_time = datetime.now() - timedelta(hours=24) - async with self.lock: - for endpoint in list(self.request_arrivals.keys()): - self.request_arrivals[endpoint] = [ - t for t in self.request_arrivals[endpoint] if t > cutoff_time - ] - if not self.request_arrivals[endpoint]: - del self.request_arrivals[endpoint] - self.request_counts.pop(endpoint, None) - self.request_times.pop(endpoint, None) - self.ip_counts.pop(endpoint, None) - - async def cleanup(self): - await self.save_stats() + async def update_stats(self, endpoint, process_time, client_ip, model, token): + async with self.db as session: + # 为每个请求创建一条新的记录 + new_request_stat = RequestStat( + endpoint=endpoint, + ip=client_ip, + token=token, + total_time=process_time, + model=model + ) + session.add(new_request_stat) + await session.commit() + + async def update_channel_stats(self, provider, model, api_key, success, first_response_time): + async with self.db as session: + channel_stat = ChannelStat( + provider=provider, + model=model, + api_key=api_key, + success=success, + first_response_time=first_response_time + ) + session.add(channel_stat) + await session.commit() # 配置 CORS 中间件 app.add_middleware( @@ -211,10 +181,10 @@ async def cleanup(self): allow_headers=["*"], # 允许所有头部字段 ) -app.add_middleware(StatsMiddleware, exclude_paths=["/stats", "/generate-api-key"]) +app.add_middleware(StatsMiddleware) # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None, token=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -269,25 +239,23 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], if request.stream: model = provider['model'][request.model] generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) - wrapped_generator = await error_handling_wrapper(generator) + wrapped_generator, first_response_time = await error_handling_wrapper(generator) response = StreamingResponse(wrapped_generator, media_type="text/event-stream") else: generator = fetch_response(app.state.client, url, headers, payload) - wrapped_generator = await error_handling_wrapper(generator) + wrapped_generator, first_response_time = await error_handling_wrapper(generator) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") first_element = json.loads(first_element) response = JSONResponse(first_element) - # 更新成功计数 - async with app.middleware_stack.app.lock: - app.middleware_stack.app.channel_success_counts[provider['provider']] += 1 + # 更新成功计数和首次响应时间 + await app.middleware_stack.app.update_channel_stats(provider['provider'], request.model, token, success=True, first_response_time=first_response_time) return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: - # 更新失败计数 - async with app.middleware_stack.app.lock: - app.middleware_stack.app.channel_failure_counts[provider['provider']] += 1 + # 更新失败计数,首次响应时间为-1表示失败 + await app.middleware_stack.app.update_channel_stats(provider['provider'], request.model, token, success=False, first_response_time=-1) raise e @@ -421,10 +389,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY") == False: auto_retry = False - return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint) + return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token) # 在 try_all_providers 函数中处理失败的情况 - async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None): + async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): status_code = 500 error_message = None num_providers = len(providers) @@ -433,7 +401,7 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe self.last_provider_index = (start_index + i) % num_providers provider = providers[self.last_provider_index] try: - response = await process_request(request, provider, endpoint) + response = await process_request(request, provider, endpoint, token) return response except HTTPException as e: logger.error(f"Error with provider {provider['provider']}: {str(e)}") @@ -510,6 +478,7 @@ async def get_user_rate_limit(api_index: str = None): return rate_limit security = HTTPBearer() + async def rate_limit_dependency(request: Request, credentials: HTTPAuthorizationCredentials = Depends(security)): token = credentials.credentials if credentials else None api_list = app.state.api_list @@ -576,24 +545,96 @@ def generate_api_key(): return JSONResponse(content={"api_key": api_key}) # 在 /stats 路由中返回成功和失败百分比 +from collections import defaultdict +from sqlalchemy import func + +from collections import defaultdict +from sqlalchemy import func, desc, case + @app.get("/stats", dependencies=[Depends(rate_limit_dependency)]) async def get_stats(request: Request, token: str = Depends(verify_admin_api_key)): - middleware = app.middleware_stack.app - if isinstance(middleware, StatsMiddleware): - async with middleware.lock: - stats = { - "channel_success_percentages": middleware.calculate_success_percentages(), - "channel_failure_percentages": middleware.calculate_failure_percentages(), - "model_counts": dict(middleware.model_counts), - "request_counts": dict(middleware.request_counts), - "request_times": dict(middleware.request_times), - "ip_counts": {k: dict(v) for k, v in middleware.ip_counts.items()}, - "request_arrivals": {k: [t.isoformat() for t in v] for k, v in middleware.request_arrivals.items()}, - "channel_success_counts": dict(middleware.channel_success_counts), - "channel_failure_counts": dict(middleware.channel_failure_counts), - } - return JSONResponse(content=stats) - return {"error": "StatsMiddleware not found"} + async with async_session() as session: + # 1. 每个渠道下面每个模型的成功率 + channel_model_stats = await session.execute( + select( + ChannelStat.provider, + ChannelStat.model, + func.count().label('total'), + func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') + ).group_by(ChannelStat.provider, ChannelStat.model) + ) + channel_model_stats = channel_model_stats.fetchall() + + # 2. 每个渠道总的成功率 + channel_stats = await session.execute( + select( + ChannelStat.provider, + func.count().label('total'), + func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') + ).group_by(ChannelStat.provider) + ) + channel_stats = channel_stats.fetchall() + + # 3. 每个模型在所有渠道总的请求次数 + model_stats = await session.execute( + select(ChannelStat.model, func.count().label('count')) + .group_by(ChannelStat.model) + .order_by(desc('count')) + ) + model_stats = model_stats.fetchall() + + # 4. 每个端点的请求次数 + endpoint_stats = await session.execute( + select(RequestStat.endpoint, func.count().label('count')) + .group_by(RequestStat.endpoint) + .order_by(desc('count')) + ) + endpoint_stats = endpoint_stats.fetchall() + + # 5. 每个ip请求的次数 + ip_stats = await session.execute( + select(RequestStat.ip, func.count().label('count')) + .group_by(RequestStat.ip) + .order_by(desc('count')) + ) + ip_stats = ip_stats.fetchall() + + # 处理统计数据并返回 + stats = { + "channel_model_success_rates": [ + { + "provider": stat.provider, + "model": stat.model, + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0 + } for stat in sorted(channel_model_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) + ], + "channel_success_rates": [ + { + "provider": stat.provider, + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0 + } for stat in sorted(channel_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) + ], + "model_request_counts": [ + { + "model": stat.model, + "count": stat.count + } for stat in model_stats + ], + "endpoint_request_counts": [ + { + "endpoint": stat.endpoint, + "count": stat.count + } for stat in endpoint_stats + ], + "ip_request_counts": [ + { + "ip": stat.ip, + "count": stat.count + } for stat in ip_stats + ] + } + + return JSONResponse(content=stats) # async def on_fetch(request, env): # import asgi diff --git a/requirements.txt b/requirements.txt index 2a6c0992..5040ab13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,9 @@ pytest uvicorn fastapi aiofiles +greenlet +aiosqlite +sqlalchemy watchfiles httpx[http2] cryptography \ No newline at end of file diff --git a/test/provider_test.py b/test/provider_test.py index e83b0de2..fc00d88b 100644 --- a/test/provider_test.py +++ b/test/provider_test.py @@ -70,7 +70,8 @@ def test_request_model(test_client, api_key, get_model): } } } - ] + ], + "tool_choice": "auto" } headers = { diff --git a/utils.py b/utils.py index dc7a8119..ff472b75 100644 --- a/utils.py +++ b/utils.py @@ -116,9 +116,12 @@ def ensure_string(item): return str(item) import asyncio +import time as time_module async def error_handling_wrapper(generator): + start_time = time_module.time() try: first_item = await generator.__anext__() + first_response_time = time_module.time() - start_time first_item_str = first_item # logger.info("first_item_str: %s", first_item_str) if isinstance(first_item_str, (bytes, bytearray)): @@ -153,7 +156,7 @@ async def new_generator(): logger.error(f"Network error in new_generator: {e}") raise - return new_generator() + return new_generator(), first_response_time except StopAsyncIteration: raise HTTPException(status_code=400, detail="data: {'error': 'No data returned'}") From 90a8f679dab71276815ca97d63e91c3d8ab8b09e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 25 Sep 2024 17:36:03 +0800 Subject: [PATCH 056/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20database=20cannot=20be=20automatically=20creat?= =?UTF-8?q?ed=20when=20it=20does=20not=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.py b/main.py index 004693b4..cdf7acee 100644 --- a/main.py +++ b/main.py @@ -31,6 +31,9 @@ async def create_tables(): @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 + if not os.path.exists("stats.db"): + # 如果数据库文件不存在，创建它 + open("stats.db", 'a').close() await create_tables() TIMEOUT = float(os.getenv("TIMEOUT", 100)) From a37c1d177519954a39116f70580169ee713f3b16 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 25 Sep 2024 17:49:49 +0800 Subject: [PATCH 057/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20database=20cannot=20be=20automatically=20creat?= =?UTF-8?q?ed=20when=20it=20does=20not=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation --- README.md | 6 +++--- README_CN.md | 6 +++--- docker-compose.yml | 2 +- main.py | 3 --- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f71cc40f..80bf942c 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ Start the container docker run --user root -p 8001:8000 --name uni-api -dit \ -e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file is already mounted, you do not need to set CONFIG_URL -v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, you do not need to mount the configuration file --v ./stats.db:/home/stats.db \ # If you do not want to save statistical data, you do not need to mount the stats.db file +-v ./stats.db:/stats.db \ # If you do not want to save statistical data, you do not need to mount the stats.db file yym68686/uni-api:latest ``` @@ -152,7 +152,7 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml # If CONFIG_URL is already set, there is no need to mount the configuration file - - ./stats.db:/home/stats.db # If you do not want to save statistical data, there is no need to mount the stats.db file + - ./stats.db:/stats.db # If you do not want to save statistical data, there is no need to mount the stats.db file ``` CONFIG_URL is used to automatically download remote configuration files. For example, if it is inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. If you are using a locally mounted configuration file, you do not need to set CONFIG_URL. CONFIG_URL is used in situations where it is inconvenient to mount the configuration file. @@ -181,7 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ --v ./stats.db:/home/stats.db \ +-v ./stats.db:/stats.db \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/README_CN.md b/README_CN.md index a46340ad..4459a4b6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -135,7 +135,7 @@ Start the container docker run --user root -p 8001:8000 --name uni-api -dit \ -e CONFIG_URL=http://file_url/api.yaml \ # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL -v ./api.yaml:/home/api.yaml \ # 如果已经设置 CONFIG_URL，不需要挂载配置文件 --v ./stats.db:/home/stats.db \ # 如果不想保存统计数据，不需要挂载 stats.db 文件 +-v ./stats.db:/stats.db \ # 如果不想保存统计数据，不需要挂载 stats.db 文件 yym68686/uni-api:latest ``` @@ -152,7 +152,7 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml # 如果已经设置 CONFIG_URL，不需要挂载配置文件 - - ./stats.db:/home/stats.db # 如果不想保存统计数据，不需要挂载 stats.db 文件 + - ./stats.db:/stats.db # 如果不想保存统计数据，不需要挂载 stats.db 文件 ``` CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。如果使用本地挂载的配置文件，不需要设置 CONFIG_URL。CONFIG_URL 是在不方便挂载配置文件的情况下使用。 @@ -181,7 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ --v ./stats.db:/home/stats.db \ +-v ./stats.db:/stats.db \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/docker-compose.yml b/docker-compose.yml index 61e2bb89..d95e84cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,4 +8,4 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml - - ./stats.db:/home/stats.db \ No newline at end of file + - ./stats.db:/stats.db \ No newline at end of file diff --git a/main.py b/main.py index cdf7acee..004693b4 100644 --- a/main.py +++ b/main.py @@ -31,9 +31,6 @@ async def create_tables(): @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 - if not os.path.exists("stats.db"): - # 如果数据库文件不存在，创建它 - open("stats.db", 'a').close() await create_tables() TIMEOUT = float(os.getenv("TIMEOUT", 100)) From 79bd233ddfb17b1eb273453ca440b5e857aab827 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 01:12:38 +0800 Subject: [PATCH 058/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20the=20host=20machine=20from=20synchronizin?= =?UTF-8?q?g=20database=20files.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- README_CN.md | 6 +++--- docker-compose.yml | 2 +- main.py | 9 ++++++++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 80bf942c..bd02ca01 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ Start the container docker run --user root -p 8001:8000 --name uni-api -dit \ -e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file is already mounted, you do not need to set CONFIG_URL -v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, you do not need to mount the configuration file --v ./stats.db:/stats.db \ # If you do not want to save statistical data, you do not need to mount the stats.db file +-v ./uniapi_db:/home/data \ # If you do not want to save statistical data, you do not need to mount the stats.db file yym68686/uni-api:latest ``` @@ -152,7 +152,7 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml # If CONFIG_URL is already set, there is no need to mount the configuration file - - ./stats.db:/stats.db # If you do not want to save statistical data, there is no need to mount the stats.db file + - ./uniapi_db:/home/data # If you do not want to save statistical data, there is no need to mount the stats.db file ``` CONFIG_URL is used to automatically download remote configuration files. For example, if it is inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. If you are using a locally mounted configuration file, you do not need to set CONFIG_URL. CONFIG_URL is used in situations where it is inconvenient to mount the configuration file. @@ -181,7 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ --v ./stats.db:/stats.db \ +-v ./uniapi_db:/home/data \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/README_CN.md b/README_CN.md index 4459a4b6..4f08f959 100644 --- a/README_CN.md +++ b/README_CN.md @@ -135,7 +135,7 @@ Start the container docker run --user root -p 8001:8000 --name uni-api -dit \ -e CONFIG_URL=http://file_url/api.yaml \ # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL -v ./api.yaml:/home/api.yaml \ # 如果已经设置 CONFIG_URL，不需要挂载配置文件 --v ./stats.db:/stats.db \ # 如果不想保存统计数据，不需要挂载 stats.db 文件 +-v ./uniapi_db:/home/data \ # 如果不想保存统计数据，不需要挂载该文件夹 yym68686/uni-api:latest ``` @@ -152,7 +152,7 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml # 如果已经设置 CONFIG_URL，不需要挂载配置文件 - - ./stats.db:/stats.db # 如果不想保存统计数据，不需要挂载 stats.db 文件 + - ./uniapi_db:/home/data # 如果不想保存统计数据，不需要挂载该文件夹 ``` CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。如果使用本地挂载的配置文件，不需要设置 CONFIG_URL。CONFIG_URL 是在不方便挂载配置文件的情况下使用。 @@ -181,7 +181,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ --v ./stats.db:/stats.db \ +-v ./uniapi_db:/home/data \ yym68686/uni-api:latest docker logs -f uni-api ``` diff --git a/docker-compose.yml b/docker-compose.yml index d95e84cb..a49a80d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,4 +8,4 @@ services: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml - - ./stats.db:/stats.db \ No newline at end of file + - ./uniapi_db:/home/data \ No newline at end of file diff --git a/main.py b/main.py index 004693b4..2b789a60 100644 --- a/main.py +++ b/main.py @@ -105,8 +105,15 @@ class ChannelStat(Base): first_response_time = Column(Float) # 新增: 记录首次响应时间 timestamp = Column(DateTime(timezone=True), server_default=func.now()) +# 获取数据库路径 +db_path = os.getenv('DB_PATH', './data/stats.db') + +# 确保 data 目录存在 +data_dir = os.path.dirname(db_path) +os.makedirs(data_dir, exist_ok=True) + # 创建异步引擎和会话 -engine = create_async_engine('sqlite+aiosqlite:///stats.db', echo=is_debug) +engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) class StatsMiddleware(BaseHTTPMiddleware): From 17409c4b90c9eeed923b3c55aa2af7d496a7016f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 05:32:16 +0800 Subject: [PATCH 059/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20v1/audio/transcriptions=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 50 ++++++++++++++++++++++++++++++++++++++++++++------ models.py | 15 ++++++++++++++- request.py | 28 ++++++++++++++++++++++++++++ response.py | 11 +++++++++-- 4 files changed, 95 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 2b789a60..ecdedb12 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError -from models import RequestModel, ImageGenerationRequest +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest from request import get_payload from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder @@ -191,7 +191,7 @@ async def update_channel_stats(self, provider, model, api_key, success, first_re app.add_middleware(StatsMiddleware) # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None, token=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], provider: Dict, endpoint=None, token=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -233,6 +233,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], engine = "dalle" request.stream = False + if endpoint == "/v1/audio/transcriptions": + engine = "whisper" + request.stream = False + if provider.get("engine"): engine = provider["engine"] @@ -241,7 +245,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], url, headers, payload = await get_payload(request, engine, provider) if is_debug: logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) - logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) + if payload.get("file"): + pass + else: + logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) try: if request.stream: model = provider['model'][request.model] @@ -356,7 +363,7 @@ def get_matching_providers(self, model_name, token): print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str, endpoint=None): config = app.state.config # api_keys_db = app.state.api_keys_db api_list = app.state.api_list @@ -399,7 +406,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token) # 在 try_all_providers 函数中处理失败的情况 - async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): + async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): status_code = 500 error_message = None num_providers = len(providers) @@ -421,6 +428,9 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") except (Exception, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: logger.error(f"Error with provider {provider['provider']}: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() error_message = str(e) if auto_retry: continue @@ -523,7 +533,7 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec return token @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) -async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)): +async def request_model(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str = Depends(verify_api_key)): # logger.info(f"Request received: {request}") return await model_handler.request_model(request, token) @@ -546,6 +556,34 @@ async def images_generations( ): return await model_handler.request_model(request, token, endpoint="/v1/images/generations") +from fastapi import UploadFile, File, Form, HTTPException +import io +@app.post("/v1/audio/transcriptions", dependencies=[Depends(rate_limit_dependency)]) +async def audio_transcriptions( + file: UploadFile = File(...), + model: str = Form(...), + token: str = Depends(verify_api_key) +): + try: + # 读取上传的文件内容 + content = await file.read() + file_obj = io.BytesIO(content) + + # 创建AudioTranscriptionRequest对象 + request = AudioTranscriptionRequest( + file=(file.filename, file_obj, file.content_type), + model=model + ) + + return await model_handler.request_model(request, token, endpoint="/v1/audio/transcriptions") + except UnicodeDecodeError: + raise HTTPException(status_code=400, detail="Invalid audio file encoding") + except Exception as e: + if is_debug: + import traceback + traceback.print_exc() + raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}") + @app.get("/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) def generate_api_key(): api_key = "sk-" + secrets.token_urlsafe(36) diff --git a/models.py b/models.py index 44887d90..c3063e06 100644 --- a/models.py +++ b/models.py @@ -1,5 +1,6 @@ +from io import IOBase from pydantic import BaseModel, Field -from typing import List, Dict, Optional, Union +from typing import List, Dict, Optional, Union, Tuple class ImageGenerationRequest(BaseModel): model: str @@ -8,6 +9,18 @@ class ImageGenerationRequest(BaseModel): size: str stream: bool = False +class AudioTranscriptionRequest(BaseModel): + file: Tuple[str, IOBase, str] + model: str + language: Optional[str] = None + prompt: Optional[str] = None + response_format: Optional[str] = None + temperature: Optional[float] = None + stream: bool = False + + class Config: + arbitrary_types_allowed = True + class FunctionParameter(BaseModel): type: str properties: Dict[str, Dict[str, Union[str, Dict[str, str]]]] diff --git a/request.py b/request.py index 75d0e3a5..1446e050 100644 --- a/request.py +++ b/request.py @@ -1040,6 +1040,32 @@ async def get_dalle_payload(request, engine, provider): return url, headers, payload +async def get_whisper_payload(request, engine, provider): + model = provider['model'][request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + url = provider['base_url'] + url = BaseAPI(url).audio_transcriptions + + payload = { + "model": model, + "file": request.file, + } + + if request.prompt: + payload["prompt"] = request.prompt + if request.response_format: + payload["response_format"] = request.response_format + if request.temperature: + payload["temperature"] = request.temperature + if request.language: + payload["language"] = request.language + + return url, headers, payload + async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) @@ -1061,5 +1087,7 @@ async def get_payload(request: RequestModel, engine, provider): return await get_cohere_payload(request, engine, provider) elif engine == "dalle": return await get_dalle_payload(request, engine, provider) + elif engine == "whisper": + return await get_whisper_payload(request, engine, provider) else: raise ValueError("Unknown payload") \ No newline at end of file diff --git a/response.py b/response.py index b67eb6ae..4acefe87 100644 --- a/response.py +++ b/response.py @@ -1,6 +1,7 @@ import json import httpx from datetime import datetime +from io import BytesIO from log_config import logger @@ -41,7 +42,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f return sse_response async def check_response(response, error_log): - if response.status_code != 200: + if response and response.status_code != 200: error_message = await response.aread() error_str = error_message.decode('utf-8', errors='replace') try: @@ -269,7 +270,13 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): yield "data: [DONE]\n\r\n" async def fetch_response(client, url, headers, payload): - response = await client.post(url, headers=headers, json=payload) + response = None + if payload.get("file"): + file = payload.pop("file") + headers.pop("Content-Type") + response = await client.post(url, headers=headers, data=payload, files={"file": file}) + else: + response = await client.post(url, headers=headers, json=payload) error_message = await check_response(response, "fetch_response") if error_message: yield error_message From 888a669000350e99b1cd71ce208af9af490fdd6f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 05:48:09 +0800 Subject: [PATCH 060/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20v1/moderations=20endpoint.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 21 ++++++++++++++++----- models.py | 5 +++++ request.py | 20 +++++++++++++++++++- response.py | 1 - utils.py | 1 + 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index ecdedb12..996aba27 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError -from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest from request import get_payload from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder @@ -191,7 +191,7 @@ async def update_channel_stats(self, provider, model, api_key, success, first_re app.add_middleware(StatsMiddleware) # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], provider: Dict, endpoint=None, token=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], provider: Dict, endpoint=None, token=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -237,6 +237,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "whisper" request.stream = False + if endpoint == "/v1/moderations": + engine = "moderation" + request.stream = False + if provider.get("engine"): engine = provider["engine"] @@ -363,7 +367,7 @@ def get_matching_providers(self, model_name, token): print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str, endpoint=None): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): config = app.state.config # api_keys_db = app.state.api_keys_db api_list = app.state.api_list @@ -406,7 +410,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token) # 在 try_all_providers 函数中处理失败的情况 - async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): + async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): status_code = 500 error_message = None num_providers = len(providers) @@ -533,7 +537,7 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec return token @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) -async def request_model(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str = Depends(verify_api_key)): +async def request_model(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str = Depends(verify_api_key)): # logger.info(f"Request received: {request}") return await model_handler.request_model(request, token) @@ -556,6 +560,13 @@ async def images_generations( ): return await model_handler.request_model(request, token, endpoint="/v1/images/generations") +@app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) +async def images_generations( + request: ModerationRequest, + token: str = Depends(verify_api_key) +): + return await model_handler.request_model(request, token, endpoint="/v1/moderations") + from fastapi import UploadFile, File, Form, HTTPException import io @app.post("/v1/audio/transcriptions", dependencies=[Depends(rate_limit_dependency)]) diff --git a/models.py b/models.py index c3063e06..3e68404b 100644 --- a/models.py +++ b/models.py @@ -21,6 +21,11 @@ class AudioTranscriptionRequest(BaseModel): class Config: arbitrary_types_allowed = True +class ModerationRequest(BaseModel): + input: str + model: Optional[str] = "text-moderation-latest" + stream: bool = False + class FunctionParameter(BaseModel): type: str properties: Dict[str, Dict[str, Union[str, Dict[str, str]]]] diff --git a/request.py b/request.py index 1446e050..e4cd087d 100644 --- a/request.py +++ b/request.py @@ -1043,7 +1043,7 @@ async def get_dalle_payload(request, engine, provider): async def get_whisper_payload(request, engine, provider): model = provider['model'][request.model] headers = { - "Content-Type": "application/json", + "Content-Type": "multipart/form-data", } if provider.get("api"): headers['Authorization'] = f"Bearer {provider['api'].next()}" @@ -1066,6 +1066,22 @@ async def get_whisper_payload(request, engine, provider): return url, headers, payload +async def get_moderation_payload(request, engine, provider): + model = provider['model'][request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + url = provider['base_url'] + url = BaseAPI(url).moderations + + payload = { + "input": request.input, + } + + return url, headers, payload + async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) @@ -1089,5 +1105,7 @@ async def get_payload(request: RequestModel, engine, provider): return await get_dalle_payload(request, engine, provider) elif engine == "whisper": return await get_whisper_payload(request, engine, provider) + elif engine == "moderation": + return await get_moderation_payload(request, engine, provider) else: raise ValueError("Unknown payload") \ No newline at end of file diff --git a/response.py b/response.py index 4acefe87..bb6b47a4 100644 --- a/response.py +++ b/response.py @@ -273,7 +273,6 @@ async def fetch_response(client, url, headers, payload): response = None if payload.get("file"): file = payload.pop("file") - headers.pop("Content-Type") response = await client.post(url, headers=headers, data=payload, files={"file": file}) else: response = await client.post(url, headers=headers, json=payload) diff --git a/utils.py b/utils.py index ff472b75..f0648405 100644 --- a/utils.py +++ b/utils.py @@ -308,6 +308,7 @@ def __init__( self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/chat/completions",) + ("",) * 3) self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) + self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/moderations",) + ("",) * 3) def safe_get(data, *keys): for key in keys: From 279749747e269fa2a574d218db3198f4fd1d3bb2 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 14:25:52 +0800 Subject: [PATCH 061/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20causing=20startup=20errors=20due=20to=20missing=20python-mul?= =?UTF-8?q?tipart=20dependency.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5040ab13..3f958dea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ aiosqlite sqlalchemy watchfiles httpx[http2] -cryptography \ No newline at end of file +cryptography +python-multipart \ No newline at end of file From f156f8ab4239853ec10d6a6ee966d3418ed7c389 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 18:33:19 +0800 Subject: [PATCH 062/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20that=20causes=20an=20error=20when=20Claude=20uploads=20a?= =?UTF-8?q?=20PNG=20image.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where fields are not automatically added when the database does not have specific fields. ✨ Feature: Add user message ethics review support. 📖 Docs: Update documentation --- README_CN.md | 2 + main.py | 113 +++++++++++++++++++++++++++++++++++++++++++++------ models.py | 13 +++++- request.py | 43 ++++++++++++++++---- utils.py | 4 +- 5 files changed, 152 insertions(+), 23 deletions(-) diff --git a/README_CN.md b/README_CN.md index 4f08f959..a9c6e11f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -105,10 +105,12 @@ api_keys: model: - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。这种写法不会匹配到other-provider提供的名为anthropic/claude-3-5-sonnet的模型。 - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 + - openai-test/text-moderation-latest # 当开启消息道德审查后，可以使用名为 openai-test 渠道下的 text-moderation-latest 模型进行道德审查。 preferences: USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 + ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 # 渠道级加权负载均衡配置示例 - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo diff --git a/main.py b/main.py index 996aba27..6a078269 100644 --- a/main.py +++ b/main.py @@ -24,10 +24,50 @@ import os is_debug = bool(os.getenv("DEBUG", False)) +from sqlalchemy import inspect, text +from sqlalchemy.sql import sqltypes + async def create_tables(): async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) + # 检查并添加缺失的列 + def check_and_add_columns(connection): + inspector = inspect(connection) + for table in [RequestStat, ChannelStat]: + table_name = table.__tablename__ + existing_columns = {col['name']: col['type'] for col in inspector.get_columns(table_name)} + + for column_name, column in table.__table__.columns.items(): + if column_name not in existing_columns: + col_type = _map_sa_type_to_sql_type(column.type) + default = _get_default_sql(column.default) + connection.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {col_type}{default}")) + + await conn.run_sync(check_and_add_columns) + +def _map_sa_type_to_sql_type(sa_type): + type_map = { + sqltypes.Integer: "INTEGER", + sqltypes.String: "TEXT", + sqltypes.Float: "REAL", + sqltypes.Boolean: "BOOLEAN", + sqltypes.DateTime: "DATETIME", + sqltypes.Text: "TEXT" + } + return type_map.get(type(sa_type), "TEXT") + +def _get_default_sql(default): + if default is None: + return "" + if isinstance(default.arg, bool): + return f" DEFAULT {str(default.arg).upper()}" + if isinstance(default.arg, (int, float)): + return f" DEFAULT {default.arg}" + if isinstance(default.arg, str): + return f" DEFAULT '{default.arg}'" + return "" + @asynccontextmanager async def lifespan(app: FastAPI): # 启动时的代码 @@ -79,7 +119,7 @@ async def parse_request_body(request: Request): from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession from sqlalchemy.orm import declarative_base, sessionmaker -from sqlalchemy import Column, Integer, String, Float, DateTime, select, Boolean +from sqlalchemy import Column, Integer, String, Float, DateTime, select, Boolean, Text from sqlalchemy.sql import func # 定义数据库模型 @@ -93,6 +133,8 @@ class RequestStat(Base): token = Column(String) total_time = Column(Float) model = Column(String) + is_flagged = Column(Boolean, default=False) + moderated_content = Column(Text) timestamp = Column(DateTime(timezone=True), server_default=func.now()) class ChannelStat(Base): @@ -113,6 +155,7 @@ class ChannelStat(Base): os.makedirs(data_dir, exist_ok=True) # 创建异步引擎和会话 +# engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=False) engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) @@ -132,37 +175,76 @@ async def dispatch(self, request: Request, call_next): start_time = time() request.state.parsed_body = await parse_request_body(request) + endpoint = f"{request.method} {request.url.path}" + client_ip = request.client.host model = "unknown" + enable_moderation = False # 默认不开启道德审查 + is_flagged = False + moderated_content = "" + + config = app.state.config + api_list = app.state.api_list + + # 根据token决定是否启用道德审查 + if token: + try: + api_index = api_list.index(token) + enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) + except ValueError: + # token不在api_list中，使用默认值（不开启） + pass + else: + # 如果token为None，检查全局设置 + enable_moderation = config.get('ENABLE_MODERATION', False) + if request.state.parsed_body: try: request_model = RequestModel(**request.state.parsed_body) model = request_model.model + moderated_content = request_model.get_last_text_message() + + if enable_moderation and moderated_content: + moderation_response = await self.moderate_content(moderated_content, token) + moderation_result = moderation_response.body + moderation_data = json.loads(moderation_result) + is_flagged = moderation_data.get('results', [{}])[0].get('flagged', False) + + if is_flagged: + logger.error(f"Content did not pass the moral check: %s", moderated_content) + process_time = time() - start_time + await self.update_stats(endpoint, process_time, client_ip, model, token, is_flagged, moderated_content) + return JSONResponse( + status_code=400, + content={"error": "Content did not pass the moral check, please modify and try again."} + ) except RequestValidationError: pass except Exception as e: - logger.error(f"Error processing request: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + + logger.error(f"处理请求或进行道德检查时出错: {str(e)}") response = await call_next(request) process_time = time() - start_time - endpoint = f"{request.method} {request.url.path}" - client_ip = request.client.host - # 异步更新数据库 - await self.update_stats(endpoint, process_time, client_ip, model, token) + await self.update_stats(endpoint, process_time, client_ip, model, token, is_flagged, moderated_content) return response - async def update_stats(self, endpoint, process_time, client_ip, model, token): + async def update_stats(self, endpoint, process_time, client_ip, model, token, is_flagged, moderated_content): async with self.db as session: - # 为每个请求创建一条新的记录 new_request_stat = RequestStat( endpoint=endpoint, ip=client_ip, token=token, total_time=process_time, - model=model + model=model, + is_flagged=is_flagged, + moderated_content=moderated_content ) session.add(new_request_stat) await session.commit() @@ -179,6 +261,14 @@ async def update_channel_stats(self, provider, model, api_key, success, first_re session.add(channel_stat) await session.commit() + async def moderate_content(self, content, token): + moderation_request = ModerationRequest(input=content) + + # 直接调用 moderations 函数 + response = await moderations(moderation_request, token) + + return response + # 配置 CORS 中间件 app.add_middleware( CORSMiddleware, @@ -561,7 +651,7 @@ async def images_generations( return await model_handler.request_model(request, token, endpoint="/v1/images/generations") @app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) -async def images_generations( +async def moderations( request: ModerationRequest, token: str = Depends(verify_api_key) ): @@ -601,9 +691,6 @@ def generate_api_key(): return JSONResponse(content={"api_key": api_key}) # 在 /stats 路由中返回成功和失败百分比 -from collections import defaultdict -from sqlalchemy import func - from collections import defaultdict from sqlalchemy import func, desc, case diff --git a/models.py b/models.py index 3e68404b..d55d6af1 100644 --- a/models.py +++ b/models.py @@ -96,4 +96,15 @@ class RequestModel(BaseModel): n: Optional[int] = 1 user: Optional[str] = None tool_choice: Optional[Union[str, ToolChoice]] = None - tools: Optional[List[Tool]] = None \ No newline at end of file + tools: Optional[List[Tool]] = None + + def get_last_text_message(self) -> Optional[str]: + for message in reversed(self.messages): + if message.content: + if isinstance(message.content, str): + return message.content + elif isinstance(message.content, list): + for item in reversed(message.content): + if item.type == "text" and item.text: + return item.text + return "" \ No newline at end of file diff --git a/request.py b/request.py index e4cd087d..6b8fb837 100644 --- a/request.py +++ b/request.py @@ -8,9 +8,20 @@ from models import RequestModel from utils import c35s, c3s, c3o, c3h, gem, BaseAPI +import imghdr + def encode_image(image_path): - with open(image_path, "rb") as image_file: - return base64.b64encode(image_file.read()).decode('utf-8') + with open(image_path, "rb") as image_file: + file_content = image_file.read() + file_type = imghdr.what(None, file_content) + base64_encoded = base64.b64encode(file_content).decode('utf-8') + + if file_type == 'png': + return f"data:image/png;base64,{base64_encoded}" + elif file_type in ['jpeg', 'jpg']: + return f"data:image/jpeg;base64,{base64_encoded}" + else: + raise ValueError(f"不支持的图片格式: {file_type}") async def get_doc_from_url(url): filename = urllib.parse.unquote(url.split("/")[-1]) @@ -37,12 +48,28 @@ async def get_encode_image(image_url): filename = await get_doc_from_url(image_url) image_path = os.getcwd() + "/" + filename base64_image = encode_image(image_path) - if filename.endswith(".png"): - prompt = f"data:image/png;base64,{base64_image}" - else: - prompt = f"data:image/jpeg;base64,{base64_image}" os.remove(image_path) - return prompt + return base64_image + +from PIL import Image +import io +def validate_image(image_data, image_type): + try: + decoded_image = base64.b64decode(image_data) + image = Image.open(io.BytesIO(decoded_image)) + + # 检查图片格式是否与声明的类型匹配 + # print("image.format", image.format) + if image_type == "image/png" and image.format != "PNG": + raise ValueError("Image is not a valid PNG") + elif image_type == "image/jpeg" and image.format not in ["JPEG", "JPG"]: + raise ValueError("Image is not a valid JPEG") + + # 如果没有异常,则图片有效 + return True + except Exception as e: + print(f"Image validation failed: {str(e)}") + return False async def get_image_message(base64_image, engine = None): if base64_image.startswith("http"): @@ -59,6 +86,8 @@ async def get_image_message(base64_image, engine = None): } } if "claude" == engine or "vertex-claude" == engine: + # if not validate_image(base64_image.split(",")[1], image_type): + # raise ValueError(f"Invalid image format. Expected {image_type}") return { "type": "image", "source": { diff --git a/utils.py b/utils.py index f0648405..804e9fd2 100644 --- a/utils.py +++ b/utils.py @@ -310,10 +310,10 @@ def __init__( self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/moderations",) + ("",) * 3) -def safe_get(data, *keys): +def safe_get(data, *keys, default=None): for key in keys: try: data = data[key] if isinstance(data, (dict, list)) else data.get(key) except (KeyError, IndexError, AttributeError, TypeError): - return None + return default return data \ No newline at end of file From 805409fcac33222538ebd6000a11592a5a861f59 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 19:10:53 +0800 Subject: [PATCH 063/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20Pillow=20dependency=20is=20not=20installed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/request.py b/request.py index 6b8fb837..35385e13 100644 --- a/request.py +++ b/request.py @@ -51,25 +51,25 @@ async def get_encode_image(image_url): os.remove(image_path) return base64_image -from PIL import Image -import io -def validate_image(image_data, image_type): - try: - decoded_image = base64.b64decode(image_data) - image = Image.open(io.BytesIO(decoded_image)) - - # 检查图片格式是否与声明的类型匹配 - # print("image.format", image.format) - if image_type == "image/png" and image.format != "PNG": - raise ValueError("Image is not a valid PNG") - elif image_type == "image/jpeg" and image.format not in ["JPEG", "JPG"]: - raise ValueError("Image is not a valid JPEG") - - # 如果没有异常,则图片有效 - return True - except Exception as e: - print(f"Image validation failed: {str(e)}") - return False +# from PIL import Image +# import io +# def validate_image(image_data, image_type): +# try: +# decoded_image = base64.b64decode(image_data) +# image = Image.open(io.BytesIO(decoded_image)) + +# # 检查图片格式是否与声明的类型匹配 +# # print("image.format", image.format) +# if image_type == "image/png" and image.format != "PNG": +# raise ValueError("Image is not a valid PNG") +# elif image_type == "image/jpeg" and image.format not in ["JPEG", "JPG"]: +# raise ValueError("Image is not a valid JPEG") + +# # 如果没有异常,则图片有效 +# return True +# except Exception as e: +# print(f"Image validation failed: {str(e)}") +# return False async def get_image_message(base64_image, engine = None): if base64_image.startswith("http"): From 1934075d809ffc2081514585305b7077a8caeba2 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 19:33:34 +0800 Subject: [PATCH 064/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 42 ++++++++++++++++++++++-------------------- README_CN.md | 6 +++--- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index bd02ca01..4ab2dda8 100644 --- a/README.md +++ b/README.md @@ -37,36 +37,36 @@ Using the api.yaml configuration file, multiple models can be configured, and ea ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name is fine, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # At least one model is required + model: # At least one model must be filled in - gpt-4o # Usable model name, required - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional - tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional + tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the following line to use the original name. - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud Project ID. Format: String, usually consists of lowercase letters, numbers, and hyphens. How to get it: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to get it: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to get it: Generated when creating the service account, can also be found in the "IAM & admin" section of the Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, you can also view the service account details in the "IAM & Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -75,14 +75,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # You can put the service provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes, otherwise a yaml syntax error, llama-3.1-8b is the renamed name, you can use a concise name instead of the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes, otherwise a yaml syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, the model name must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a concise name instead of the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # The model name must be enclosed in quotes, otherwise yaml syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -91,34 +91,36 @@ providers: - causallm-35b-beta2ep-q6k: causallm-35b - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # Force the use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional + engine: openrouter # Force to use a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required model: # Models that this API Key can use, required - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models role: admin - - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy + - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. This method will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will take the entire anthropic/claude-3-5-sonnet as the model name. This method can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Other providers' claude-3-5-sonnet models cannot be used. This way of writing will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will treat the entire anthropic/claude-3-5-sonnet as the model name. This way of writing can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - openai-test/text-moderation-latest # When message moderation is enabled, you can use the text-moderation-latest model under the channel named openai-test for moderation. preferences: - USE_ROUND_ROBIN: true # Whether to use round-robin load balancing, true to use, false to not use, default is true. When enabled, each request to the model will be made in sequence according to the model configuration. It has nothing to do with the original channel order in providers. Therefore, you can set a different request order for each API key. + USE_ROUND_ROBIN: true # Whether to use polling load balancing, true to use, false to not use, default is true. When polling is enabled, each request will be made in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request orders for each API key. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true - RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + RATE_LIMIT: 2/min # Supports rate limiting, the maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + ENABLE_MODERATION: true # Whether to enable message moderation, true to enable, false to not enable, default is false. When enabled, it will conduct moderation on the user's message, if inappropriate messages are found, it will return an error message. # Channel-level weighted load balancing configuration example - - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - gcp1/*: 5 # The number after the colon is the weight, the weight only supports positive integers. - gcp2/*: 3 # The larger the number, the greater the probability of the request. - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, it will request according to the original channel order, if there is weight, it will request according to the weighted order. + USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, it will request in the original channel order, if there is weight, it will request in the weighted order. AUTO_RETRY: true ``` diff --git a/README_CN.md b/README_CN.md index a9c6e11f..4f34e013 100644 --- a/README_CN.md +++ b/README_CN.md @@ -94,14 +94,14 @@ providers: engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 api_keys: - - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 model: # 该 API Key 可以使用的模型，必填 - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 role: admin - - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy + - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。这种写法不会匹配到other-provider提供的名为anthropic/claude-3-5-sonnet的模型。 - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 @@ -113,7 +113,7 @@ api_keys: ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 # 渠道级加权负载均衡配置示例 - - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - gcp1/*: 5 # 冒号后面就是权重，权重仅支持正整数。 - gcp2/*: 3 # 数字的大小代表权重，数字越大，请求的概率越大。 From 7c8f29f3993caf34b64961cb859902736f1deea8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 26 Sep 2024 19:44:48 +0800 Subject: [PATCH 065/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ README_CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 4ab2dda8..7d2d5e3a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ If used personally, one/new-api is too complex and has many commercial features - Supports automatic retry, when an API channel response fails, automatically retry the next API channel. - Supports fine-grained access control. Supports using wildcards to set specific models for API key available channels. - Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. +- Supports multiple standard OpenAI format interfaces: `/v1/chat/completions`, `/v1/images/generations`, `/v1/audio/transcriptions`, `/v1/moderations`, `/v1/models`. +- Supports OpenAI moderation for ethical review, allowing for ethical review of user messages. If inappropriate messages are detected, an error message will be returned. This reduces the risk of the backend API being banned by providers. ## Configuration diff --git a/README_CN.md b/README_CN.md index 4f34e013..acb14586 100644 --- a/README_CN.md +++ b/README_CN.md @@ -30,6 +30,8 @@ - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 - 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 +- 支持多个标准 OpenAI 格式的接口：`/v1/chat/completions`，`/v1/images/generations`，`/v1/audio/transcriptions`，`/v1/moderations`，`/v1/models`。 +- 支持 OpenAI moderation 道德审查，可以对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。降低后台 API 被提供商封禁的风险。 ## Configuration From b40e306968b5be225c3b3971087872e5ebf9f3be Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 00:40:11 +0800 Subject: [PATCH 066/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20the=20voice=20function=20from=20being=20us?= =?UTF-8?q?ed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index 35385e13..548d669f 100644 --- a/request.py +++ b/request.py @@ -1072,7 +1072,7 @@ async def get_dalle_payload(request, engine, provider): async def get_whisper_payload(request, engine, provider): model = provider['model'][request.model] headers = { - "Content-Type": "multipart/form-data", + # "Content-Type": "multipart/form-data", } if provider.get("api"): headers['Authorization'] = f"Bearer {provider['api'].next()}" From c2ae7714ef80c1abc9c4e00285acb9196c374c65 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 01:05:39 +0800 Subject: [PATCH 067/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20Docker=20version=20number?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 22 +++++++++++++++++++++- VERSION | 1 + 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 VERSION diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30aac54d..2c481860 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,7 @@ on: branches: - main paths: + - VERSION - main.py - utils.py - models.py @@ -36,6 +37,23 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Get current version + id: get_version + run: | + VERSION=$(cat VERSION || echo "0.0.0") + echo "Current version: $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Bump version + id: bump_version + run: | + IFS='.' read -ra VERSION_PARTS <<< "${{ steps.get_version.outputs.version }}" + PATCH=$((VERSION_PARTS[2] + 1)) + NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH" + echo $NEW_VERSION > VERSION + echo "New version: $NEW_VERSION" + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + - name: Build and push Docker image uses: docker/build-push-action@v2.7.0 with: @@ -43,4 +61,6 @@ jobs: file: Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: yym68686/uni-api:latest \ No newline at end of file + tags: | + yym68686/uni-api:latest + yym68686/uni-api:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..bd52db81 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.0 \ No newline at end of file From 13a1a61e0cf518c83de2a6e1e1976106fe139f37 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 01:24:27 +0800 Subject: [PATCH 068/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20main.ym?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2c481860..adc9bf2e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,6 +54,14 @@ jobs: echo "New version: $NEW_VERSION" echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + - name: Commit version bump + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git add VERSION + git commit -m "Bump version to ${{ steps.bump_version.outputs.new_version }}" + git push + - name: Build and push Docker image uses: docker/build-push-action@v2.7.0 with: From 32f4b84e73c876b65b68877399acf561f0422597 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 01:41:22 +0800 Subject: [PATCH 069/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20main.ym?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index adc9bf2e..a308f087 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,12 +55,14 @@ jobs: echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT - name: Commit version bump + env: + GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add VERSION git commit -m "Bump version to ${{ steps.bump_version.outputs.new_version }}" - git push + git push https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }} HEAD:main - name: Build and push Docker image uses: docker/build-push-action@v2.7.0 From 98718354deb18fa3d5e698266546158db08ecf4c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 02:17:56 +0800 Subject: [PATCH 070/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20a=20bug=20t?= =?UTF-8?q?hat=20may=20occur=20in=20a=20concurrent=20environment=20when=20?= =?UTF-8?q?multiple=20requests=20attempt=20to=20use=20the=20same=20databas?= =?UTF-8?q?e=20session=20simultaneously.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 2 +- main.py | 55 ++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a308f087..f495d5cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,7 +62,7 @@ jobs: git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add VERSION git commit -m "Bump version to ${{ steps.bump_version.outputs.new_version }}" - git push https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }} HEAD:main + git push - name: Build and push Docker image uses: docker/build-push-action@v2.7.0 diff --git a/main.py b/main.py index 6a078269..b6ee84e3 100644 --- a/main.py +++ b/main.py @@ -162,7 +162,6 @@ class ChannelStat(Base): class StatsMiddleware(BaseHTTPMiddleware): def __init__(self, app): super().__init__(app) - self.db = async_session() async def dispatch(self, request: Request, call_next): if request.headers.get("x-api-key"): @@ -236,30 +235,40 @@ async def dispatch(self, request: Request, call_next): return response async def update_stats(self, endpoint, process_time, client_ip, model, token, is_flagged, moderated_content): - async with self.db as session: - new_request_stat = RequestStat( - endpoint=endpoint, - ip=client_ip, - token=token, - total_time=process_time, - model=model, - is_flagged=is_flagged, - moderated_content=moderated_content - ) - session.add(new_request_stat) - await session.commit() + async with async_session() as session: + async with session.begin(): + try: + new_request_stat = RequestStat( + endpoint=endpoint, + ip=client_ip, + token=token, + total_time=process_time, + model=model, + is_flagged=is_flagged, + moderated_content=moderated_content + ) + session.add(new_request_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating stats: {str(e)}") async def update_channel_stats(self, provider, model, api_key, success, first_response_time): - async with self.db as session: - channel_stat = ChannelStat( - provider=provider, - model=model, - api_key=api_key, - success=success, - first_response_time=first_response_time - ) - session.add(channel_stat) - await session.commit() + async with async_session() as session: + async with session.begin(): + try: + channel_stat = ChannelStat( + provider=provider, + model=model, + api_key=api_key, + success=success, + first_response_time=first_response_time + ) + session.add(channel_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating channel stats: {str(e)}") async def moderate_content(self, content, token): moderation_request = ModerationRequest(input=content) From d9c09cec1ca247017c60a32a9756b8b894edc4e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 18:20:48 +0000 Subject: [PATCH 071/476] Bump version to 0.0.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bd52db81..8acdd82b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.0 \ No newline at end of file +0.0.1 From 31fa10f8e6b0eb558f01cdc1dc2a6b3a70e90a6b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 02:49:10 +0800 Subject: [PATCH 072/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20update=20ghcr=20?= =?UTF-8?q?image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f495d5cd..834f9a76 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,6 +37,13 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.PACK_TOKEN }} + - name: Get current version id: get_version run: | @@ -73,4 +80,6 @@ jobs: push: true tags: | yym68686/uni-api:latest - yym68686/uni-api:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file + yym68686/uni-api:${{ steps.bump_version.outputs.new_version }} + ghcr.io/${{ github.repository }}/uni-api:latest + ghcr.io/${{ github.repository }}/uni-api:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file From 6b4b833e9e539add0abadd529d23e3dfda827bab Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 19:04:20 +0000 Subject: [PATCH 073/476] Bump version to 0.0.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8acdd82b..4e379d2b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.1 +0.0.2 From c6cc6a0ed5f7367247f05fce3909bb8477816d35 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 03:24:37 +0800 Subject: [PATCH 074/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Simplify=20GitHu?= =?UTF-8?q?b=20packages=20mirror=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 834f9a76..ebd8b91f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,5 +81,5 @@ jobs: tags: | yym68686/uni-api:latest yym68686/uni-api:${{ steps.bump_version.outputs.new_version }} - ghcr.io/${{ github.repository }}/uni-api:latest - ghcr.io/${{ github.repository }}/uni-api:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file + ghcr.io/${{ github.repository }}:latest + ghcr.io/${{ github.repository }}:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file From 0712d85ad8546ad548fdc9fb838080498607dd52 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 19:24:55 +0000 Subject: [PATCH 075/476] Bump version to 0.0.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4e379d2b..bcab45af 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.2 +0.0.3 From c34a2a5d7542f4b2aa676922e5323a583ea44304 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 27 Sep 2024 04:52:44 +0800 Subject: [PATCH 076/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20request=20to=20generate=20images=20cannot=20be?= =?UTF-8?q?=20parsed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 2 +- main.py | 34 ++++++++-------- models.py | 81 +++++++++++++++++++++++++------------- 3 files changed, 73 insertions(+), 44 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ebd8b91f..7fa3c869 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add VERSION - git commit -m "Bump version to ${{ steps.bump_version.outputs.new_version }}" + git commit -m "📖 Bump version to ${{ steps.bump_version.outputs.new_version }}" git push - name: Build and push Docker image diff --git a/main.py b/main.py index b6ee84e3..7b676f1b 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError -from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest from request import get_payload from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder @@ -164,30 +164,27 @@ def __init__(self, app): super().__init__(app) async def dispatch(self, request: Request, call_next): - if request.headers.get("x-api-key"): - token = request.headers.get("x-api-key") - elif request.headers.get("Authorization"): - token = request.headers.get("Authorization").split(" ")[1] - else: - token = None - start_time = time() - request.state.parsed_body = await parse_request_body(request) endpoint = f"{request.method} {request.url.path}" client_ip = request.client.host model = "unknown" - enable_moderation = False # 默认不开启道德审查 is_flagged = False moderated_content = "" + enable_moderation = False # 默认不开启道德审查 config = app.state.config - api_list = app.state.api_list - # 根据token决定是否启用道德审查 + if request.headers.get("x-api-key"): + token = request.headers.get("x-api-key") + elif request.headers.get("Authorization"): + token = request.headers.get("Authorization").split(" ")[1] + else: + token = None if token: try: + api_list = app.state.api_list api_index = api_list.index(token) enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) except ValueError: @@ -197,11 +194,16 @@ async def dispatch(self, request: Request, call_next): # 如果token为None，检查全局设置 enable_moderation = config.get('ENABLE_MODERATION', False) - if request.state.parsed_body: + parsed_body = await parse_request_body(request) + if parsed_body: try: - request_model = RequestModel(**request.state.parsed_body) + request_model = UnifiedRequest.model_validate(parsed_body).data model = request_model.model - moderated_content = request_model.get_last_text_message() + + if request_model.request_type == "chat": + moderated_content = request_model.get_last_text_message() + elif request_model.request_type == "image": + moderated_content = request_model.prompt if enable_moderation and moderated_content: moderation_response = await self.moderate_content(moderated_content, token) @@ -636,7 +638,7 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec return token @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) -async def request_model(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str = Depends(verify_api_key)): +async def request_model(request: RequestModel, token: str = Depends(verify_api_key)): # logger.info(f"Request received: {request}") return await model_handler.request_model(request, token) diff --git a/models.py b/models.py index d55d6af1..17643911 100644 --- a/models.py +++ b/models.py @@ -1,30 +1,6 @@ from io import IOBase -from pydantic import BaseModel, Field -from typing import List, Dict, Optional, Union, Tuple - -class ImageGenerationRequest(BaseModel): - model: str - prompt: str - n: int - size: str - stream: bool = False - -class AudioTranscriptionRequest(BaseModel): - file: Tuple[str, IOBase, str] - model: str - language: Optional[str] = None - prompt: Optional[str] = None - response_format: Optional[str] = None - temperature: Optional[float] = None - stream: bool = False - - class Config: - arbitrary_types_allowed = True - -class ModerationRequest(BaseModel): - input: str - model: Optional[str] = "text-moderation-latest" - stream: bool = False +from pydantic import BaseModel, Field, model_validator +from typing import List, Dict, Optional, Union, Tuple, Literal class FunctionParameter(BaseModel): type: str @@ -82,6 +58,7 @@ class ToolChoice(BaseModel): function: Optional[FunctionChoice] = None class RequestModel(BaseModel): + request_type: Literal["chat"] = "chat" model: str messages: List[Message] logprobs: Optional[bool] = None @@ -107,4 +84,54 @@ def get_last_text_message(self) -> Optional[str]: for item in reversed(message.content): if item.type == "text" and item.text: return item.text - return "" \ No newline at end of file + return "" + +class ImageGenerationRequest(BaseModel): + request_type: Literal["image"] = "image" + prompt: str + model: Optional[str] = "dall-e-3" + n: Optional[int] = 1 + size: Optional[str] = "1024x1024" + stream: bool = False + +class AudioTranscriptionRequest(BaseModel): + request_type: Literal["audio"] = "audio" + file: Tuple[str, IOBase, str] + model: str + language: Optional[str] = None + prompt: Optional[str] = None + response_format: Optional[str] = None + temperature: Optional[float] = None + stream: bool = False + + class Config: + arbitrary_types_allowed = True + +class ModerationRequest(BaseModel): + request_type: Literal["moderation"] = "moderation" + input: str + model: Optional[str] = "text-moderation-latest" + stream: bool = False + +class UnifiedRequest(BaseModel): + data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest] = Field(..., discriminator="request_type") + + @model_validator(mode='before') + @classmethod + def set_request_type(cls, values): + if isinstance(values, dict): + if "messages" in values: + values["request_type"] = "chat" + values["data"] = RequestModel(**values) + elif "prompt" in values: + values["request_type"] = "image" + values["data"] = ImageGenerationRequest(**values) + elif "file" in values: + values["request_type"] = "audio" + values["data"] = AudioTranscriptionRequest(**values) + elif "input" in values: + values["request_type"] = "moderation" + values["data"] = ModerationRequest(**values) + else: + raise ValueError("无法确定请求类型") + return values \ No newline at end of file From 81035a5f32d69549ff9e33e521365a5488ca773b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Sep 2024 20:53:05 +0000 Subject: [PATCH 077/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bcab45af..81340c7e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.3 +0.0.4 From bead939d4d40df0942632543519f0a0fc1cef921 Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:47:28 +0800 Subject: [PATCH 078/476] fix: Generate a random API key using only alphanumeric characters --- .gitignore | 3 ++- main.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index fd12d49c..d00e56d2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ node_modules *.jpg *.json *.png -*.db \ No newline at end of file +*.db +.aider* diff --git a/main.py b/main.py index 7b676f1b..c9173f99 100644 --- a/main.py +++ b/main.py @@ -22,6 +22,9 @@ from urllib.parse import urlparse import os +import string +import json + is_debug = bool(os.getenv("DEBUG", False)) from sqlalchemy import inspect, text @@ -463,9 +466,8 @@ def get_matching_providers(self, model_name, token): # if model_name in provider['model'].keys(): # provider_list.append(provider) if is_debug: - import json for provider in provider_list: - print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) + logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): @@ -698,7 +700,11 @@ async def audio_transcriptions( @app.get("/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) def generate_api_key(): - api_key = "sk-" + secrets.token_urlsafe(36) + # Define the character set (only alphanumeric) + chars = string.ascii_letters + string.digits + # Generate a random string of 36 characters + random_string = ''.join(secrets.choice(chars) for _ in range(36)) + api_key = "sk-" + random_string return JSONResponse(content={"api_key": api_key}) # 在 /stats 路由中返回成功和失败百分比 From 331ab5a2ba9313a48871205754edf0f00f63be1a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 27 Sep 2024 06:50:40 +0000 Subject: [PATCH 079/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 81340c7e..bbdeab62 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.4 +0.0.5 From 3e27f73cfe81234b8964d4834089881d3b84dc07 Mon Sep 17 00:00:00 2001 From: dray Date: Sat, 28 Sep 2024 00:20:02 +0800 Subject: [PATCH 080/476] Refactor: Add support for retrieving all models when the model name is "*" --- main.py | 6 ++++++ utils.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index c9173f99..5f2ae619 100644 --- a/main.py +++ b/main.py @@ -419,6 +419,12 @@ def get_matching_providers(self, model_name, token): provider_rules = [] for model in config['api_keys'][api_index]['model']: + if model == "*": + # 如果模型名为 *，则返回所有模型 + for provider in config["providers"]: + for model in provider["model"].keys(): + provider_rules.append(provider["provider"] + "/" + model) + break if "/" in model: if model.startswith("<") and model.endswith(">"): model = model[1:-1] diff --git a/utils.py b/utils.py index 804e9fd2..099c1c54 100644 --- a/utils.py +++ b/utils.py @@ -62,7 +62,7 @@ async def load_config(app=None): # is_quoted = not token.plain # print(f"值: {value}, 是否被引号包裹: {is_quoted}") - with open('./api.yaml', 'r') as f: + with open("./api.yaml", "r", encoding="utf-8") as f: # 判断是否为空文件 conf = yaml.safe_load(f) # conf = None @@ -170,6 +170,10 @@ def post_all_models(token, config, api_list): api_index = api_list.index(token) if config['api_keys'][api_index]['model']: for model in config['api_keys'][api_index]['model']: + if model == "*": + # 如果模型名为 *，则返回所有模型 + all_models = get_all_models(config) + return all_models if "/" in model: provider = model.split("/")[0] model = model.split("/")[1] From b5244f27fa391b31948f9bef1094cf81d291b592 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 27 Sep 2024 17:08:52 +0000 Subject: [PATCH 081/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bbdeab62..1750564f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.5 +0.0.6 From 09c584b9ab175077e08bb25492a83b1e6de3b857 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 01:08:38 +0800 Subject: [PATCH 082/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20bug=20in=20?= =?UTF-8?q?total=20time=20calculation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add support for storing the number ofinput and output tokens to the database. 📖 Docs: Update documentation 💻 Code: Refactor code: Use UUID for each request --- .gitignore | 1 + README.md | 11 +++ README_CN.md | 11 +++ main.py | 246 +++++++++++++++++++++++++++++++++++++++++++-------- models.py | 26 +++--- response.py | 3 +- 6 files changed, 245 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index d00e56d2..6a59d552 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ node_modules *.png *.db .aider* +.idea \ No newline at end of file diff --git a/README.md b/README.md index 7d2d5e3a..947655d5 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,17 @@ api_keys: AUTO_RETRY: true ``` +If you do not want to set available channels for each `api` one by one in `api_keys`, `uni-api` supports setting the `api key` to be able to use all models. The configuration is as follows: + +```yaml +# ... providers configuration unchanged ... +api_keys: + - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to request uni-api, required + model: # The model that can be used with this API Key, required + - * # Can use all models in all channels set under providers, no need to add available channels one by one. +# ... other configurations unchanged ... +``` + ## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional diff --git a/README_CN.md b/README_CN.md index acb14586..88ed03e9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -126,6 +126,17 @@ api_keys: AUTO_RETRY: true ``` +如果你不想在 `api_keys` 里面给每个 `api` 一个个设置可用渠道，`uni-api` 支持将 `api key` 设置为可以使用所有模型，配置如下： + +```yaml +# ... providers 配置不变 ... +api_keys: + - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户请求 uni-api 需要 API key，必填 + model: # 该 API Key 可以使用的模型，必填 + - * # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 +# ... 其他配置不变 ... +``` + ## 环境变量 - CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 diff --git a/main.py b/main.py index 5f2ae619..228c62d0 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import secrets import time as time_module from contextlib import asynccontextmanager +from starlette.middleware.base import BaseHTTPMiddleware from fastapi.middleware.cors import CORSMiddleware from fastapi import FastAPI, HTTPException, Depends, Request @@ -26,6 +27,7 @@ import json is_debug = bool(os.getenv("DEBUG", False)) +# is_debug = False from sqlalchemy import inspect, text from sqlalchemy.sql import sqltypes @@ -106,11 +108,12 @@ async def http_exception_handler(request: Request, exc: HTTPException): content={"message": exc.detail}, ) +import uuid +import json import asyncio from time import time -from collections import defaultdict -from starlette.middleware.base import BaseHTTPMiddleware -import json +import contextvars +request_info = contextvars.ContextVar('request_info', default={}) async def parse_request_body(request: Request): if request.method == "POST" and "application/json" in request.headers.get("content-type", ""): @@ -131,23 +134,31 @@ async def parse_request_body(request: Request): class RequestStat(Base): __tablename__ = 'request_stats' id = Column(Integer, primary_key=True) + request_id = Column(String) endpoint = Column(String) - ip = Column(String) - token = Column(String) - total_time = Column(Float) + client_ip = Column(String) + process_time = Column(Float) + first_response_time = Column(Float) + provider = Column(String) model = Column(String) + # success = Column(Boolean, default=False) + api_key = Column(String) is_flagged = Column(Boolean, default=False) - moderated_content = Column(Text) + text = Column(Text) + prompt_tokens = Column(Integer, default=0) + completion_tokens = Column(Integer, default=0) + total_tokens = Column(Integer, default=0) + # cost = Column(Float, default=0) timestamp = Column(DateTime(timezone=True), server_default=func.now()) class ChannelStat(Base): __tablename__ = 'channel_stats' id = Column(Integer, primary_key=True) + request_id = Column(String) provider = Column(String) model = Column(String) api_key = Column(String) - success = Column(Boolean) - first_response_time = Column(Float) # 新增: 记录首次响应时间 + success = Column(Boolean, default=False) timestamp = Column(DateTime(timezone=True), server_default=func.now()) # 获取数据库路径 @@ -162,6 +173,123 @@ class ChannelStat(Base): engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) +from starlette.types import Scope, Receive, Send +from starlette.responses import Response + +from decimal import Decimal, getcontext + +# 设置全局精度 +getcontext().prec = 17 # 设置为17是为了确保15位小数的精度 + +def calculate_cost(model: str, input_tokens: int, output_tokens: int) -> Decimal: + costs = { + "gpt-4": {"input": Decimal('5.0') / Decimal('1000000'), "output": Decimal('15.0') / Decimal('1000000')}, + "claude-3-sonnet": {"input": Decimal('3.0') / Decimal('1000000'), "output": Decimal('15.0') / Decimal('1000000')} + } + + if model not in costs: + logger.error(f"Unknown model: {model}") + return 0 + + model_costs = costs[model] + input_cost = Decimal(input_tokens) * model_costs["input"] + output_cost = Decimal(output_tokens) * model_costs["output"] + total_cost = input_cost + output_cost + + # 返回精确到15位小数的结果 + return total_cost.quantize(Decimal('0.000000000000001')) + +class LoggingStreamingResponse(Response): + def __init__(self, content, status_code=200, headers=None, media_type=None, current_info=None): + super().__init__(content=None, status_code=status_code, headers=headers, media_type=media_type) + self.body_iterator = content + self._closed = False + self.current_info = current_info + + # Remove Content-Length header if it exists + if 'content-length' in self.headers: + del self.headers['content-length'] + # Set Transfer-Encoding to chunked + self.headers['transfer-encoding'] = 'chunked' + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await send({ + 'type': 'http.response.start', + 'status': self.status_code, + 'headers': self.raw_headers, + }) + + try: + async for chunk in self._logging_iterator(): + await send({ + 'type': 'http.response.body', + 'body': chunk, + 'more_body': True, + }) + finally: + await send({ + 'type': 'http.response.body', + 'body': b'', + 'more_body': False, + }) + if hasattr(self.body_iterator, 'aclose') and not self._closed: + await self.body_iterator.aclose() + self._closed = True + + process_time = time() - self.current_info["start_time"] + self.current_info["process_time"] = process_time + await self.update_stats() + + async def update_stats(self): + # 这里添加更新数据库的逻辑 + # print("current_info2") + async with async_session() as session: + async with session.begin(): + try: + columns = [column.key for column in RequestStat.__table__.columns] + filtered_info = {k: v for k, v in self.current_info.items() if k in columns} + new_request_stat = RequestStat(**filtered_info) + session.add(new_request_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating stats: {str(e)}") + + async def _logging_iterator(self): + try: + async for chunk in self.body_iterator: + if isinstance(chunk, str): + chunk = chunk.encode('utf-8') + line = chunk.decode() + if is_debug: + logger.info(f"{line}") + if line.startswith("data:"): + line = line.lstrip("data: ") + if not line.startswith("[DONE]"): + resp: dict = json.loads(line) + input_tokens = safe_get(resp, "message", "usage", "input_tokens", default=0) + input_tokens = safe_get(resp, "usage", "prompt_tokens", default=0) + output_tokens = safe_get(resp, "usage", "completion_tokens", default=0) + total_tokens = input_tokens + output_tokens + + model = self.current_info.get("model", "") + # total_cost = calculate_cost(model, input_tokens, output_tokens) + self.current_info["prompt_tokens"] = input_tokens + self.current_info["completion_tokens"] = output_tokens + self.current_info["total_tokens"] = total_tokens + # self.current_info["cost"] = total_cost + yield chunk + except Exception as e: + raise + finally: + logger.debug("_logging_iterator finished") + + async def close(self): + if not self._closed: + self._closed = True + if hasattr(self.body_iterator, 'aclose'): + await self.body_iterator.aclose() + class StatsMiddleware(BaseHTTPMiddleware): def __init__(self, app): super().__init__(app) @@ -169,12 +297,6 @@ def __init__(self, app): async def dispatch(self, request: Request, call_next): start_time = time() - endpoint = f"{request.method} {request.url.path}" - client_ip = request.client.host - - model = "unknown" - is_flagged = False - moderated_content = "" enable_moderation = False # 默认不开启道德审查 config = app.state.config @@ -197,16 +319,47 @@ async def dispatch(self, request: Request, call_next): # 如果token为None，检查全局设置 enable_moderation = config.get('ENABLE_MODERATION', False) + # 在 app.state 中存储此请求的信息 + request_id = str(uuid.uuid4()) + + # 初始化请求信息 + request_info_data = { + "request_id": request_id, + "start_time": start_time, + "endpoint": f"{request.method} {request.url.path}", + "client_ip": request.client.host, + "process_time": 0, + "first_response_time": -1, + "provider": None, + "model": None, + "success": False, + "api_key": token, + "is_flagged": False, + "text": None, + "prompt_tokens": 0, + "completion_tokens": 0, + # "cost": 0, + "total_tokens": 0 + } + + # 设置请求信息到上下文 + current_request_info = request_info.set(request_info_data) + current_info = request_info.get() + parsed_body = await parse_request_body(request) if parsed_body: try: request_model = UnifiedRequest.model_validate(parsed_body).data model = request_model.model + current_info["model"] = model if request_model.request_type == "chat": moderated_content = request_model.get_last_text_message() elif request_model.request_type == "image": moderated_content = request_model.prompt + if moderated_content: + current_info["text"] = moderated_content + if enable_moderation and moderated_content: moderation_response = await self.moderate_content(moderated_content, token) @@ -217,12 +370,15 @@ async def dispatch(self, request: Request, call_next): if is_flagged: logger.error(f"Content did not pass the moral check: %s", moderated_content) process_time = time() - start_time - await self.update_stats(endpoint, process_time, client_ip, model, token, is_flagged, moderated_content) + current_info["process_time"] = process_time + current_info["is_flagged"] = is_flagged + await self.update_stats(current_info) return JSONResponse( status_code=400, content={"error": "Content did not pass the moral check, please modify and try again."} ) except RequestValidationError: + logger.error(f"Invalid request body: {parsed_body}") pass except Exception as e: if is_debug: @@ -231,43 +387,51 @@ async def dispatch(self, request: Request, call_next): logger.error(f"处理请求或进行道德检查时出错: {str(e)}") - response = await call_next(request) - process_time = time() - start_time - - # 异步更新数据库 - await self.update_stats(endpoint, process_time, client_ip, model, token, is_flagged, moderated_content) + try: + response = await call_next(request) + + if isinstance(response, StreamingResponse): + response = LoggingStreamingResponse( + content=response.body_iterator, + status_code=response.status_code, + media_type=response.media_type, + headers=response.headers, + current_info=current_info, + ) + elif hasattr(response, 'json'): + logger.info(f"Response: {await response.json()}") + else: + logger.info(f"Response: type={type(response).__name__}, status_code={response.status_code}, headers={response.headers}") - return response + return response + finally: + # print("current_request_info", current_request_info) + request_info.reset(current_request_info) - async def update_stats(self, endpoint, process_time, client_ip, model, token, is_flagged, moderated_content): + async def update_stats(self, current_info): + # 这里添加更新数据库的逻辑 async with async_session() as session: async with session.begin(): try: - new_request_stat = RequestStat( - endpoint=endpoint, - ip=client_ip, - token=token, - total_time=process_time, - model=model, - is_flagged=is_flagged, - moderated_content=moderated_content - ) + columns = [column.key for column in RequestStat.__table__.columns] + filtered_info = {k: v for k, v in current_info.items() if k in columns} + new_request_stat = RequestStat(**filtered_info) session.add(new_request_stat) await session.commit() except Exception as e: await session.rollback() logger.error(f"Error updating stats: {str(e)}") - async def update_channel_stats(self, provider, model, api_key, success, first_response_time): + async def update_channel_stats(self, request_id, provider, model, api_key, success): async with async_session() as session: async with session.begin(): try: channel_stat = ChannelStat( + request_id=request_id, provider=provider, model=model, api_key=api_key, success=success, - first_response_time=first_response_time ) session.add(channel_stat) await session.commit() @@ -357,6 +521,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A pass else: logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) + current_info = request_info.get() try: if request.stream: model = provider['model'][request.model] @@ -372,12 +537,14 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A response = JSONResponse(first_element) # 更新成功计数和首次响应时间 - await app.middleware_stack.app.update_channel_stats(provider['provider'], request.model, token, success=True, first_response_time=first_response_time) + await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + current_info["first_response_time"] = first_response_time + current_info["success"] = True + current_info["provider"] = provider['provider'] return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: - # 更新失败计数,首次响应时间为-1表示失败 - await app.middleware_stack.app.update_channel_stats(provider['provider'], request.model, token, success=False, first_response_time=-1) + await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) raise e @@ -550,6 +717,10 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe else: raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") + current_info = request_info.get() + current_info["first_response_time"] = -1 + current_info["success"] = False + current_info["provider"] = None raise HTTPException(status_code=status_code, detail=f"All {request.model} error: {error_message}") model_handler = ModelRequestHandler() @@ -647,7 +818,6 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def request_model(request: RequestModel, token: str = Depends(verify_api_key)): - # logger.info(f"Request received: {request}") return await model_handler.request_model(request, token) @app.options("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) diff --git a/models.py b/models.py index 17643911..5a01ff1e 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,7 @@ from io import IOBase from pydantic import BaseModel, Field, model_validator from typing import List, Dict, Optional, Union, Tuple, Literal +from log_config import logger class FunctionParameter(BaseModel): type: str @@ -57,8 +58,10 @@ class ToolChoice(BaseModel): type: str function: Optional[FunctionChoice] = None -class RequestModel(BaseModel): - request_type: Literal["chat"] = "chat" +class BaseRequest(BaseModel): + request_type: Optional[Literal["chat", "image", "audio", "moderation"]] = Field(default=None, exclude=True) + +class RequestModel(BaseRequest): model: str messages: List[Message] logprobs: Optional[bool] = None @@ -86,16 +89,14 @@ def get_last_text_message(self) -> Optional[str]: return item.text return "" -class ImageGenerationRequest(BaseModel): - request_type: Literal["image"] = "image" +class ImageGenerationRequest(BaseRequest): prompt: str model: Optional[str] = "dall-e-3" n: Optional[int] = 1 size: Optional[str] = "1024x1024" stream: bool = False -class AudioTranscriptionRequest(BaseModel): - request_type: Literal["audio"] = "audio" +class AudioTranscriptionRequest(BaseRequest): file: Tuple[str, IOBase, str] model: str language: Optional[str] = None @@ -107,31 +108,30 @@ class AudioTranscriptionRequest(BaseModel): class Config: arbitrary_types_allowed = True -class ModerationRequest(BaseModel): - request_type: Literal["moderation"] = "moderation" +class ModerationRequest(BaseRequest): input: str model: Optional[str] = "text-moderation-latest" stream: bool = False class UnifiedRequest(BaseModel): - data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest] = Field(..., discriminator="request_type") + data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest] @model_validator(mode='before') @classmethod def set_request_type(cls, values): if isinstance(values, dict): if "messages" in values: - values["request_type"] = "chat" values["data"] = RequestModel(**values) + values["data"].request_type = "chat" elif "prompt" in values: - values["request_type"] = "image" values["data"] = ImageGenerationRequest(**values) + values["data"].request_type = "image" elif "file" in values: - values["request_type"] = "audio" values["data"] = AudioTranscriptionRequest(**values) + values["data"].request_type = "audio" elif "input" in values: - values["request_type"] = "moderation" values["data"] = ModerationRequest(**values) + values["data"].request_type = "moderation" else: raise ValueError("无法确定请求类型") return values \ No newline at end of file diff --git a/response.py b/response.py index bb6b47a4..ee931d0c 100644 --- a/response.py +++ b/response.py @@ -1,7 +1,6 @@ import json import httpx from datetime import datetime -from io import BytesIO from log_config import logger @@ -32,7 +31,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f sample_data["choices"][0]["delta"] = {"role": role, "content": ""} if total_tokens: total_tokens = prompt_tokens + completion_tokens - sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens,"total_tokens": total_tokens} + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} sample_data["choices"] = [] json_data = json.dumps(sample_data, ensure_ascii=False) From dd2744868de0ed73c23ed573291e77edf4635e86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 28 Sep 2024 17:09:07 +0000 Subject: [PATCH 083/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1750564f..5a5831ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.6 +0.0.7 From 27b2ccadb64c000f877aebd856779232b0375ef9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 03:54:29 +0800 Subject: [PATCH 084/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20FastAPI=20internally=20used=20different=20types=20of?= =?UTF-8?q?=20streaming=20responses.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 228c62d0..2e33b1e5 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,9 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi import FastAPI, HTTPException, Depends, Request -from fastapi.responses import StreamingResponse, JSONResponse +from fastapi.responses import JSONResponse +from fastapi.responses import StreamingResponse as FastAPIStreamingResponse +from starlette.responses import StreamingResponse as StarletteStreamingResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError @@ -390,7 +392,7 @@ async def dispatch(self, request: Request, call_next): try: response = await call_next(request) - if isinstance(response, StreamingResponse): + if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)): response = LoggingStreamingResponse( content=response.body_iterator, status_code=response.status_code, @@ -527,14 +529,15 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A model = provider['model'][request.model] generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) - response = StreamingResponse(wrapped_generator, media_type="text/event-stream") + response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: generator = fetch_response(app.state.client, url, headers, payload) wrapped_generator, first_response_time = await error_handling_wrapper(generator) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") first_element = json.loads(first_element) - response = JSONResponse(first_element) + response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") + # response = JSONResponse(first_element) # 更新成功计数和首次响应时间 await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) From d65ef8e59732d39af2cd283f81311c5ac6853a36 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 28 Sep 2024 19:54:48 +0000 Subject: [PATCH 085/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5a5831ab..d169b2f2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.7 +0.0.8 From 7f50586d4a79032324fcb75308682a27543a3693 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 04:00:03 +0800 Subject: [PATCH 086/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20FastAPI=20internally=20used=20different=20types=20of?= =?UTF-8?q?=20streaming=20responses.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 2e33b1e5..020afa87 100644 --- a/main.py +++ b/main.py @@ -392,7 +392,7 @@ async def dispatch(self, request: Request, call_next): try: response = await call_next(request) - if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)): + if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)) or type(response).__name__ == '_StreamingResponse': response = LoggingStreamingResponse( content=response.body_iterator, status_code=response.status_code, From 9c2088c1f93ab442a26c7db94971b579f27330a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 28 Sep 2024 20:00:24 +0000 Subject: [PATCH 087/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d169b2f2..c5d54ec3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.8 +0.0.9 From 179cefd37db1a199f0257333b0ade4b74e7b7c25 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 14:03:01 +0800 Subject: [PATCH 088/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20makes=20moral=20checks=20unusable.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 020afa87..850ef40d 100644 --- a/main.py +++ b/main.py @@ -365,9 +365,7 @@ async def dispatch(self, request: Request, call_next): if enable_moderation and moderated_content: moderation_response = await self.moderate_content(moderated_content, token) - moderation_result = moderation_response.body - moderation_data = json.loads(moderation_result) - is_flagged = moderation_data.get('results', [{}])[0].get('flagged', False) + is_flagged = moderation_response.get('results', [{}])[0].get('flagged', False) if is_flagged: logger.error(f"Content did not pass the moral check: %s", moderated_content) @@ -447,7 +445,18 @@ async def moderate_content(self, content, token): # 直接调用 moderations 函数 response = await moderations(moderation_request, token) - return response + # 读取流式响应的内容 + moderation_result = b"" + async for chunk in response.body_iterator: + if isinstance(chunk, str): + moderation_result += chunk.encode('utf-8') + else: + moderation_result += chunk + + # 解码并解析 JSON + moderation_data = json.loads(moderation_result.decode('utf-8')) + + return moderation_data # 配置 CORS 中间件 app.add_middleware( From 9a6b4f490f7e5eeb21a7a34a9ca941b757847e35 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 29 Sep 2024 06:03:21 +0000 Subject: [PATCH 089/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c5d54ec3..7c1886bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.9 +0.0.10 From 6101f6db9fe6618162a0b16aea1d99e20882fa08 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 17:10:35 +0800 Subject: [PATCH 090/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20permissions=20control=20for=20model=20distribu?= =?UTF-8?q?tors=20is=20incorrect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 850ef40d..cdc7e401 100644 --- a/main.py +++ b/main.py @@ -620,9 +620,14 @@ def get_matching_providers(self, model_name, token): models_list.extend(list(provider['model'].keys())) # print("models_list", models_list) # print("model_name", model_name) + # print("model_name_split", model_name_split) # print("model", model) - if (model_name_split and model_name in models_list) or (model_name_split == "*" and model_name in models_list): - provider_rules.append(provider_name) + if model_name_split == "*": + if model_name in models_list: + provider_rules.append(provider_name) + elif model_name_split == model_name: + if model_name in models_list: + provider_rules.append(provider_name) else: for provider in config['providers']: if model in provider['model'].keys(): @@ -666,7 +671,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False)) if not matching_providers: raise HTTPException(status_code=404, detail="No matching model found") - + # exit(0) # 检查是否启用轮询 api_index = api_list.index(token) weights = safe_get(config, 'api_keys', api_index, "weights") From d7169b08fbbdc88c2dd9095f0f511433bd16d933 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 29 Sep 2024 09:11:55 +0000 Subject: [PATCH 091/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7c1886bb..2cfabea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.10 +0.0.11 From ea47d2878374cd84f5d06dc1b485b69abb734197 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 29 Sep 2024 17:18:29 +0800 Subject: [PATCH 092/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20there=20is=20no=20error=20handling=20for=20response?= =?UTF-8?q?=20JSON=20parsing=20errors.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index cdc7e401..8560df11 100644 --- a/main.py +++ b/main.py @@ -268,18 +268,19 @@ async def _logging_iterator(self): if line.startswith("data:"): line = line.lstrip("data: ") if not line.startswith("[DONE]"): - resp: dict = json.loads(line) - input_tokens = safe_get(resp, "message", "usage", "input_tokens", default=0) - input_tokens = safe_get(resp, "usage", "prompt_tokens", default=0) - output_tokens = safe_get(resp, "usage", "completion_tokens", default=0) - total_tokens = input_tokens + output_tokens - - model = self.current_info.get("model", "") - # total_cost = calculate_cost(model, input_tokens, output_tokens) - self.current_info["prompt_tokens"] = input_tokens - self.current_info["completion_tokens"] = output_tokens - self.current_info["total_tokens"] = total_tokens - # self.current_info["cost"] = total_cost + try: + resp: dict = json.loads(line) + input_tokens = safe_get(resp, "message", "usage", "input_tokens", default=0) + input_tokens = safe_get(resp, "usage", "prompt_tokens", default=0) + output_tokens = safe_get(resp, "usage", "completion_tokens", default=0) + total_tokens = input_tokens + output_tokens + + self.current_info["prompt_tokens"] = input_tokens + self.current_info["completion_tokens"] = output_tokens + self.current_info["total_tokens"] = total_tokens + except Exception as e: + logger.error(f"Error parsing response: {str(e)}, line: {repr(line)}") + continue yield chunk except Exception as e: raise From d4b22a2cba5b6bfa86428789a5a7370b7ea9bf95 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 29 Sep 2024 09:18:48 +0000 Subject: [PATCH 093/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2cfabea2..8cbf02c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.11 +0.0.12 From b197d5b36652ebbbf6007031652235d66158376f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 1 Oct 2024 03:28:50 +0800 Subject: [PATCH 094/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20The=20/stats=20endpoint=20supports=20passing=20an=20'hours'?= =?UTF-8?q?=20parameter=20to=20specify=20the=20number=20of=20hours=20of=20?= =?UTF-8?q?historical=20statistics=20data=20to=20return?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 8560df11..917f0c74 100644 --- a/main.py +++ b/main.py @@ -902,12 +902,20 @@ def generate_api_key(): return JSONResponse(content={"api_key": api_key}) # 在 /stats 路由中返回成功和失败百分比 -from collections import defaultdict +from datetime import datetime, timedelta, timezone from sqlalchemy import func, desc, case +from fastapi import Query @app.get("/stats", dependencies=[Depends(rate_limit_dependency)]) -async def get_stats(request: Request, token: str = Depends(verify_admin_api_key)): +async def get_stats( + request: Request, + token: str = Depends(verify_admin_api_key), + hours: int = Query(default=24, ge=1, le=720, description="Number of hours to look back for stats (1-720)") +): async with async_session() as session: + # 计算指定时间范围的开始时间 + start_time = datetime.now(timezone.utc) - timedelta(hours=hours) + # 1. 每个渠道下面每个模型的成功率 channel_model_stats = await session.execute( select( @@ -915,7 +923,9 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) ChannelStat.model, func.count().label('total'), func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') - ).group_by(ChannelStat.provider, ChannelStat.model) + ) + .where(ChannelStat.timestamp >= start_time) + .group_by(ChannelStat.provider, ChannelStat.model) ) channel_model_stats = channel_model_stats.fetchall() @@ -925,14 +935,17 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) ChannelStat.provider, func.count().label('total'), func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') - ).group_by(ChannelStat.provider) + ) + .where(ChannelStat.timestamp >= start_time) + .group_by(ChannelStat.provider) ) channel_stats = channel_stats.fetchall() # 3. 每个模型在所有渠道总的请求次数 model_stats = await session.execute( - select(ChannelStat.model, func.count().label('count')) - .group_by(ChannelStat.model) + select(RequestStat.model, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.model) .order_by(desc('count')) ) model_stats = model_stats.fetchall() @@ -940,6 +953,7 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) # 4. 每个端点的请求次数 endpoint_stats = await session.execute( select(RequestStat.endpoint, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) .group_by(RequestStat.endpoint) .order_by(desc('count')) ) @@ -947,25 +961,29 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) # 5. 每个ip请求的次数 ip_stats = await session.execute( - select(RequestStat.ip, func.count().label('count')) - .group_by(RequestStat.ip) + select(RequestStat.client_ip, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.client_ip) .order_by(desc('count')) ) ip_stats = ip_stats.fetchall() # 处理统计数据并返回 stats = { + "time_range": f"Last {hours} hours", "channel_model_success_rates": [ { "provider": stat.provider, "model": stat.model, - "success_rate": stat.success_count / stat.total if stat.total > 0 else 0 + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0, + "total_requests": stat.total } for stat in sorted(channel_model_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) ], "channel_success_rates": [ { "provider": stat.provider, - "success_rate": stat.success_count / stat.total if stat.total > 0 else 0 + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0, + "total_requests": stat.total } for stat in sorted(channel_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) ], "model_request_counts": [ @@ -982,7 +1000,7 @@ async def get_stats(request: Request, token: str = Depends(verify_admin_api_key) ], "ip_request_counts": [ { - "ip": stat.ip, + "ip": stat.client_ip, "count": stat.count } for stat in ip_stats ] From f1141e223b1de78b0568425371e440d25bced566 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 30 Sep 2024 19:29:09 +0000 Subject: [PATCH 095/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8cbf02c3..43b29618 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.12 +0.0.13 From adf018ae3a95ce2105a302e37ec008b64f366597 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 1 Oct 2024 18:20:48 +0800 Subject: [PATCH 096/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20using=20asterisks=20for=20api=20key=20does=20not=20c?= =?UTF-8?q?onform=20to=20yaml=20syntax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- main.py | 2 +- utils.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 947655d5..f474ee82 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ If you do not want to set available channels for each `api` one by one in `api_k api_keys: - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to request uni-api, required model: # The model that can be used with this API Key, required - - * # Can use all models in all channels set under providers, no need to add available channels one by one. + - all # Can use all models in all channels set under providers, no need to add available channels one by one. # ... other configurations unchanged ... ``` diff --git a/README_CN.md b/README_CN.md index 88ed03e9..7714ae09 100644 --- a/README_CN.md +++ b/README_CN.md @@ -133,7 +133,7 @@ api_keys: api_keys: - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户请求 uni-api 需要 API key，必填 model: # 该 API Key 可以使用的模型，必填 - - * # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 + - all # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 # ... 其他配置不变 ... ``` diff --git a/main.py b/main.py index 917f0c74..bcb93037 100644 --- a/main.py +++ b/main.py @@ -599,7 +599,7 @@ def get_matching_providers(self, model_name, token): provider_rules = [] for model in config['api_keys'][api_index]['model']: - if model == "*": + if model == "all": # 如果模型名为 *，则返回所有模型 for provider in config["providers"]: for model in provider["model"].keys(): diff --git a/utils.py b/utils.py index 099c1c54..14b46eeb 100644 --- a/utils.py +++ b/utils.py @@ -170,8 +170,8 @@ def post_all_models(token, config, api_list): api_index = api_list.index(token) if config['api_keys'][api_index]['model']: for model in config['api_keys'][api_index]['model']: - if model == "*": - # 如果模型名为 *，则返回所有模型 + if model == "all": + # 如果模型名为 all，则返回所有模型 all_models = get_all_models(config) return all_models if "/" in model: From e0e0c2dfb982bc1f305fdb9113abf2e638cf9c6f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 1 Oct 2024 10:22:33 +0000 Subject: [PATCH 097/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.14?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 43b29618..9789c4cc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.13 +0.0.14 From 4b9412983c5de5d82a96a3e1c877d8b931be556c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 2 Oct 2024 03:52:40 +0800 Subject: [PATCH 098/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20the=20option=20request=20attempts=20to=20parse=20?= =?UTF-8?q?the=20OK=20stream=20message.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug of uneven allocation in the round-robin model request. --- main.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index bcb93037..3ee53aaf 100644 --- a/main.py +++ b/main.py @@ -267,7 +267,7 @@ async def _logging_iterator(self): logger.info(f"{line}") if line.startswith("data:"): line = line.lstrip("data: ") - if not line.startswith("[DONE]"): + if not line.startswith("[DONE]") and not line.startswith("OK"): try: resp: dict = json.loads(line) input_tokens = safe_get(resp, "message", "usage", "input_tokens", default=0) @@ -587,7 +587,8 @@ def weighted_round_robin(weights): import asyncio class ModelRequestHandler: def __init__(self): - self.last_provider_index = -1 + self.last_provider_indices = defaultdict(lambda: -1) + self.locks = defaultdict(asyncio.Lock) def get_matching_providers(self, model_name, token): config = app.state.config @@ -708,10 +709,18 @@ async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRe status_code = 500 error_message = None num_providers = len(providers) - start_index = self.last_provider_index + 1 if use_round_robin else 0 + model_name = request.model + + if use_round_robin: + async with self.locks[model_name]: + self.last_provider_indices[model_name] = (self.last_provider_indices[model_name] + 1) % num_providers + start_index = self.last_provider_indices[model_name] + else: + start_index = 0 + for i in range(num_providers + 1): - self.last_provider_index = (start_index + i) % num_providers - provider = providers[self.last_provider_index] + current_index = (start_index + i) % num_providers + provider = providers[current_index] try: response = await process_request(request, provider, endpoint, token) return response From 2d653d6ac820080fa3e850c25617d5ba4223ba69 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 1 Oct 2024 19:53:05 +0000 Subject: [PATCH 099/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.15?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9789c4cc..ceddfb28 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.14 +0.0.15 From 0a30d42e274baa16611f67d267d11c30fe292dcd Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 4 Oct 2024 02:10:49 +0800 Subject: [PATCH 100/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Change=20the=20la?= =?UTF-8?q?st=20character=20of=20the=20SSE=20message=20to=20\n\n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/response.py b/response.py index ee931d0c..c0603fc6 100644 --- a/response.py +++ b/response.py @@ -4,6 +4,12 @@ from log_config import logger +# end_of_line = "\n\r\n" +# end_of_line = "\r\n" +# end_of_line = "\n\r" +end_of_line = "\n\n" +# end_of_line = "\r" +# end_of_line = "\n" async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): sample_data = { @@ -36,7 +42,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f json_data = json.dumps(sample_data, ensure_ascii=False) # 构建SSE响应 - sse_response = f"data: {json_data}\n\r\n" + sse_response = f"data: {json_data}" + end_of_line return sse_response @@ -94,7 +100,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response = json.dumps(function_call["functionCall"]["args"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id="chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV", function_call_name=None, function_call_content=function_full_response) yield sse_string - yield "data: [DONE]\n\r\n" + yield "data: [DONE]" + end_of_line async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) @@ -141,7 +147,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod function_full_response = json.dumps(function_call["input"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id=function_call_id, function_call_name=None, function_call_content=function_full_response) yield sse_string - yield "data: [DONE]\n\r\n" + yield "data: [DONE]" + end_of_line async def fetch_gpt_response_stream(client, url, headers, payload): async with client.stream('POST', url, headers=headers, json=payload) as response: @@ -157,7 +163,7 @@ async def fetch_gpt_response_stream(client, url, headers, payload): line, buffer = buffer.split("\n", 1) # logger.info("line: %s", repr(line)) if line and line != "data: " and line != "data:" and not line.startswith(": "): - yield line.strip() + "\n\r\n" + yield line.strip() + end_of_line async def fetch_cloudflare_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) @@ -176,7 +182,7 @@ async def fetch_cloudflare_response_stream(client, url, headers, payload, model) if line.startswith("data:"): line = line.lstrip("data: ") if line == "[DONE]": - yield "data: [DONE]\n\r\n" + yield "data: [DONE]" + end_of_line return resp: dict = json.loads(line) message = resp.get("response") @@ -200,7 +206,7 @@ async def fetch_cohere_response_stream(client, url, headers, payload, model): # logger.info("line: %s", repr(line)) resp: dict = json.loads(line) if resp.get("is_finished") == True: - yield "data: [DONE]\n\r\n" + yield "data: [DONE]" + end_of_line return if resp.get("event_type") == "text-generation": message = resp.get("text") @@ -266,7 +272,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): function_call_content = delta["partial_json"] sse_string = await generate_sse_response(timestamp, model, None, None, None, function_call_content) yield sse_string - yield "data: [DONE]\n\r\n" + yield "data: [DONE]" + end_of_line async def fetch_response(client, url, headers, payload): response = None From d4d650a1d6834564f6a2ebe9e07d25da4ff27df1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Oct 2024 18:11:10 +0000 Subject: [PATCH 101/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.16?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ceddfb28..e3b86dd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.15 +0.0.16 From 2ec384d592cf456ebffbd13c4898e333d8e0379d Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Sat, 5 Oct 2024 09:59:07 +0800 Subject: [PATCH 102/476] feat: add TextToSpeechRequest model and implement audio speech endpoint with processing logic --- main.py | 20 +++++++++++++++++--- models.py | 10 +++++++++- request.py | 29 ++++++++++++++++++++++++++++- response.py | 5 ++++- utils.py | 1 + 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 3ee53aaf..4cc6834f 100644 --- a/main.py +++ b/main.py @@ -15,7 +15,7 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError -from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, TextToSpeechRequest, UnifiedRequest from request import get_payload from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder @@ -360,6 +360,9 @@ async def dispatch(self, request: Request, call_next): moderated_content = request_model.get_last_text_message() elif request_model.request_type == "image": moderated_content = request_model.prompt + elif model.startswith("tts"): + moderated_content = request_model.input + if moderated_content: current_info["text"] = moderated_content @@ -521,6 +524,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "moderation" request.stream = False + if endpoint == "/v1/audio/speech": + engine = "tts" + request.stream = False + if provider.get("engine"): engine = provider["engine"] @@ -662,7 +669,7 @@ def get_matching_providers(self, model_name, token): logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, TextToSpeechRequest], token: str, endpoint=None): config = app.state.config # api_keys_db = app.state.api_keys_db api_list = app.state.api_list @@ -705,7 +712,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token) # 在 try_all_providers 函数中处理失败的情况 - async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): + async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, TextToSpeechRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): status_code = 500 error_message = None num_providers = len(providers) @@ -866,6 +873,13 @@ async def images_generations( ): return await model_handler.request_model(request, token, endpoint="/v1/images/generations") +@app.post("/v1/audio/speech", dependencies=[Depends(rate_limit_dependency)]) +async def audio_speech( + request: TextToSpeechRequest, + token: str = Depends(verify_api_key) +): + return await model_handler.request_model(request, token, endpoint="/v1/audio/speech") + @app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) async def moderations( request: ModerationRequest, diff --git a/models.py b/models.py index 5a01ff1e..0719d643 100644 --- a/models.py +++ b/models.py @@ -134,4 +134,12 @@ def set_request_type(cls, values): values["data"].request_type = "moderation" else: raise ValueError("无法确定请求类型") - return values \ No newline at end of file + return values + +class TextToSpeechRequest(BaseRequest): + model: str + input: str + voice: str + response_format: Optional[str] = "mp3" + speed: Optional[float] = 1.0 + stream: Optional[bool] = False # Add this line diff --git a/request.py b/request.py index 548d669f..9426a3ec 100644 --- a/request.py +++ b/request.py @@ -1,6 +1,7 @@ import os import re import json +from venv import logger import httpx import base64 import urllib.parse @@ -1134,7 +1135,33 @@ async def get_payload(request: RequestModel, engine, provider): return await get_dalle_payload(request, engine, provider) elif engine == "whisper": return await get_whisper_payload(request, engine, provider) + elif engine == "tts": + return await get_tts_payload(request, engine, provider) elif engine == "moderation": return await get_moderation_payload(request, engine, provider) else: - raise ValueError("Unknown payload") \ No newline at end of file + raise ValueError("Unknown payload") + +async def get_tts_payload(request, engine, provider): + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + url = provider['base_url'] + url = BaseAPI(url).audio_speech + + payload = { + "model": provider['model'][request.model], + "input": request.input, + "voice": request.voice, + } + + if request.response_format: + payload["response_format"] = request.response_format + if request.speed: + payload["speed"] = request.speed + if request.stream is not None: + payload["stream"] = request.stream + + return url, headers, payload diff --git a/response.py b/response.py index c0603fc6..a7e4715e 100644 --- a/response.py +++ b/response.py @@ -285,7 +285,10 @@ async def fetch_response(client, url, headers, payload): if error_message: yield error_message return - yield response.json() + if url.endswith("/v1/audio/speech"): + yield response.read() + else: + yield response.json() async def fetch_response_stream(client, url, headers, payload, engine, model): try: diff --git a/utils.py b/utils.py index 14b46eeb..c3748988 100644 --- a/utils.py +++ b/utils.py @@ -313,6 +313,7 @@ def __init__( self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/moderations",) + ("",) * 3) + self.audio_speech: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/speech",) + ("",) * 3) def safe_get(data, *keys, default=None): for key in keys: From 599e110cd9975fac43356b8764616a055cf1ce74 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 7 Oct 2024 03:31:12 +0800 Subject: [PATCH 103/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20OpenAI=20JSON=20format=20output.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation --- README.md | 16 ++++++++++++++++ README_CN.md | 16 ++++++++++++++++ models.py | 11 ++++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f474ee82..60e7104c 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,22 @@ api_keys: - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional - TIMEOUT: Request timeout, default is 100 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional +## Retrieve Statistical Data + +Use `/stats` to get usage statistics for each channel over the last 24 hours. Include your own uni-api admin API key. + +The data includes: + +1. Success rate for each model under each channel, sorted from highest to lowest success rate. +2. Overall success rate for each channel, sorted from highest to lowest. +3. Total number of requests for each model across all channels. +4. Number of requests for each endpoint. +5. Number of requests from each IP address. + +`/stats?hours=48` The `hours` parameter can control how many hours of recent data statistics are returned. If the `hours` parameter is not provided, it defaults to statistics for the last 24 hours. + +There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed ethical review, text content of each request, API key for each request, input token count, and output token count for each request. + ## Docker Local Deployment Start the container diff --git a/README_CN.md b/README_CN.md index 7714ae09..04a2d959 100644 --- a/README_CN.md +++ b/README_CN.md @@ -142,6 +142,22 @@ api_keys: - CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 - TIMEOUT: 请求超时时间，默认为 100 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +## 获取统计数据 + +使用 `/stats` 获取最近 24 小时各个渠道的使用情况统计。同时带上 自己的 uni-api 的 admin API key。 + +数据包括： + +1. 每个渠道下面每个模型的成功率，成功率从高到低排序。 +2. 每个渠道总的成功率，成功率从高到低排序。 +3. 每个模型在所有渠道总的请求次数。 +4. 每个端点的请求次数。 +5. 每个ip请求的次数。 + +`/stats?hours=48` 参数 `hours` 可以控制返回最近多少小时的数据统计，不传 `hours` 这个参数，默认统计最近 24 小时的统计数据。 + +还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 + ## Docker Local Deployment Start the container diff --git a/models.py b/models.py index 5a01ff1e..fd366973 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,6 @@ from io import IOBase from pydantic import BaseModel, Field, model_validator -from typing import List, Dict, Optional, Union, Tuple, Literal +from typing import List, Dict, Optional, Union, Tuple, Literal, Any from log_config import logger class FunctionParameter(BaseModel): @@ -61,6 +61,14 @@ class ToolChoice(BaseModel): class BaseRequest(BaseModel): request_type: Optional[Literal["chat", "image", "audio", "moderation"]] = Field(default=None, exclude=True) +class JsonSchema(BaseModel): + name: str + schema: Dict[str, Any] + +class ResponseFormat(BaseModel): + type: Literal["text", "json_object", "json_schema"] + json_schema: Optional[JsonSchema] = None + class RequestModel(BaseRequest): model: str messages: List[Message] @@ -77,6 +85,7 @@ class RequestModel(BaseRequest): user: Optional[str] = None tool_choice: Optional[Union[str, ToolChoice]] = None tools: Optional[List[Tool]] = None + response_format: Optional[ResponseFormat] = None # 新增字段 def get_last_text_message(self) -> Optional[str]: for message in reversed(self.messages): From 74dc0c00adb2a79f76ec642754a5ca43ffb1e228 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 6 Oct 2024 19:31:34 +0000 Subject: [PATCH 104/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.17?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e3b86dd9..cd231804 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.16 +0.0.17 From 1a3aa56461eb40aeafd17f7d5bb4a0f06be8723f Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:56:01 +0800 Subject: [PATCH 105/476] remove unused import of logger --- request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/request.py b/request.py index 9426a3ec..a054ff7e 100644 --- a/request.py +++ b/request.py @@ -1,7 +1,6 @@ import os import re import json -from venv import logger import httpx import base64 import urllib.parse From 8cf055521c3934b1de248c6151987b07ba31b582 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 7 Oct 2024 21:28:49 +0800 Subject: [PATCH 106/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20channel=20responses=20fail=20but=20are=20replied=20t?= =?UTF-8?q?o=20normally=20and=20are=20not=20recognized=20as=20a=20request?= =?UTF-8?q?=20failure.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 14b46eeb..620cbec2 100644 --- a/utils.py +++ b/utils.py @@ -133,7 +133,10 @@ async def error_handling_wrapper(generator): logger.error("error_handling_wrapper [DONE]!") raise StopAsyncIteration if "The bot's usage is covered by the developer" in first_item_str: - logger.error("error const string!") + logger.error("error const string: %s", first_item_str) + raise StopAsyncIteration + if "process this request due to overload or policy" in first_item_str: + logger.error("error const string: %s", first_item_str) raise StopAsyncIteration try: first_item_str = json.loads(first_item_str) From 8ad5d0e0ffdec87eec5eba7d6a804065b60ef702 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 7 Oct 2024 13:29:08 +0000 Subject: [PATCH 107/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.18?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cd231804..32786aa4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.17 +0.0.18 From 923b3788ae69cd1abbf3dcdea381461bc97bc92b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 13 Oct 2024 03:09:01 +0800 Subject: [PATCH 108/476] =?UTF-8?q?=E2=9C=A8=20Feature:=201.=20Add=20suppo?= =?UTF-8?q?rt=20for=20experimental=20frontend.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Add support for configuration files without listing models. --- .gitignore | 2 +- components/provider_table.py | 230 +++++++++++++++ main.py | 39 ++- models.py | 14 +- request.py | 70 +++-- requirements.txt | 3 +- test/test_ruamel_yaml.py | 44 +++ test/xue/test_dropdown_sheet.py | 75 +++++ test/xue/test_form_uni_api.py | 116 ++++++++ test/xue/test_home.py | 476 ++++++++++++++++++++++++++++++++ utils.py | 141 ++++++---- 11 files changed, 1100 insertions(+), 110 deletions(-) create mode 100644 components/provider_table.py create mode 100644 test/test_ruamel_yaml.py create mode 100644 test/xue/test_dropdown_sheet.py create mode 100644 test/xue/test_form_uni_api.py create mode 100644 test/xue/test_home.py diff --git a/.gitignore b/.gitignore index 6a59d552..bcdffe78 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ api.json -api.yaml +*.yaml .env __pycache__ .vscode diff --git a/components/provider_table.py b/components/provider_table.py new file mode 100644 index 00000000..f858d72b --- /dev/null +++ b/components/provider_table.py @@ -0,0 +1,230 @@ +from xue import Div, Table, Thead, Tbody, Tr, Th, Td, Button, Input, Script, Head, Style, Span +from xue.components.checkbox import checkbox +from xue.components.dropdown import dropdown_menu, dropdown_menu_content +from xue.components.button import button +from xue.components.input import input + +Head.add_default_children([ + Style(""" + .data-table-container { + width: 100%; + overflow-x: auto; + border: 1px solid #e2e8f0; + border-radius: 0.5rem; + overflow-x: visible !important; + } + .data-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + } + .data-table th, .data-table td { + padding: 0.75rem 1rem; + text-align: left; + border-bottom: 1px solid #e2e8f0; + } + .data-table th { + font-weight: 500; + font-size: 0.875rem; + color: #4b5563; + height: 2.5rem; + transition: background-color 0.2s; + } + .data-table thead tr:hover th, + .data-table tbody tr:hover { + background-color: #f8fafc; + } + .data-table tbody tr:last-child td { + border-bottom: none; + } + .sortable-header { + cursor: pointer; + user-select: none; + display: inline-flex; + align-items: center; + padding: 0.25rem 0.5rem; + border-radius: 0.25rem; + transition: background-color 0.2s; + } + .sortable-header:hover { + background-color: #e5e7eb; + } + .sort-icon { + display: inline-block; + width: 1rem; + height: 1rem; + margin-left: 0.25rem; + transition: transform 0.2s; + opacity: 0; + } + .sortable-header:hover .sort-icon, + .sort-asc .sort-icon, + .sort-desc .sort-icon { + opacity: 1; + } + .sort-asc .sort-icon { + transform: rotate(180deg); + } + .table-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + } + .table-footer { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + .pagination { + display: flex; + gap: 0.5rem; + } + @media (prefers-color-scheme: dark) { + .data-table-container { + border-color: #4b5563; + } + .data-table th, .data-table td { + border-color: #4b5563; + } + .data-table th { + color: #d1d5db; + } + .data-table thead tr:hover th, + .data-table tbody tr:hover { + background-color: #1f2937; + } + .sortable-header:hover { + background-color: #374151; + } + } + """, id="data-table-style"), + Script(""" + function toggleAllRows(checked) { + const checkboxes = document.querySelectorAll('.row-checkbox'); + checkboxes.forEach(cb => cb.checked = checked); + updateSelectedCount(); + } + + function updateSelectedCount() { + const selectedCount = document.querySelectorAll('.row-checkbox:checked').length; + const totalCount = document.querySelectorAll('.row-checkbox').length; + document.getElementById('selected-count').textContent = `${selectedCount} of ${totalCount} row(s) selected.`; + } + + function sortTable(columnIndex, accessor) { + const table = document.querySelector('.data-table'); + const header = table.querySelector(`th[data-accessor="${accessor}"]`); + const isAscending = !header.classList.contains('sort-asc'); + + // Update sort direction + table.querySelectorAll('th').forEach(th => th.classList.remove('sort-asc', 'sort-desc')); + header.classList.add(isAscending ? 'sort-asc' : 'sort-desc'); + + // Sort the table + const rows = Array.from(table.querySelectorAll('tbody tr')); + rows.sort((a, b) => { + const aValue = a.querySelector(`td[data-accessor="${accessor}"]`).textContent; + const bValue = b.querySelector(`td[data-accessor="${accessor}"]`).textContent; + return isAscending ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue); + }); + + // Update the table + const tbody = table.querySelector('tbody'); + rows.forEach(row => tbody.appendChild(row)); + } + + document.addEventListener('change', function(event) { + if (event.target.classList.contains('row-checkbox')) { + updateSelectedCount(); + } + }); + """, id="data-table-script"), +]) + +def data_table(columns, data, id, with_filter=True): + return Div( + Div( + input(type="text", placeholder="Filter...", id=f"{id}-filter", class_="mr-auto"), + Div( + button( + "Add Provider", + variant="secondary", + hx_get="/add-provider-sheet", + hx_target="#sheet-container", + hx_swap="innerHTML", + class_="h-[2.625rem]" + ), + dropdown_menu("Columns"), + ), + class_="table-header flex items-center" + ) if with_filter else None, + Div( + Div( + Table( + Thead( + Tr( + Th(checkbox("select-all", "", onclick="toggleAllRows(this.checked)")), + *[Th( + Div( + col['label'], + Span("▼", class_="sort-icon"), + class_="sortable-header" if col.get('sortable', False) else "", + onclick=f"sortTable({i}, '{col['value']}')" if col.get('sortable', False) else None + ), + data_accessor=col['value'] + ) for i, col in enumerate(columns)], + Th("Actions") # 新增的操作列 + ) + ), + Tbody( + *[Tr( + Td(checkbox(f"row-{i}", "", class_="row-checkbox")), + *[Td(row[col['value']], data_accessor=col['value']) for col in columns], + Td(row_actions_menu(i)), # 使用行索引作为 row_id + id=f"row-{i}" + ) for i, row in enumerate(data)] + ), + class_="data-table" + ), + class_="data-table-container" + ), + Div( + Div(id="selected-count", class_="text-sm text-gray-500"), + Div( + button("Previous", variant="outline", class_="mr-2"), + button("Next", variant="outline"), + class_="pagination" + ), + class_="table-footer" + ), + id=id + ), + ) + +def get_column_visibility_menu(id, columns): + return dropdown_menu_content(id, [ + {"label": col['label'], "value": col['value']} + for col in columns if col.get('can_hide', True) + ]) + +def row_actions_menu(row_id): + return dropdown_menu("⋮", id=f"row-actions-menu-{row_id}", hx_get=f"/dropdown-menu/dropdown-menu-⋮/{row_id}") + +def get_row_actions_menu(row_id): + return dropdown_menu_content(f"row-actions-{row_id}", [ + {"label": "Edit", "icon": "pencil"}, + {"label": "Duplicate", "icon": "copy"}, + {"label": "Delete", "icon": "trash"}, + "separator", + {"label": "More...", "icon": "more-horizontal"}, + ]) + +def render_row(row_data, row_id, columns): + return Tr( + Td(checkbox(f"row-{row_id}", "", class_="row-checkbox")), + *[Td(row_data[col['value']], data_accessor=col['value']) for col in columns], + Td(row_actions_menu(row_id)), + id=f"row-{row_id}" + ).render() \ No newline at end of file diff --git a/main.py b/main.py index 3ee53aaf..5ec8f1ff 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest from request import get_payload from response import fetch_response, fetch_response_stream -from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder +from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder, get_model_dict from collections import defaultdict from typing import List, Dict, Union @@ -492,20 +492,21 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A else: engine = "gpt" - if "claude" not in provider['model'][request.model] \ - and "gpt" not in provider['model'][request.model] \ - and "gemini" not in provider['model'][request.model] \ + model_dict = get_model_dict(provider) + if "claude" not in model_dict[request.model] \ + and "gpt" not in model_dict[request.model] \ + and "gemini" not in model_dict[request.model] \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" - if "claude" in provider['model'][request.model] and engine == "vertex": + if "claude" in model_dict[request.model] and engine == "vertex": engine = "vertex-claude" - if "gemini" in provider['model'][request.model] and engine == "vertex": + if "gemini" in model_dict[request.model] and engine == "vertex": engine = "vertex-gemini" - if "o1-preview" in provider['model'][request.model] or "o1-mini" in provider['model'][request.model]: + if "o1-preview" in model_dict[request.model] or "o1-mini" in model_dict[request.model]: engine = "o1" request.stream = False @@ -536,7 +537,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A current_info = request_info.get() try: if request.stream: - model = provider['model'][request.model] + model = model_dict[request.model] generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") @@ -603,7 +604,8 @@ def get_matching_providers(self, model_name, token): if model == "all": # 如果模型名为 *，则返回所有模型 for provider in config["providers"]: - for model in provider["model"].keys(): + model_dict = get_model_dict(provider) + for model in model_dict.keys(): provider_rules.append(provider["provider"] + "/" + model) break if "/" in model: @@ -611,15 +613,17 @@ def get_matching_providers(self, model_name, token): model = model[1:-1] # 处理带斜杠的模型名 for provider in config['providers']: - if model in provider['model'].keys(): + model_dict = get_model_dict(provider) + if model in model_dict.keys(): provider_rules.append(provider['provider'] + "/" + model) else: provider_name = model.split("/")[0] model_name_split = "/".join(model.split("/")[1:]) models_list = [] for provider in config['providers']: + model_dict = get_model_dict(provider) if provider['provider'] == provider_name: - models_list.extend(list(provider['model'].keys())) + models_list.extend(list(model_dict.keys())) # print("models_list", models_list) # print("model_name", model_name) # print("model_name_split", model_name_split) @@ -632,7 +636,8 @@ def get_matching_providers(self, model_name, token): provider_rules.append(provider_name) else: for provider in config['providers']: - if model in provider['model'].keys(): + model_dict = get_model_dict(provider) + if model in model_dict.keys(): provider_rules.append(provider['provider'] + "/" + model) provider_list = [] @@ -642,10 +647,13 @@ def get_matching_providers(self, model_name, token): # print("provider", provider, provider['provider'] == item, item) if "/" in item: if provider['provider'] == item.split("/")[0]: - if model_name in provider['model'].keys() and "/".join(item.split("/")[1:]) == model_name: + model_dict = get_model_dict(provider) + if model_name in model_dict.keys() and "/".join(item.split("/")[1:]) == model_name: provider_list.append(provider) + # 如果 item 不包含 /，则直接匹配 provider，说明整个渠道所有模型都能用 elif provider['provider'] == item: - if model_name in provider['model'].keys(): + model_dict = get_model_dict(provider) + if model_name in model_dict.keys(): provider_list.append(provider) else: pass @@ -655,7 +663,8 @@ def get_matching_providers(self, model_name, token): # if item.split("/")[1] == model_name: # provider_list.append(provider) # else: - # if model_name in provider['model'].keys(): + # model_dict = get_model_dict(provider) + # if model_name in model_dict.keys(): # provider_list.append(provider) if is_debug: for provider in provider_list: diff --git a/models.py b/models.py index fd366973..3a301c41 100644 --- a/models.py +++ b/models.py @@ -1,5 +1,5 @@ from io import IOBase -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field, model_validator, ConfigDict from typing import List, Dict, Optional, Union, Tuple, Literal, Any from log_config import logger @@ -61,10 +61,16 @@ class ToolChoice(BaseModel): class BaseRequest(BaseModel): request_type: Optional[Literal["chat", "image", "audio", "moderation"]] = Field(default=None, exclude=True) -class JsonSchema(BaseModel): - name: str - schema: Dict[str, Any] +def create_json_schema_class(): + class JsonSchema(BaseModel): + name: str + + model_config = ConfigDict(protected_namespaces=()) + + JsonSchema.__annotations__['schema'] = Dict[str, Any] + return JsonSchema +JsonSchema = create_json_schema_class() class ResponseFormat(BaseModel): type: Literal["text", "json_object", "json_schema"] json_schema: Optional[JsonSchema] = None diff --git a/request.py b/request.py index 548d669f..99748e6e 100644 --- a/request.py +++ b/request.py @@ -6,7 +6,7 @@ import urllib.parse from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, BaseAPI +from utils import c35s, c3s, c3o, c3h, gem, BaseAPI, get_model_dict, provider_api_circular_list import imghdr @@ -120,13 +120,14 @@ async def get_gemini_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] gemini_stream = "streamGenerateContent" url = provider['base_url'] if url.endswith("v1beta"): - url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api'].next()) + url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next()) if url.endswith("v1"): - url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api'].next()) + url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next()) messages = [] systemInstruction = None @@ -312,7 +313,8 @@ async def get_vertex_gemini_payload(request, engine, provider): project_id = provider.get("project_id") gemini_stream = "streamGenerateContent" - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] location = gem url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) @@ -449,7 +451,8 @@ async def get_vertex_claude_payload(request, engine, provider): if provider.get("project_id"): project_id = provider.get("project_id") - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] if "claude-3-5-sonnet" in model: location = c35s elif "claude-3-opus" in model: @@ -460,7 +463,7 @@ async def get_vertex_claude_payload(request, engine, provider): location = c3h claude_stream = "streamRawPredict" - url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/anthropic/models/{MODEL}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL=model, stream=claude_stream) + url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/anthropic/models/{MODEL}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL=model, stream=claude_stream) messages = [] system_prompt = None @@ -534,7 +537,8 @@ async def get_vertex_claude_payload(request, engine, provider): else: message_index = message_index + 1 - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "anthropic_version": "vertex-2023-10-16", "messages": messages, @@ -593,7 +597,7 @@ async def get_gpt_payload(request, engine, provider): 'Content-Type': 'application/json', } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] messages = [] @@ -633,7 +637,8 @@ async def get_gpt_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -659,7 +664,7 @@ async def get_openrouter_payload(request, engine, provider): 'Content-Type': 'application/json' } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] @@ -691,7 +696,8 @@ async def get_openrouter_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -725,7 +731,7 @@ async def get_cohere_payload(request, engine, provider): 'Content-Type': 'application/json' } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] @@ -753,7 +759,8 @@ async def get_cohere_payload(request, engine, provider): else: messages.append({"role": role_map[msg.role], "message": content}) - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] chat_history = messages[:-1] query = messages[-1].get("message") payload = { @@ -792,9 +799,10 @@ async def get_cloudflare_payload(request, engine, provider): 'Content-Type': 'application/json' } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] url = "https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model_id}".format(cf_account_id=provider['cf_account_id'], cf_model_id=model) msg = request.messages[-1] @@ -808,7 +816,7 @@ async def get_cloudflare_payload(request, engine, provider): content = msg.content name = msg.name - model = provider['model'][request.model] + model = model_dict[request.model] payload = { "prompt": content, } @@ -841,7 +849,7 @@ async def get_o1_payload(request, engine, provider): 'Content-Type': 'application/json' } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] @@ -863,7 +871,8 @@ async def get_o1_payload(request, engine, provider): elif msg.role != "system": messages.append({"role": msg.role, "content": content}) - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -912,10 +921,11 @@ async def gpt2claude_tools_json(json_dict): return json_dict async def get_claude_payload(request, engine, provider): - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] headers = { "content-type": "application/json", - "x-api-key": f"{provider['api'].next()}", + "x-api-key": f"{await provider_api_circular_list[provider['provider']].next()}", "anthropic-version": "2023-06-01", "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" if "claude-3-5-sonnet" in model else "tools-2024-05-16", } @@ -993,7 +1003,8 @@ async def get_claude_payload(request, engine, provider): else: message_index = message_index + 1 - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -1051,12 +1062,13 @@ async def get_claude_payload(request, engine, provider): return url, headers, payload async def get_dalle_payload(request, engine, provider): - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] headers = { "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] url = BaseAPI(url).image_url @@ -1070,12 +1082,13 @@ async def get_dalle_payload(request, engine, provider): return url, headers, payload async def get_whisper_payload(request, engine, provider): - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] headers = { # "Content-Type": "multipart/form-data", } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] url = BaseAPI(url).audio_transcriptions @@ -1096,12 +1109,13 @@ async def get_whisper_payload(request, engine, provider): return url, headers, payload async def get_moderation_payload(request, engine, provider): - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] headers = { "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api'].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" url = provider['base_url'] url = BaseAPI(url).moderations diff --git a/requirements.txt b/requirements.txt index 3f958dea..8fbaf43e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pyyaml +xue pytest uvicorn fastapi @@ -7,6 +7,7 @@ greenlet aiosqlite sqlalchemy watchfiles +ruamel.yaml httpx[http2] cryptography python-multipart \ No newline at end of file diff --git a/test/test_ruamel_yaml.py b/test/test_ruamel_yaml.py new file mode 100644 index 00000000..d91820c9 --- /dev/null +++ b/test/test_ruamel_yaml.py @@ -0,0 +1,44 @@ +from ruamel.yaml import YAML + +# 假设我们有以下 YAML 内容 +yaml_content = """ +# 这是顶级注释 +key1: value1 # 行尾注释 +key2: value2 + +# 这是嵌套结构的注释 +nested: + subkey1: subvalue1 + subkey2: subvalue2 # 嵌套的行尾注释 + +# 列表的注释 +list_key: + - item1 + - item2 # 列表项的注释 +""" + +# 创建 YAML 对象 +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +with open('api.yaml', 'r', encoding='utf-8') as file: + data = yaml.load(file) + +# data = yaml.load(yaml_content) +# 加载 YAML 数据 +print(data) + +# # 修改数据 +# data['key1'] = 'new_value1' +# data['nested']['subkey1'] = 'new_subvalue1' +# data['list_key'].append('new_item') + +# 将修改后的数据写回文件（这里我们使用 StringIO 来模拟文件操作） +# from io import StringIO +# output = StringIO() +# yaml.dump(data, output) +# print(output.getvalue()) + +with open('formatted.yaml', 'w', encoding='utf-8') as file: + yaml.dump(data, file) \ No newline at end of file diff --git a/test/xue/test_dropdown_sheet.py b/test/xue/test_dropdown_sheet.py new file mode 100644 index 00000000..2f65e4cf --- /dev/null +++ b/test/xue/test_dropdown_sheet.py @@ -0,0 +1,75 @@ +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue.components import dropdown, sheet, button, form, input + +xue_initialize(tailwind=True) +app = FastAPI() + +@app.get("/", response_class=HTMLResponse) +async def root(): + result = HTML( + Head( + title="Dropdown with Edit Sheet Example", + ), + Body( + Div( + dropdown.dropdown_menu("Actions"), + Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 + class_="container mx-auto p-4" + ) + ) + ).render() + print(result) + return result + +@app.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse) +async def get_dropdown_menu_content(menu_id: str): + items = [ + { + "icon": "pencil", + "label": "Edit", + "hx-get": "/edit-sheet", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + {"icon": "trash", "label": "Delete"}, + {"icon": "copy", "label": "Duplicate"}, + ] + result = dropdown.dropdown_menu_content(menu_id, items).render() + print("dropdown-menu result", result) + return result + +@app.get("/edit-sheet", response_class=HTMLResponse) +async def get_edit_sheet(): + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + form.Form( + form.FormField("Name", "name", placeholder="Enter item name"), + form.FormField("Description", "description", placeholder="Enter item description"), + Div( + button.button("Save", class_="bg-blue-500 text-white"), + button.button("Cancel", class_="bg-gray-300 text-gray-700 ml-2", data_close_sheet="true"), + class_="flex justify-end mt-4" + ), + class_="space-y-4" + ) + ) + ) + + result = sheet.Sheet( + "edit-sheet", + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/test/xue/test_form_uni_api.py b/test/xue/test_form_uni_api.py new file mode 100644 index 00000000..9aaeb832 --- /dev/null +++ b/test/xue/test_form_uni_api.py @@ -0,0 +1,116 @@ +from fastapi import FastAPI, Form as FastAPIForm +from fastapi.responses import HTMLResponse +from xue import HTML, Head, Body, Div, xue_initialize, Strong, Span, Ul, Li +from xue.components import form, button, checkbox, input +from xue.components.model_config_row import model_config_row +from typing import List, Optional +import time + +xue_initialize(tailwind=True) +app = FastAPI() + +@app.get("/", response_class=HTMLResponse) +async def root(): + result = HTML( + Head( + title="Provider Configuration Form" + ), + Body( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="password", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + model_config_row("model1", "gpt-4o: deepbricks-gpt-4o-mini", True), + model_config_row("model2", "gpt-4o"), + model_config_row("model3", "gpt-3.5-turbo"), + model_config_row("model4", "claude-3-5-sonnet-20240620: claude-3-5-sonnet"), + model_config_row("model5", "o1-mini-all"), + model_config_row("model6", "o1-preview-all"), + model_config_row("model7", "whisper-1"), + id="models-container" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=True), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", class_="bg-blue-500 text-white"), + button.button("Cancel", class_="bg-gray-300 text-gray-700 ml-2"), + class_="flex justify-end mt-4" + ), + hx_post="/submit", + hx_swap="outerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ).render() + print(result) + return result + +@app.post("/add-model", response_class=HTMLResponse) +async def add_model(): + new_model_id = f"model{hash(str(time.time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +def form_success_message(provider, base_url, api_key, models, tools_enabled, notes): + return Div( + Strong("Success!", class_="font-bold"), + Span("Form submitted successfully.", class_="block sm:inline"), + Ul( + Li(f"Provider: {provider}"), + Li(f"Base URL: {base_url}"), + Li(f"API Key: {'*' * len(api_key)}"), + Li(f"Models: {', '.join(models)}"), + Li(f"Tools Enabled: {'Yes' if tools_enabled else 'No'}"), + Li(f"Notes: {notes}"), + class_="mt-3" + ), + class_="bg-green-100 border border-green-400 text-green-700 px-4 py-3 rounded relative", + role="alert" + ) + +@app.post("/submit", response_class=HTMLResponse) +async def submit_form( + provider: str = FastAPIForm(...), + base_url: str = FastAPIForm(...), + api_key: str = FastAPIForm(...), + models: List[str] = FastAPIForm([]), + tools: Optional[str] = FastAPIForm(None), + notes: Optional[str] = FastAPIForm(None) +): + # 处理提交的数据 + print(f"Received: provider={provider}, base_url={base_url}, api_key={api_key}") + print(f"Models: {models}") + print(f"Tools Enabled: {tools is not None}") + print(f"Notes: {notes}") + + # 返回处理结果 + return form_success_message( + provider, + base_url, + api_key, + models, + tools is not None, + notes or "No notes provided" + ).render() + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/test/xue/test_home.py b/test/xue/test_home.py new file mode 100644 index 00000000..59250ebd --- /dev/null +++ b/test/xue/test_home.py @@ -0,0 +1,476 @@ +from fastapi import FastAPI, Request +from fastapi import Form as FastapiForm, HTTPException, Depends +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from fastapi.security import APIKeyHeader +from typing import Optional, List + +from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue.components.menubar import ( + Menubar, MenubarMenu, MenubarTrigger, MenubarContent, + MenubarItem, MenubarSeparator +) +from xue.components import input +from xue.components import dropdown, sheet, form, button, checkbox +from xue.components.model_config_row import model_config_row +import time + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from components.provider_table import data_table + + +from ruamel.yaml import YAML +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +xue_initialize(tailwind=True) + +from starlette.middleware.base import BaseHTTPMiddleware +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class RequestBodyLoggerMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + if request.method == "POST" and request.url.path.startswith("/submit/"): + # if request.method == "POST": + body = await request.body() + logger.info(f"Request body for {request.url.path}: {body.decode()}") + + response = await call_next(request) + return response + +from utils import load_config +from contextlib import asynccontextmanager +@asynccontextmanager +async def lifespan(app: FastAPI): + # app.state.client = httpx.AsyncClient(timeout=timeout) + app.state.config, app.state.api_keys_db, app.state.api_list = await load_config() + for item in app.state.api_keys_db: + if item.get("role") == "admin": + app.state.admin_api_key = item.get("api") + if not hasattr(app.state, "admin_api_key"): + if len(app.state.api_keys_db) >= 1: + app.state.admin_api_key = app.state.api_keys_db[0].get("api") + else: + raise Exception("No admin API key found") + + global data + # providers_data = app.state.config["providers"] + + # print("data", data) + yield + # 关闭时的代码 + await app.state.client.aclose() + +app = FastAPI(lifespan=lifespan) +# app.add_middleware(RequestBodyLoggerMiddleware) +app.add_middleware(RequestBodyLoggerMiddleware) + +data_table_columns = [ + # {"label": "Status", "value": "status", "sortable": True}, + {"label": "Provider", "value": "provider", "sortable": True}, + {"label": "Base url", "value": "base_url", "sortable": True}, + # {"label": "Engine", "value": "engine", "sortable": True}, + {"label": "Tools", "value": "tools", "sortable": True}, +] + +API_KEY_NAME = "X-API-Key" +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) + +@app.get("/login", response_class=HTMLResponse) +async def login_page(): + return HTML( + Head(title="登录"), + Body( + Div( + form.Form( + form.FormField("API Key", "x_api_key", type="password", placeholder="输入API密钥", required=True), + Div(id="error-message", class_="text-red-500 mt-2"), + Div( + button.button("提交", variant="primary", type="submit"), + class_="flex justify-end mt-4" + ), + hx_post="/verify-api-key", + hx_target="#error-message", + hx_swap="innerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-md" + ) + ) + ).render() + + +@app.post("/verify-api-key", response_class=HTMLResponse) +async def verify_api_key(x_api_key: str = FastapiForm(...)): + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + response = JSONResponse(content={"success": True}) + response.headers["HX-Redirect"] = "/" # 添加这一行 + response.set_cookie( + key="x_api_key", + value=x_api_key, + httponly=True, + max_age=1800, # 30分钟 + secure=False, # 在开发环境中设置为False，生产环境中使用HTTPS时设置为True + samesite="lax" # 改为"lax"以允许重定向时携带cookie + ) + return response + else: + return Div("无效的API密钥", class_="text-red-500").render() + +async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_key_header)): + if not x_api_key: + x_api_key = request.cookies.get("x_api_key") or request.query_params.get("x_api_key") + # print(f"Cookie x_api_key: {request.cookies.get('x_api_key')}") # 添加此行 + # print(f"Query param x_api_key: {request.query_params.get('x_api_key')}") # 添加此行 + # print(f"Header x_api_key: {x_api_key}") # 添加此行 + # logger.info(f"x_api_key: {x_api_key} {x_api_key == 'your_admin_api_key'}") + + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + return x_api_key + else: + return None + +@app.get("/", response_class=HTMLResponse) +async def root(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = HTML( + Head( + Script(""" + document.addEventListener('DOMContentLoaded', function() { + const filterInput = document.getElementById('users-table-filter'); + filterInput.addEventListener('input', function() { + const filterValue = this.value; + htmx.ajax('GET', `/filter-table?filter=${filterValue}`, '#users-table'); + }); + }); + """), + title="Menubar Example" + ), + Body( + Div( + Menubar( + MenubarMenu( + MenubarTrigger("File", "file-menu"), + MenubarContent( + MenubarItem("New Tab", shortcut="⌘T"), + MenubarItem("New Window", shortcut="⌘N"), + MenubarItem("New Incognito Window", disabled=True), + MenubarSeparator(), + MenubarItem("Print...", shortcut="⌘P"), + ), + id="file-menu" + ), + MenubarMenu( + MenubarTrigger("Edit", "edit-menu"), + MenubarContent( + MenubarItem("Undo", shortcut="⌘Z"), + MenubarItem("Redo", shortcut="⇧⌘Z"), + MenubarSeparator(), + MenubarItem("Cut"), + MenubarItem("Copy"), + MenubarItem("Paste"), + ), + id="edit-menu" + ), + MenubarMenu( + MenubarTrigger("View", "view-menu"), + MenubarContent( + MenubarItem("Always Show Bookmarks Bar"), + MenubarItem("Always Show Full URLs"), + MenubarSeparator(), + MenubarItem("Reload", shortcut="⌘R"), + MenubarItem("Force Reload", shortcut="⇧⌘R", disabled=True), + MenubarSeparator(), + MenubarItem("Toggle Fullscreen"), + MenubarItem("Hide Sidebar"), + ), + id="view-menu" + ), + ), + class_="p-4" + ), + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 + class_="container mx-auto", + id="body" + ) + ).render() + # print(result) + return result + +@app.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse) +async def get_columns_menu(menu_id: str, row_id: str): + columns = [ + { + "label": "Edit", + "value": "edit", + "hx-get": f"/edit-sheet/{row_id}", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + { + "label": "Duplicate", + "value": "duplicate", + "hx-post": f"/duplicate/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML" + }, + { + "label": "Delete", + "value": "delete", + "hx-delete": f"/delete/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML", + "hx-confirm": "确定要删除这个配置吗？" + }, + ] + result = dropdown.dropdown_menu_content(menu_id, columns).render() + print(result) + return result + +@app.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse) +async def get_columns_menu(menu_id: str): + result = dropdown.dropdown_menu_content(menu_id, data_table_columns).render() + print(result) + return result + +@app.get("/filter-table", response_class=HTMLResponse) +async def filter_table(filter: str = ""): + filtered_data = [ + provider for provider in app.state.config["providers"] + if filter.lower() in str(provider["provider"]).lower() or + filter.lower() in str(provider["base_url"]).lower() or + filter.lower() in str(provider["tools"]).lower() + ] + return data_table(data_table_columns, filtered_data, "users-table", with_filter=False).render() + +@app.post("/add-model", response_class=HTMLResponse) +async def add_model(): + new_model_id = f"model{hash(str(time.time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +@app.get("/edit-sheet/{row_id}", response_class=HTMLResponse) +async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): + row_data = get_row_data(row_id) + print("row_data", row_data) + + model_list = [] + for index, model in enumerate(row_data["model"]): + if isinstance(model, str): + model_list.append(model_config_row(f"model{index}", model, "", True)) + if isinstance(model, dict): + # print("model", model, list(model.items())[0]) + key, value = list(model.items())[0] + model_list.append(model_config_row(f"model{index}", key, value, True)) + + sheet_id = "edit-sheet" + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", value=row_data["provider"], placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", value=row_data["base_url"], placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + *model_list, + id="models-container" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=row_data["tools"], name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", value=row_data.get("notes", ""), placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), + class_="flex justify-end mt-4" + ), + hx_post=f"/submit/{row_id}", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + sheet_id, + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +@app.get("/add-provider-sheet", response_class=HTMLResponse) +async def get_add_provider_sheet(): + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Add New Provider"), + sheet.SheetDescription("Enter details for the new provider.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div(id="models-container"), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", class_="ml-2"), + class_="flex justify-end mt-4" + ), + hx_post="/submit/new", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + "add-provider-sheet", + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +def get_row_data(row_id): + index = int(row_id) + # print(app.state.config["providers"]) + return app.state.config["providers"][index] + +def update_row_data(row_id, updated_data): + print(row_id, updated_data) + index = int(row_id) + app.state.config["providers"][index] = updated_data + with open("./api1.yaml", "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + +@app.post("/submit/{row_id}", response_class=HTMLResponse) +async def submit_form( + row_id: str, + request: Request, + provider: str = FastapiForm(...), + base_url: str = FastapiForm(...), + api_key: Optional[str] = FastapiForm(None), + tools: Optional[str] = FastapiForm(None), + notes: Optional[str] = FastapiForm(None), + x_api_key: str = Depends(get_api_key) +): + form_data = await request.form() + + # 收集模型数据 + models = [] + for key, value in form_data.items(): + if key.startswith("model_name_"): + model_id = key.split("_")[-1] + enabled = form_data.get(f"model_enabled_{model_id}") == "on" + rename = form_data.get(f"model_rename_{model_id}") + if value: + if rename: + models.append({value: rename}) + else: + models.append(value) + + updated_data = { + "provider": provider, + "base_url": base_url, + "api": api_key, + "model": models, + "tools": tools == "on", + "notes": notes, + } + + print("updated_data", updated_data) + + if row_id == "new": + # 添加新提供者 + app.state.config["providers"].append(updated_data) + else: + # 更新现有提供者 + update_row_data(row_id, updated_data) + + # 保存更新后的配置 + with open("./api1.yaml", "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + + return await root() + +@app.post("/duplicate/{row_id}", response_class=HTMLResponse) +async def duplicate_row(row_id: str): + index = int(row_id) + original_data = app.state.config["providers"][index] + new_data = original_data.copy() + new_data["provider"] += "-copy" + app.state.config["providers"].insert(index + 1, new_data) + + # 保存更新后的配置 + with open("./api1.yaml", "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + + return await root() + +@app.delete("/delete/{row_id}", response_class=HTMLResponse) +async def delete_row(row_id: str): + index = int(row_id) + del app.state.config["providers"][index] + + # 保存更新后的配置 + with open("./api1.yaml", "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + + return await root() + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/utils.py b/utils.py index 620cbec2..ab683f63 100644 --- a/utils.py +++ b/utils.py @@ -3,26 +3,74 @@ import httpx from log_config import logger +from collections import defaultdict + +import asyncio + +class ThreadSafeCircularList: + def __init__(self, items): + self.items = items + self.index = 0 + self.lock = asyncio.Lock() + + async def next(self): + async with self.lock: + item = self.items[self.index] + self.index = (self.index + 1) % len(self.items) + return item + +def circular_list_encoder(obj): + if isinstance(obj, ThreadSafeCircularList): + return obj.to_dict() + raise TypeError(f'Object of type {obj.__class__.__name__} is not JSON serializable') + +provider_api_circular_list = defaultdict(ThreadSafeCircularList) + +def get_model_dict(provider): + model_dict = {} + for model in provider['model']: + if type(model) == str: + model_dict[model] = model + if type(model) == dict: + model_dict.update({new: old for old, new in model.items()}) + return model_dict + +def update_initial_model(api_url, api): + try: + endpoint = BaseAPI(api_url=api_url) + endpoint_models_url = endpoint.v1_models + response = httpx.get( + endpoint_models_url, + headers={"Authorization": f"Bearer {api}"}, + ) + models = response.json() + # print(models) + models_list = models["data"] + models_id = [model["id"] for model in models_list] + set_models = set() + for model_item in models_id: + set_models.add(model_item) + models_id = list(set_models) + # print(models_id) + return models_id + except Exception as e: + print("error:", e) + return [] def update_config(config_data): for index, provider in enumerate(config_data['providers']): - model_dict = {} - for model in provider['model']: - if type(model) == str: - model_dict[model] = model - if type(model) == dict: - model_dict.update({new: old for old, new in model.items()}) - provider['model'] = model_dict if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' if provider.get('cf_account_id'): provider['base_url'] = 'https://api.cloudflare.com/' - if provider.get('api'): - if isinstance(provider.get('api'), str): - provider['api'] = CircularList([provider.get('api')]) - if isinstance(provider.get('api'), list): - provider['api'] = CircularList(provider.get('api')) + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList([provider.get('api', None)]) + + if not provider.get("model"): + provider["model"] = update_initial_model(provider['base_url'], provider['api']) + + if provider.get("tools") == None: + provider["tools"] = True config_data['providers'][index] = provider @@ -46,31 +94,24 @@ def update_config(config_data): api_keys_db[index]['model'] = models api_list = [item["api"] for item in api_keys_db] - # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False, default=circular_list_encoder)) + # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False)) return config_data, api_keys_db, api_list # 读取YAML配置文件 async def load_config(app=None): - import yaml try: - # with open('./api.yaml', 'r') as f: - # tokens = yaml.scan(f) - # for token in tokens: - # if isinstance(token, yaml.ScalarToken): - # value = token.value - # # 如果plain为False，表示字符串被引号包裹 - # is_quoted = not token.plain - # print(f"值: {value}, 是否被引号包裹: {is_quoted}") - - with open("./api.yaml", "r", encoding="utf-8") as f: - # 判断是否为空文件 - conf = yaml.safe_load(f) - # conf = None - if conf: - config, api_keys_db, api_list = update_config(conf) - else: - # logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") - config, api_keys_db, api_list = [], [], [] + from ruamel.yaml import YAML + yaml = YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=4, offset=2) + with open('api.yaml', 'r', encoding='utf-8') as file: + conf = yaml.load(file) + + if conf: + config, api_keys_db, api_list = update_config(conf) + else: + # logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") + config, api_keys_db, api_list = [], [], [] except FileNotFoundError: logger.error("'api.yaml' not found. Please check the file path.") config, api_keys_db, api_list = [], [], [] @@ -228,7 +269,8 @@ def get_all_models(config): unique_models = set() for provider in config["providers"]: - for model in provider['model'].keys(): + model_dict = get_model_dict(provider) + for model in model_dict.keys(): if model not in unique_models: unique_models.add(model) model_info = { @@ -260,35 +302,12 @@ def get_all_models(config): # europe-west1 # europe-west4 -def circular_list_encoder(obj): - if isinstance(obj, CircularList): - return obj.to_dict() - raise TypeError(f'Object of type {obj.__class__.__name__} is not JSON serializable') - -from collections import deque -class CircularList: - def __init__(self, items): - self.queue = deque(items) - - def next(self): - if not self.queue: - return None - item = self.queue.popleft() - self.queue.append(item) - return item - - def to_dict(self): - return { - 'queue': list(self.queue) - } - - -c35s = CircularList(["us-east5", "europe-west1"]) -c3s = CircularList(["us-east5", "us-central1", "asia-southeast1"]) -c3o = CircularList(["us-east5"]) -c3h = CircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) -gem = CircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) +c35s = ThreadSafeCircularList(["us-east5", "europe-west1"]) +c3s = ThreadSafeCircularList(["us-east5", "us-central1", "asia-southeast1"]) +c3o = ThreadSafeCircularList(["us-east5"]) +c3h = ThreadSafeCircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) +gem = ThreadSafeCircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) class BaseAPI: def __init__( From 6d80128c4c0791cae3040bdabc3c74d55c0d1e6d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Oct 2024 19:09:27 +0000 Subject: [PATCH 109/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.19?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 32786aa4..44517d51 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.18 +0.0.19 From a04542e80b73a0209e50716df468c0358631072c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 13 Oct 2024 04:20:06 +0800 Subject: [PATCH 110/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20frontend=20page=20operation=20configuration=20files.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 514 ++++++++++++++++++++++++++++++++++++++++-- test/xue/test_home.py | 20 +- 2 files changed, 504 insertions(+), 30 deletions(-) diff --git a/main.py b/main.py index 5ec8f1ff..35459312 100644 --- a/main.py +++ b/main.py @@ -3,12 +3,12 @@ import re import httpx import secrets -import time as time_module +from time import time from contextlib import asynccontextmanager from starlette.middleware.base import BaseHTTPMiddleware from fastapi.middleware.cors import CORSMiddleware -from fastapi import FastAPI, HTTPException, Depends, Request +from fastapi import FastAPI, HTTPException, Depends, Request, APIRouter from fastapi.responses import JSONResponse from fastapi.responses import StreamingResponse as FastAPIStreamingResponse from starlette.responses import StreamingResponse as StarletteStreamingResponse @@ -77,6 +77,13 @@ def _get_default_sql(default): @asynccontextmanager async def lifespan(app: FastAPI): + # print("Main app routes:") + # for route in app.routes: + # print(f"Route: {route.path}, methods: {route.methods}") + + # print("\nFrontend router routes:") + # for route in frontend_router.routes: + # print(f"Route: {route.path}, methods: {route.methods}") # 启动时的代码 await create_tables() @@ -95,6 +102,16 @@ async def lifespan(app: FastAPI): ) # app.state.client = httpx.AsyncClient(timeout=timeout) app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) + + for item in app.state.api_keys_db: + if item.get("role") == "admin": + app.state.admin_api_key = item.get("api") + if not hasattr(app.state, "admin_api_key"): + if len(app.state.api_keys_db) >= 1: + app.state.admin_api_key = app.state.api_keys_db[0].get("api") + else: + raise Exception("No admin API key found") + yield # 关闭时的代码 await app.state.client.aclose() @@ -113,7 +130,6 @@ async def http_exception_handler(request: Request, exc: HTTPException): import uuid import json import asyncio -from time import time import contextvars request_info = contextvars.ContextVar('request_info', default={}) @@ -391,18 +407,19 @@ async def dispatch(self, request: Request, call_next): try: response = await call_next(request) - if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)) or type(response).__name__ == '_StreamingResponse': - response = LoggingStreamingResponse( - content=response.body_iterator, - status_code=response.status_code, - media_type=response.media_type, - headers=response.headers, - current_info=current_info, - ) - elif hasattr(response, 'json'): - logger.info(f"Response: {await response.json()}") - else: - logger.info(f"Response: type={type(response).__name__}, status_code={response.status_code}, headers={response.headers}") + if request.url.path.startswith("/v1"): + if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)) or type(response).__name__ == '_StreamingResponse': + response = LoggingStreamingResponse( + content=response.body_iterator, + status_code=response.status_code, + media_type=response.media_type, + headers=response.headers, + current_info=current_info, + ) + elif hasattr(response, 'json'): + logger.info(f"Response: {await response.json()}") + else: + logger.info(f"Response: type={type(response).__name__}, status_code={response.status_code}, headers={response.headers}") return response finally: @@ -793,7 +810,7 @@ def __init__(self): self.requests = defaultdict(list) async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: - now = time_module.time() + now = time() self.requests[key] = [req for req in self.requests[key] if req > now - period] if len(self.requests[key]) >= limit: return True @@ -910,7 +927,7 @@ async def audio_transcriptions( traceback.print_exc() raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}") -@app.get("/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) +@app.get("/v1/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) def generate_api_key(): # Define the character set (only alphanumeric) chars = string.ascii_letters + string.digits @@ -924,7 +941,7 @@ def generate_api_key(): from sqlalchemy import func, desc, case from fastapi import Query -@app.get("/stats", dependencies=[Depends(rate_limit_dependency)]) +@app.get("/v1/stats", dependencies=[Depends(rate_limit_dependency)]) async def get_stats( request: Request, token: str = Depends(verify_admin_api_key), @@ -1026,6 +1043,467 @@ async def get_stats( return JSONResponse(content=stats) + + +from fastapi import FastAPI, Request +from fastapi import Form as FastapiForm, HTTPException, Depends +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from fastapi.security import APIKeyHeader +from typing import Optional, List + +from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue.components.menubar import ( + Menubar, MenubarMenu, MenubarTrigger, MenubarContent, + MenubarItem, MenubarSeparator +) +from xue.components import input +from xue.components import dropdown, sheet, form, button, checkbox +from xue.components.model_config_row import model_config_row +# import sys +# import os +# sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from components.provider_table import data_table + +from ruamel.yaml import YAML +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + + +frontend_router = APIRouter() + +API_KEY_NAME = "X-API-Key" +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) +async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_key_header)): + if not x_api_key: + x_api_key = request.cookies.get("x_api_key") or request.query_params.get("x_api_key") + # print(f"Cookie x_api_key: {request.cookies.get('x_api_key')}") # 添加此行 + # print(f"Query param x_api_key: {request.query_params.get('x_api_key')}") # 添加此行 + # print(f"Header x_api_key: {x_api_key}") # 添加此行 + # logger.info(f"x_api_key: {x_api_key} {x_api_key == 'your_admin_api_key'}") + + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + return x_api_key + else: + return None + +async def frontend_rate_limit_dependency(request: Request, x_api_key: str = Depends(get_api_key)): + token = x_api_key if x_api_key else None + limit, period = 100, 60 + + # 使用 IP 地址和 token（如果有）作为限制键 + client_ip = request.client.host + rate_limit_key = f"{client_ip}:{token}" if token else client_ip + + if await rate_limiter.is_rate_limited(rate_limit_key, limit, period): + raise HTTPException(status_code=429, detail="Too many requests") + +# def get_backend_router_api_list(): +# api_list = [] +# for route in frontend_router.routes: +# api_list.append({ +# "path": f"/api{route.path}", # 加上前缀 +# "method": route.methods, +# "name": route.name, +# "summary": route.summary +# }) +# return api_list + +# @app.get("/backend-router-api-list") +# async def backend_router_api_list(): +# return get_backend_router_api_list() + +xue_initialize(tailwind=True) + +API_YAML_PATH = "./api.yaml" + +data_table_columns = [ + # {"label": "Status", "value": "status", "sortable": True}, + {"label": "Provider", "value": "provider", "sortable": True}, + {"label": "Base url", "value": "base_url", "sortable": True}, + # {"label": "Engine", "value": "engine", "sortable": True}, + {"label": "Tools", "value": "tools", "sortable": True}, +] + +@frontend_router.get("/login", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def login_page(): + return HTML( + Head(title="登录"), + Body( + Div( + form.Form( + form.FormField("API Key", "x_api_key", type="password", placeholder="输入API密钥", required=True), + Div(id="error-message", class_="text-red-500 mt-2"), + Div( + button.button("提交", variant="primary", type="submit"), + class_="flex justify-end mt-4" + ), + hx_post="/verify-api-key", + hx_target="#error-message", + hx_swap="innerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-md" + ) + ) + ).render() + + +@frontend_router.post("/verify-api-key", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def verify_api_key(x_api_key: str = FastapiForm(...)): + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + response = JSONResponse(content={"success": True}) + response.headers["HX-Redirect"] = "/" # 添加这一行 + response.set_cookie( + key="x_api_key", + value=x_api_key, + httponly=True, + max_age=1800, # 30分钟 + secure=False, # 在开发环境中设置为False，生产环境中使用HTTPS时设置为True + samesite="lax" # 改为"lax"以允许重定向时携带cookie + ) + return response + else: + return Div("无效的API密钥", class_="text-red-500").render() + +@frontend_router.get("/", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def root(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = HTML( + Head( + Script(""" + document.addEventListener('DOMContentLoaded', function() { + const filterInput = document.getElementById('users-table-filter'); + filterInput.addEventListener('input', function() { + const filterValue = this.value; + htmx.ajax('GET', `/filter-table?filter=${filterValue}`, '#users-table'); + }); + }); + """), + title="Menubar Example" + ), + Body( + Div( + Menubar( + MenubarMenu( + MenubarTrigger("File", "file-menu"), + MenubarContent( + MenubarItem("New Tab", shortcut="⌘T"), + MenubarItem("New Window", shortcut="⌘N"), + MenubarItem("New Incognito Window", disabled=True), + MenubarSeparator(), + MenubarItem("Print...", shortcut="⌘P"), + ), + id="file-menu" + ), + MenubarMenu( + MenubarTrigger("Edit", "edit-menu"), + MenubarContent( + MenubarItem("Undo", shortcut="⌘Z"), + MenubarItem("Redo", shortcut="⇧⌘Z"), + MenubarSeparator(), + MenubarItem("Cut"), + MenubarItem("Copy"), + MenubarItem("Paste"), + ), + id="edit-menu" + ), + MenubarMenu( + MenubarTrigger("View", "view-menu"), + MenubarContent( + MenubarItem("Always Show Bookmarks Bar"), + MenubarItem("Always Show Full URLs"), + MenubarSeparator(), + MenubarItem("Reload", shortcut="⌘R"), + MenubarItem("Force Reload", shortcut="⇧⌘R", disabled=True), + MenubarSeparator(), + MenubarItem("Toggle Fullscreen"), + MenubarItem("Hide Sidebar"), + ), + id="view-menu" + ), + ), + class_="p-4" + ), + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 + class_="container mx-auto", + id="body" + ) + ).render() + # print(result) + return result + +@frontend_router.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_columns_menu(menu_id: str, row_id: str): + columns = [ + { + "label": "Edit", + "value": "edit", + "hx-get": f"/edit-sheet/{row_id}", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + { + "label": "Duplicate", + "value": "duplicate", + "hx-post": f"/duplicate/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML" + }, + { + "label": "Delete", + "value": "delete", + "hx-delete": f"/delete/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML", + "hx-confirm": "确定要删除这个配置吗？" + }, + ] + result = dropdown.dropdown_menu_content(menu_id, columns).render() + print(result) + return result + +@frontend_router.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_columns_menu(menu_id: str): + result = dropdown.dropdown_menu_content(menu_id, data_table_columns).render() + print(result) + return result + +@frontend_router.get("/filter-table", response_class=HTMLResponse) +async def filter_table(filter: str = ""): + filtered_data = [ + provider for provider in app.state.config["providers"] + if filter.lower() in str(provider["provider"]).lower() or + filter.lower() in str(provider["base_url"]).lower() or + filter.lower() in str(provider["tools"]).lower() + ] + return data_table(data_table_columns, filtered_data, "users-table", with_filter=False).render() + +@frontend_router.post("/add-model", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def add_model(): + new_model_id = f"model{hash(str(time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +@frontend_router.get("/edit-sheet/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): + row_data = get_row_data(row_id) + print("row_data", row_data) + + model_list = [] + for index, model in enumerate(row_data["model"]): + if isinstance(model, str): + model_list.append(model_config_row(f"model{index}", model, "", True)) + if isinstance(model, dict): + # print("model", model, list(model.items())[0]) + key, value = list(model.items())[0] + model_list.append(model_config_row(f"model{index}", key, value, True)) + + sheet_id = "edit-sheet" + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", value=row_data["provider"], placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", value=row_data["base_url"], placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + *model_list, + id="models-container" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=row_data["tools"], name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", value=row_data.get("notes", ""), placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), + class_="flex justify-end mt-4" + ), + hx_post=f"/submit/{row_id}", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + sheet_id, + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +@frontend_router.get("/add-provider-sheet", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_add_provider_sheet(): + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Add New Provider"), + sheet.SheetDescription("Enter details for the new provider.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div(id="models-container"), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", class_="ml-2"), + class_="flex justify-end mt-4" + ), + hx_post="/submit/new", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + "add-provider-sheet", + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +def get_row_data(row_id): + index = int(row_id) + # print(app.state.config["providers"]) + return app.state.config["providers"][index] + +def update_row_data(row_id, updated_data): + print(row_id, updated_data) + index = int(row_id) + app.state.config["providers"][index] = updated_data + save_api_yaml() + +def save_api_yaml(): + with open(API_YAML_PATH, "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + +@frontend_router.post("/submit/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def submit_form( + row_id: str, + request: Request, + provider: str = FastapiForm(...), + base_url: str = FastapiForm(...), + api_key: Optional[str] = FastapiForm(None), + tools: Optional[str] = FastapiForm(None), + notes: Optional[str] = FastapiForm(None), + x_api_key: str = Depends(get_api_key) +): + form_data = await request.form() + + # 收集模型数据 + models = [] + for key, value in form_data.items(): + if key.startswith("model_name_"): + model_id = key.split("_")[-1] + enabled = form_data.get(f"model_enabled_{model_id}") == "on" + rename = form_data.get(f"model_rename_{model_id}") + if value: + if rename: + models.append({value: rename}) + else: + models.append(value) + + updated_data = { + "provider": provider, + "base_url": base_url, + "api": api_key, + "model": models, + "tools": tools == "on", + "notes": notes, + } + + print("updated_data", updated_data) + + if row_id == "new": + # 添加新提供者 + app.state.config["providers"].append(updated_data) + else: + # 更新现有提供者 + update_row_data(row_id, updated_data) + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +@frontend_router.post("/duplicate/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def duplicate_row(row_id: str): + index = int(row_id) + original_data = app.state.config["providers"][index] + new_data = original_data.copy() + new_data["provider"] += "-copy" + app.state.config["providers"].insert(index + 1, new_data) + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +@frontend_router.delete("/delete/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def delete_row(row_id: str): + index = int(row_id) + del app.state.config["providers"][index] + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +app.include_router(frontend_router, tags=["frontend"]) + # async def on_fetch(request, env): # import asgi # return await asgi.fetch(app, request, env) diff --git a/test/xue/test_home.py b/test/xue/test_home.py index 59250ebd..231280a8 100644 --- a/test/xue/test_home.py +++ b/test/xue/test_home.py @@ -33,6 +33,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +API_YAML_PATH = "./api.yaml" class RequestBodyLoggerMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): if request.method == "POST" and request.url.path.startswith("/submit/"): @@ -47,7 +48,6 @@ async def dispatch(self, request: Request, call_next): from contextlib import asynccontextmanager @asynccontextmanager async def lifespan(app: FastAPI): - # app.state.client = httpx.AsyncClient(timeout=timeout) app.state.config, app.state.api_keys_db, app.state.api_list = await load_config() for item in app.state.api_keys_db: if item.get("role") == "admin": @@ -58,10 +58,6 @@ async def lifespan(app: FastAPI): else: raise Exception("No admin API key found") - global data - # providers_data = app.state.config["providers"] - - # print("data", data) yield # 关闭时的代码 await app.state.client.aclose() @@ -393,7 +389,10 @@ def update_row_data(row_id, updated_data): print(row_id, updated_data) index = int(row_id) app.state.config["providers"][index] = updated_data - with open("./api1.yaml", "w", encoding="utf-8") as f: + save_api_yaml() + +def save_api_yaml(): + with open(API_YAML_PATH, "w", encoding="utf-8") as f: yaml.dump(app.state.config, f) @app.post("/submit/{row_id}", response_class=HTMLResponse) @@ -441,8 +440,7 @@ async def submit_form( update_row_data(row_id, updated_data) # 保存更新后的配置 - with open("./api1.yaml", "w", encoding="utf-8") as f: - yaml.dump(app.state.config, f) + save_api_yaml() return await root() @@ -455,8 +453,7 @@ async def duplicate_row(row_id: str): app.state.config["providers"].insert(index + 1, new_data) # 保存更新后的配置 - with open("./api1.yaml", "w", encoding="utf-8") as f: - yaml.dump(app.state.config, f) + save_api_yaml() return await root() @@ -466,8 +463,7 @@ async def delete_row(row_id: str): del app.state.config["providers"][index] # 保存更新后的配置 - with open("./api1.yaml", "w", encoding="utf-8") as f: - yaml.dump(app.state.config, f) + save_api_yaml() return await root() From 1e9a2ac89d1cb1833ea6c2c6d3c3cb1083e6a5e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Oct 2024 20:20:26 +0000 Subject: [PATCH 111/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 44517d51..fe04e7f6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.19 +0.0.20 From 60e30aa1a967b4e7005b4aef43cdcadac1729b5b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 13 Oct 2024 04:33:11 +0800 Subject: [PATCH 112/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20/v1/models=20does=20not=20work.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index ab683f63..3b942e70 100644 --- a/utils.py +++ b/utils.py @@ -225,7 +225,8 @@ def post_all_models(token, config, api_list): for provider_item in config["providers"]: if provider_item['provider'] != provider: continue - for model_item in provider_item['model'].keys(): + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys(): if model_item not in unique_models: unique_models.add(model_item) model_info = { @@ -240,7 +241,8 @@ def post_all_models(token, config, api_list): for provider_item in config["providers"]: if provider_item['provider'] != provider: continue - for model_item in provider_item['model'].keys() : + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys() : if model_item not in unique_models and model_item == model: unique_models.add(model_item) model_info = { From 18084682596016814e228d9b75238429b93660d9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Oct 2024 20:33:38 +0000 Subject: [PATCH 113/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.21?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fe04e7f6..236c7ad0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.20 +0.0.21 From 859400e940647e0f50170a4950fd114b6d348106 Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Sun, 13 Oct 2024 17:37:36 +0800 Subject: [PATCH 114/476] fix: api could be a list --- utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils.py b/utils.py index 3b942e70..b7b67131 100644 --- a/utils.py +++ b/utils.py @@ -39,6 +39,8 @@ def update_initial_model(api_url, api): try: endpoint = BaseAPI(api_url=api_url) endpoint_models_url = endpoint.v1_models + if isinstance(api, list): + api = api[0] response = httpx.get( endpoint_models_url, headers={"Authorization": f"Bearer {api}"}, @@ -55,6 +57,8 @@ def update_initial_model(api_url, api): return models_id except Exception as e: print("error:", e) + import traceback + traceback.print_exc() return [] def update_config(config_data): From 19a2bc12d7951a03b777b1a4270b60bc3daa8b41 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 13 Oct 2024 09:45:19 +0000 Subject: [PATCH 115/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.22?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 236c7ad0..818944f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.21 +0.0.22 From 00ec2167442c3ff82aa49298388b6233232de797 Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Sun, 13 Oct 2024 17:58:37 +0800 Subject: [PATCH 116/476] if api key is invalid, raise except --- utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.py b/utils.py index b7b67131..e024b495 100644 --- a/utils.py +++ b/utils.py @@ -46,6 +46,8 @@ def update_initial_model(api_url, api): headers={"Authorization": f"Bearer {api}"}, ) models = response.json() + if models.get("error"): + raise Exception({"error": models.get("error"), "endpoint": endpoint_models_url, "api": api}) # print(models) models_list = models["data"] models_id = [model["id"] for model in models_list] From 3d5dca97687dbfe9b61c3a8ee4c98fea6eb3db0c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 13 Oct 2024 10:03:36 +0000 Subject: [PATCH 117/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.23?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 818944f5..df5db66f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.22 +0.0.23 From a3a84d22e9ddf1a14db147d9ff9dcbab62c3ae01 Mon Sep 17 00:00:00 2001 From: Benedict King <2107330+BenedictKing@users.noreply.github.com> Date: Sun, 13 Oct 2024 20:44:19 +0800 Subject: [PATCH 118/476] fix: create ThreadSafeCircularList with correct api list --- utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index e024b495..3401292f 100644 --- a/utils.py +++ b/utils.py @@ -70,7 +70,12 @@ def update_config(config_data): if provider.get('cf_account_id'): provider['base_url'] = 'https://api.cloudflare.com/' - provider_api_circular_list[provider['provider']] = ThreadSafeCircularList([provider.get('api', None)]) + provider_api = provider.get('api', None) + if provider_api: + if isinstance(provider_api, str): + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList([provider_api]) + if isinstance(provider_api, list): + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList(provider_api) if not provider.get("model"): provider["model"] = update_initial_model(provider['base_url'], provider['api']) From 1784de10320693d8922a5734ae105bf1e1d70b7f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 13 Oct 2024 19:14:56 +0000 Subject: [PATCH 119/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.24?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index df5db66f..b056f412 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.23 +0.0.24 From 5ce094b1c3ef2bc61a0d761fb5614298905b5354 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 14 Oct 2024 03:48:44 +0800 Subject: [PATCH 120/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20remote=20configuration=20file=20cannot=20be=20?= =?UTF-8?q?parsed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++- utils.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index bcdffe78..d6dcf7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ node_modules *.png *.db .aider* -.idea \ No newline at end of file +.idea +docker-compose-test.yml \ No newline at end of file diff --git a/utils.py b/utils.py index 3401292f..ada1a8c8 100644 --- a/utils.py +++ b/utils.py @@ -58,7 +58,7 @@ def update_initial_model(api_url, api): # print(models_id) return models_id except Exception as e: - print("error:", e) + # print("error:", e) import traceback traceback.print_exc() return [] @@ -110,11 +110,11 @@ def update_config(config_data): # 读取YAML配置文件 async def load_config(app=None): + from ruamel.yaml import YAML + yaml = YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=4, offset=2) try: - from ruamel.yaml import YAML - yaml = YAML() - yaml.preserve_quotes = True - yaml.indent(mapping=2, sequence=4, offset=2) with open('api.yaml', 'r', encoding='utf-8') as file: conf = yaml.load(file) @@ -144,7 +144,7 @@ async def load_config(app=None): response = await app.state.client.get(config_url) # logger.info(f"Fetching config from {response.text}") response.raise_for_status() - config_data = yaml.safe_load(response.text) + config_data = yaml.load(response.text) # 更新配置 # logger.info(config_data) if config_data: From 026c519606874f2a2927da2748c7d8707d18646e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 13 Oct 2024 19:49:12 +0000 Subject: [PATCH 121/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.25?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b056f412..2678ff8d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.24 +0.0.25 From 1262e5b158b7b0932d31399ac5efab836e110987 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 14 Oct 2024 17:11:27 +0800 Subject: [PATCH 122/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20/v1/models=20endpoint=20returns=20incomplete?= =?UTF-8?q?=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index ada1a8c8..de66c0d3 100644 --- a/utils.py +++ b/utils.py @@ -31,7 +31,7 @@ def get_model_dict(provider): for model in provider['model']: if type(model) == str: model_dict[model] = model - if type(model) == dict: + if isinstance(model, dict): model_dict.update({new: old for old, new in model.items()}) return model_dict From eb0ff00c8b8eb016d9e3d5c5a2ac0ffd12b43a3d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Oct 2024 09:11:45 +0000 Subject: [PATCH 123/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.26?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2678ff8d..c4475d3b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.25 +0.0.26 From 776255ba28945bc5b1bf41d4ff7a519203ead4e9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 14 Oct 2024 21:22:02 +0800 Subject: [PATCH 124/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20sheet=20cannot=20be=20closed=20when=20clicking?= =?UTF-8?q?=20the=20cancel=20button=20without=20filling=20in=20the=20conte?= =?UTF-8?q?nt=20on=20the=20frontend=20add=20model=20sheet.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 35459312..299a79a5 100644 --- a/main.py +++ b/main.py @@ -1363,6 +1363,7 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): @frontend_router.get("/add-provider-sheet", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def get_add_provider_sheet(): + sheet_id = "add-provider-sheet" edit_sheet_content = sheet.SheetContent( sheet.SheetHeader( sheet.SheetTitle("Add New Provider"), @@ -1393,7 +1394,7 @@ async def get_add_provider_sheet(): form.FormField("Notes", "notes", placeholder="Enter any additional notes"), Div( button.button("Submit", variant="primary", type="submit"), - button.button("Cancel", variant="outline", class_="ml-2"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), class_="flex justify-end mt-4" ), hx_post="/submit/new", @@ -1407,7 +1408,7 @@ async def get_add_provider_sheet(): ) result = sheet.Sheet( - "add-provider-sheet", + sheet_id, Div(), edit_sheet_content, width="80%", From 06afdf81a944a8ff7090f6c8d1a83a4f76531c9b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Oct 2024 13:22:24 +0000 Subject: [PATCH 125/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.27?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c4475d3b..24ff8558 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.26 +0.0.27 From 80fec79e5b0066f6c64cc97e6f9fea8c4f4ba7d4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 14 Oct 2024 21:58:38 +0800 Subject: [PATCH 126/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20after=20filtering=20through=20the=20filter,=20if=20t?= =?UTF-8?q?here=20are=20a=20total=20of=20four=20rows,=20when=20clicking=20?= =?UTF-8?q?the=20edit=20configuration=20on=20the=20fourth=20row,=20the=20s?= =?UTF-8?q?heet=20still=20displays=20the=20data=20from=20the=20fourth=20ro?= =?UTF-8?q?w=20before=20the=20filter=20was=20applied.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/provider_table.py | 23 ++++++++++++++--------- main.py | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/components/provider_table.py b/components/provider_table.py index f858d72b..5e933c0e 100644 --- a/components/provider_table.py +++ b/components/provider_table.py @@ -143,7 +143,19 @@ """, id="data-table-script"), ]) -def data_table(columns, data, id, with_filter=True): +def data_table(columns, data, id, with_filter=True, row_ids=None): + if row_ids is None: + row_ids = range(len(data)) + + tbody_content = Tbody( + *[Tr( + Td(checkbox(f"row-{i}", "", class_="row-checkbox")), + *[Td(row[col['value']], data_accessor=col['value']) for col in columns], + Td(row_actions_menu(row_id)), + id=f"row-{row_id}" + ) for i, (row, row_id) in enumerate(zip(data, row_ids))] + ) + return Div( Div( input(type="text", placeholder="Filter...", id=f"{id}-filter", class_="mr-auto"), @@ -178,14 +190,7 @@ def data_table(columns, data, id, with_filter=True): Th("Actions") # 新增的操作列 ) ), - Tbody( - *[Tr( - Td(checkbox(f"row-{i}", "", class_="row-checkbox")), - *[Td(row[col['value']], data_accessor=col['value']) for col in columns], - Td(row_actions_menu(i)), # 使用行索引作为 row_id - id=f"row-{i}" - ) for i, row in enumerate(data)] - ), + tbody_content, class_="data-table" ), class_="data-table-container" diff --git a/main.py b/main.py index 299a79a5..2f6fa453 100644 --- a/main.py +++ b/main.py @@ -1278,12 +1278,12 @@ async def get_columns_menu(menu_id: str): @frontend_router.get("/filter-table", response_class=HTMLResponse) async def filter_table(filter: str = ""): filtered_data = [ - provider for provider in app.state.config["providers"] + (i, provider) for i, provider in enumerate(app.state.config["providers"]) if filter.lower() in str(provider["provider"]).lower() or filter.lower() in str(provider["base_url"]).lower() or filter.lower() in str(provider["tools"]).lower() ] - return data_table(data_table_columns, filtered_data, "users-table", with_filter=False).render() + return data_table(data_table_columns, [p for _, p in filtered_data], "users-table", with_filter=False, row_ids=[i for i, _ in filtered_data]).render() @frontend_router.post("/add-model", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def add_model(): From 5596e070887705470131d06149b35675e8e1e62d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Oct 2024 13:59:17 +0000 Subject: [PATCH 127/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 24ff8558..1fe69585 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.27 +0.0.28 From 390708442c36252b07963a622632350ef66c47c5 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 15 Oct 2024 00:27:53 +0800 Subject: [PATCH 128/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20sheet=20page=20cannot=20fully=20display=20when?= =?UTF-8?q?=20there=20are=20too=20many=20models.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 8 +++++--- utils.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 2f6fa453..6daf6ec3 100644 --- a/main.py +++ b/main.py @@ -1321,7 +1321,8 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): Div("Models", class_="text-lg font-semibold mb-2"), Div( *model_list, - id="models-container" + id="models-container", + class_="space-y-2 max-h-[40vh] overflow-y-auto" ), button.button( "Add Model", @@ -1349,7 +1350,8 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): ), class_="container mx-auto p-4 max-w-2xl" ) - ) + ), + class_="max-h-[90vh] overflow-y-auto" ) result = sheet.Sheet( @@ -1429,7 +1431,7 @@ def update_row_data(row_id, updated_data): def save_api_yaml(): with open(API_YAML_PATH, "w", encoding="utf-8") as f: - yaml.dump(app.state.config, f) + yaml.round_trip_dump(app.state.config, f) @frontend_router.post("/submit/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def submit_form( diff --git a/utils.py b/utils.py index de66c0d3..6e62a402 100644 --- a/utils.py +++ b/utils.py @@ -110,7 +110,7 @@ def update_config(config_data): # 读取YAML配置文件 async def load_config(app=None): - from ruamel.yaml import YAML + from ruamel.yaml import YAML, YAMLError yaml = YAML() yaml.preserve_quotes = True yaml.indent(mapping=2, sequence=4, offset=2) @@ -126,8 +126,8 @@ async def load_config(app=None): except FileNotFoundError: logger.error("'api.yaml' not found. Please check the file path.") config, api_keys_db, api_list = [], [], [] - except yaml.YAMLError: - logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。") + except YAMLError as e: + logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。%s", e) config, api_keys_db, api_list = [], [], [] except OSError as e: logger.error(f"open 'api.yaml' failed: {e}") From 89394eed4c5bb81d3cf2c11085df2ac77e1a1c15 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Oct 2024 16:28:33 +0000 Subject: [PATCH 129/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.29?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1fe69585..369bd4c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.28 +0.0.29 From 93f77d4ef3ad324a883ddf2d06cc3bcca338885d Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 15 Oct 2024 00:36:51 +0800 Subject: [PATCH 130/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20an=20"imag?= =?UTF-8?q?e"=20field=20to=20the=20configuration=20file=20to=20customize?= =?UTF-8?q?=20control=20over=20whether=20to=20enable=20image=20reading=20s?= =?UTF-8?q?upport.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/request.py b/request.py index 99748e6e..a2f371e4 100644 --- a/request.py +++ b/request.py @@ -142,7 +142,7 @@ async def get_gemini_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -331,7 +331,7 @@ async def get_vertex_gemini_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -477,7 +477,7 @@ async def get_vertex_claude_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -610,7 +610,7 @@ async def get_gpt_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -677,7 +677,7 @@ async def get_openrouter_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -943,7 +943,7 @@ async def get_claude_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: From 038271bd34c1d1ad937905599593397fa4698a07 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Oct 2024 16:37:15 +0000 Subject: [PATCH 131/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.30?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 369bd4c2..f092e2be 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.29 +0.0.30 From fd7bf5e1e20b68833492dc71ac71db37a2849d2a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 16 Oct 2024 04:15:01 +0800 Subject: [PATCH 132/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20front-end=20sheet=20table=20can=20only=20displ?= =?UTF-8?q?ay=20one=20key=20when=20there=20are=20multiple=20API=20keys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 6daf6ec3..dcadffc5 100644 --- a/main.py +++ b/main.py @@ -1051,7 +1051,7 @@ async def get_stats( from fastapi.security import APIKeyHeader from typing import Optional, List -from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue import HTML, Head, Body, Div, xue_initialize, Script, Ul, Li from xue.components.menubar import ( Menubar, MenubarMenu, MenubarTrigger, MenubarContent, MenubarItem, MenubarSeparator @@ -1069,7 +1069,6 @@ async def get_stats( yaml.preserve_quotes = True yaml.indent(mapping=2, sequence=4, offset=2) - frontend_router = APIRouter() API_KEY_NAME = "X-API-Key" @@ -1262,7 +1261,7 @@ async def get_columns_menu(menu_id: str, row_id: str): "hx-delete": f"/delete/{row_id}", "hx-target": "body", "hx-swap": "outerHTML", - "hx-confirm": "确定要删除这个配置吗？" + "hx-confirm": "Are you sure you want to delete this configuration?" }, ] result = dropdown.dropdown_menu_content(menu_id, columns).render() @@ -1291,6 +1290,36 @@ async def add_model(): new_model = model_config_row(new_model_id).render() return new_model +def render_api_keys(row_id, api_keys): + return Ul( + *[Li( + Div( + Div( + input.input( + type="text", + placeholder="Enter API key", + value=api_key, + name=f"api_key_{i}", + class_="flex-grow w-full" + ), + class_="flex-grow" + ), + button.button( + "Delete", + variant="outline", + type="button", + class_="ml-2", + hx_delete=f"/delete-api-key/{row_id}/{i}", + hx_target="#api-keys-container", + hx_swap="outerHTML" + ), + class_="flex items-center mb-2 w-full" + ) + ) for i, api_key in enumerate(api_keys)], + id="api-keys-container", + class_="space-y-2 w-full" + ) + @frontend_router.get("/edit-sheet/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): row_data = get_row_data(row_id) @@ -1305,6 +1334,10 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): key, value = list(model.items())[0] model_list.append(model_config_row(f"model{index}", key, value, True)) + # 处理多个 API keys + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + api_key_inputs = render_api_keys(row_id, api_keys) + sheet_id = "edit-sheet" edit_sheet_content = sheet.SheetContent( sheet.SheetHeader( @@ -1316,7 +1349,19 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): form.Form( form.FormField("Provider", "provider", value=row_data["provider"], placeholder="Enter provider name", required=True), form.FormField("Base URL", "base_url", value=row_data["base_url"], placeholder="Enter base URL", required=True), - form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + # form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + Div( + Div("API Keys", class_="text-lg font-semibold mb-2"), + api_key_inputs, + button.button( + "Add API Key", + class_="mt-2", + hx_post=f"/add-api-key/{row_id}", + hx_target="#api-keys-container", + hx_swap="outerHTML" + ), + class_="mb-4" + ), Div( Div("Models", class_="text-lg font-semibold mb-2"), Div( @@ -1363,6 +1408,27 @@ async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): ).render() return result +@frontend_router.post("/add-api-key/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def add_api_key(row_id: str): + row_data = get_row_data(row_id) + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + api_keys.append("") # 添加一个空的API key + + api_key_inputs = render_api_keys(row_id, api_keys) + + return api_key_inputs.render() + +@frontend_router.delete("/delete-api-key/{row_id}/{index}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def delete_api_key(row_id: str, index: int): + row_data = get_row_data(row_id) + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + if len(api_keys) > 1: + del api_keys[index] + + api_key_inputs = render_api_keys(row_id, api_keys) + + return api_key_inputs.render() + @frontend_router.get("/add-provider-sheet", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def get_add_provider_sheet(): sheet_id = "add-provider-sheet" @@ -1427,11 +1493,10 @@ def update_row_data(row_id, updated_data): print(row_id, updated_data) index = int(row_id) app.state.config["providers"][index] = updated_data - save_api_yaml() def save_api_yaml(): with open(API_YAML_PATH, "w", encoding="utf-8") as f: - yaml.round_trip_dump(app.state.config, f) + yaml.dump(app.state.config, f) @frontend_router.post("/submit/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def submit_form( @@ -1439,13 +1504,15 @@ async def submit_form( request: Request, provider: str = FastapiForm(...), base_url: str = FastapiForm(...), - api_key: Optional[str] = FastapiForm(None), + # api_key: Optional[str] = FastapiForm(None), tools: Optional[str] = FastapiForm(None), notes: Optional[str] = FastapiForm(None), x_api_key: str = Depends(get_api_key) ): form_data = await request.form() + api_keys = [value for key, value in form_data.items() if key.startswith("api_key_") and value] + # 收集模型数据 models = [] for key, value in form_data.items(): @@ -1462,7 +1529,7 @@ async def submit_form( updated_data = { "provider": provider, "base_url": base_url, - "api": api_key, + "api": api_keys[0] if len(api_keys) == 1 else api_keys, # 如果只有一个 API key，就不使用列表 "model": models, "tools": tools == "on", "notes": notes, From 212407ab1b37433d33c4595cc55209b3d2823ac7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Oct 2024 20:15:20 +0000 Subject: [PATCH 133/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.31?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f092e2be..d788d433 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.30 +0.0.31 From ee08a6f3e5883e8d97e96c11fa522b7631e9161c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 19 Oct 2024 05:17:53 +0800 Subject: [PATCH 134/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Added=20lottery?= =?UTF-8?q?=20scheduling=20algorithm=20and=20support=20for=20random=20sche?= =?UTF-8?q?duling=20algorithm.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation 💻 Code: Refactored the code. --- README.md | 176 +++++++++++++++++++++++++++++---------------------- README_CN.md | 66 +++++++++++++------ main.py | 73 ++++++++++++--------- utils.py | 3 +- 4 files changed, 193 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 60e7104c..17574152 100644 --- a/README.md +++ b/README.md @@ -13,62 +13,83 @@ ## Introduction -If used personally, one/new-api is too complex and has many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project that unifies the management of large model APIs, allowing multiple backend services to be called through a unified API interface and uniformly converted to the OpenAI format, supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, DeepBricks, OpenRouter, etc. - -## Features - -- No frontend, pure configuration file setup for API channels. You can run your own API site by just writing one file, with detailed configuration guides in the documentation, beginner-friendly. -- Unified management of multiple backend services, supporting providers like OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in the OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex AI, Cohere, Groq, Cloudflare. Vertex supports both Claude and Gemini APIs. -- Supports OpenAI, Anthropic, Gemini, Vertex native tool use function calls. -- Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. -- Supports four types of load balancing. - 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. Disabled by default, requires channel weight configuration. - 2. Supports Vertex regional load balancing, supports Vertex high concurrency, and can increase Gemini, Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. - 3. In addition to Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. +For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, DeepBricks, OpenRouter, and more. + +## ✨ Features + +- No front-end, pure configuration file to configure API channels. You can run your own API station just by writing a file, and the documentation has a detailed configuration guide, beginner-friendly. +- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. +- Simultaneously supports Anthropic, Gemini, Vertex AI, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. +- Support OpenAI, Anthropic, Gemini, Vertex native tool use function calls. +- Support OpenAI, Anthropic, Gemini, Vertex native image recognition API. +- Support four types of load balancing. + 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. + 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. -- Supports automatic retry, when an API channel response fails, automatically retry the next API channel. -- Supports fine-grained access control. Supports using wildcards to set specific models for API key available channels. -- Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. +- Support automatic retry, when an API channel response fails, automatically retry the next API channel. +- Support fine-grained permission control. Support using wildcards to set specific models available for API key channels. +- Support rate limiting, you can set the maximum number of requests per minute as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. - Supports multiple standard OpenAI format interfaces: `/v1/chat/completions`, `/v1/images/generations`, `/v1/audio/transcriptions`, `/v1/moderations`, `/v1/models`. -- Supports OpenAI moderation for ethical review, allowing for ethical review of user messages. If inappropriate messages are detected, an error message will be returned. This reduces the risk of the backend API being banned by providers. +- Support OpenAI moderation moral review, which can conduct moral reviews of user messages. If inappropriate messages are found, an error message will be returned. This reduces the risk of the backend API being banned by providers. -## Configuration +## Usage method -Using the api.yaml configuration file, multiple models can be configured, and each model can be configured with multiple backend services, supporting load balancing. Below is an example of the api.yaml configuration file: +To start uni-api, a configuration file must be used. There are two ways to start with a configuration file: + +1. The first method is to use the `CONFIG_URL` environment variable to fill in the configuration file URL, which will be automatically downloaded when uni-api starts. +2. The second method is to mount a configuration file named `api.yaml` into the container. + +### Method 1: Mount the `api.yaml` configuration file to start uni-api + +You must fill in the configuration file in advance to start `uni-api`, and you must use a configuration file named `api.yaml` to start `uni-api`, you can configure multiple models, each model can configure multiple backend services, and support load balancing. Below is an example of the minimum `api.yaml` configuration file that can be run: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name is fine, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required + base_url: https://api.your.com/v1/chat/completions # Backend service API address, required + api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required, automatically uses base_url and api to get all available models through the /v1/models endpoint. + # Multiple providers can be configured here, each provider can have multiple API Keys, and each API Key can have multiple models configured. +api_keys: + - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key, required for user requests to uni-api + model: # Models that can be used by this API Key, required. Channel-level round-robin load balancing is enabled by default, and each request to the model follows the order configured in model. It is independent of the original channel order in providers. Therefore, you can set a different request order for each API key. + - all # Can use all models from all channels set under providers, no need to add available channels one by one. If you don't want to set available channels for each api in api_keys, uni-api supports setting the api key to use all models from all channels under providers. +``` + +Detailed advanced configuration of `api.yaml`: + +```yaml +providers: + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # At least one model must be filled in + model: # Optional, if model is not configured, all available models will be automatically retrieved through base_url and api via the /v1/models endpoint. - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a concise name instead of the original complex name, optional - tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional + tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the following line to use the original name. + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the following line to use the original name - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, you can also view the service account details in the "IAM & Admin" section of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON-formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON-formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, or can be obtained by viewing service account details in the "IAM & Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -77,14 +98,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # Can include the provider's website, notes, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, the model name must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a concise name instead of the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # The model name must be enclosed in quotes, otherwise yaml syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes to avoid YAML syntax error, llama-3.1-8b is the renamed name, you can use a simpler name instead of the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes to avoid YAML syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -93,11 +114,11 @@ providers: - causallm-35b-beta2ep-q6k: causallm-35b - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # Force to use a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional + engine: openrouter # Force use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required - model: # Models that this API Key can use, required + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service + model: # The models that this API Key can use, required. Channel-level round-robin load balancing is enabled by default, and each request model is requested in the order configured in the model. It is unrelated to the original channel order in providers. Therefore, you can set different request orders for each API key. - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models @@ -105,68 +126,76 @@ api_keys: - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Other providers' claude-3-5-sonnet models cannot be used. This way of writing will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will treat the entire anthropic/claude-3-5-sonnet as the model name. This way of writing can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. - - openai-test/text-moderation-latest # When message moderation is enabled, you can use the text-moderation-latest model under the channel named openai-test for moderation. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models named claude-3-5-sonnet from other providers cannot be used. This notation will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets around the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but instead treat the entire anthropic/claude-3-5-sonnet as the model name. This notation can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for message moderation. preferences: - USE_ROUND_ROBIN: true # Whether to use polling load balancing, true to use, false to not use, default is true. When polling is enabled, each request will be made in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request orders for each API key. - AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true - RATE_LIMIT: 2/min # Supports rate limiting, the maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional - ENABLE_MODERATION: true # Whether to enable message moderation, true to enable, false to not enable, default is false. When enabled, it will conduct moderation on the user's message, if inappropriate messages are found, it will return an error message. + SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, fixed priority scheduling is used, always executing the channel of the first model with a request. Modify the default channel round-robin load balancing. SCHEDULING_ALGORITHM options are: fixed_priority, weighted_round_robin, lottery, random. + # When SCHEDULING_ALGORITHM is random, random round-robin load balancing is used, randomly requesting the channel of the model with a request. + AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true + RATE_LIMIT: 2/min # Supports rate limiting, the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + ENABLE_MODERATION: true # Whether to enable message moderation, true to enable, false to disable, default is false, when enabled, user messages will be moderated, and if inappropriate messages are found, an error message will be returned. # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, the weight only supports positive integers. + - gcp1/*: 5 # The number after the colon is the weight, weight only supports positive integers. - gcp2/*: 3 # The larger the number, the greater the probability of the request. - - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp3/*: 2 # In this example, there are a total of 10 weights across all channels, and out of 10 requests, 5 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, it will request in the original channel order, if there is weight, it will request in the weighted order. + SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and if the above channels have weights, requests will be made according to the weighted order. Use weighted round-robin load balancing, request the channel of the model with a request according to the weight order. When SCHEDULING_ALGORITHM is lottery, use lottery round-robin load balancing, request the channel of the model with a request according to the weight randomly. AUTO_RETRY: true ``` -If you do not want to set available channels for each `api` one by one in `api_keys`, `uni-api` supports setting the `api key` to be able to use all models. The configuration is as follows: +Mount the configuration file and start the uni-api docker container: -```yaml -# ... providers configuration unchanged ... -api_keys: - - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to request uni-api, required - model: # The model that can be used with this API Key, required - - all # Can use all models in all channels set under providers, no need to add available channels one by one. -# ... other configurations unchanged ... +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +``` + +### Method two: Start uni-api using the `CONFIG_URL` environment variable + +After writing the configuration file according to method one, upload it to the cloud disk, get the file's direct link, and then use the `CONFIG_URL` environment variable to start the uni-api docker container: + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-e CONFIG_URL=http://file_url/api.yaml \ +yym68686/uni-api:latest ``` -## Environment Variables +## Environment variable -- CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 100 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional +- CONFIG_URL: The download address of the configuration file, which can be a local file or a remote file, optional +- TIMEOUT: Request timeout, default is 100 seconds. The timeout can control the time needed to switch to the next channel when one channel does not respond. Optional -## Retrieve Statistical Data +## Get statistical data -Use `/stats` to get usage statistics for each channel over the last 24 hours. Include your own uni-api admin API key. +Use `/stats` to get the usage statistics of each channel for the past 24 hours. Also include your uni-api admin API key. -The data includes: +Data includes: -1. Success rate for each model under each channel, sorted from highest to lowest success rate. -2. Overall success rate for each channel, sorted from highest to lowest. -3. Total number of requests for each model across all channels. -4. Number of requests for each endpoint. -5. Number of requests from each IP address. +1. The success rate of each model under each channel, sorted from high to low. +2. The overall success rate of each channel, sorted from high to low. +3. The total number of requests for each model across all channels. +4. The number of requests for each endpoint. +5. The number of requests per IP. -`/stats?hours=48` The `hours` parameter can control how many hours of recent data statistics are returned. If the `hours` parameter is not provided, it defaults to statistics for the last 24 hours. +The `hours` parameter in `/stats?hours=48` allows you to control how many hours of recent data statistics to return. If the `hours` parameter is not provided, it defaults to statistics for the last 24 hours. -There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed ethical review, text content of each request, API key for each request, input token count, and output token count for each request. +There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed content moderation, the text content of each request, the API key for each request, the number of input tokens, and the number of output tokens for each request. -## Docker Local Deployment +## Docker local deployment Start the container ```bash docker run --user root -p 8001:8000 --name uni-api -dit \ --e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file is already mounted, you do not need to set CONFIG_URL --v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, you do not need to mount the configuration file --v ./uniapi_db:/home/data \ # If you do not want to save statistical data, you do not need to mount the stats.db file +-e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file has already been mounted, there is no need to set CONFIG_URL +-v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, there is no need to mount the configuration file +-v ./uniapi_db:/home/data \ # If you do not want to save statistical data, there is no need to mount this folder yym68686/uni-api:latest ``` @@ -178,15 +207,15 @@ services: container_name: uni-api image: yym68686/uni-api:latest environment: - - CONFIG_URL=http://file_url/api.yaml # If the local configuration file is already mounted, there is no need to set CONFIG_URL + - CONFIG_URL=http://file_url/api.yaml # If a local configuration file is already mounted, there is no need to set CONFIG_URL ports: - 8001:8000 volumes: - ./api.yaml:/home/api.yaml # If CONFIG_URL is already set, there is no need to mount the configuration file - - ./uniapi_db:/home/data # If you do not want to save statistical data, there is no need to mount the stats.db file + - ./uniapi_db:/home/data # If you do not want to save statistical data, there is no need to mount this folder ``` -CONFIG_URL is used to automatically download remote configuration files. For example, if it is inconvenient to modify the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. If you are using a locally mounted configuration file, you do not need to set CONFIG_URL. CONFIG_URL is used in situations where it is inconvenient to mount the configuration file. +CONFIG_URL is the URL of the remote configuration file that can be automatically downloaded. For example, if you are not comfortable modifying the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link to uni-api to download, which is the CONFIG_URL. If you are using a local mounted configuration file, there is no need to set CONFIG_URL. CONFIG_URL is used when it is not convenient to mount the configuration file. Run Docker Compose container in the background @@ -226,8 +255,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` - -## Star History +## ⭐ Star History Star History Chart diff --git a/README_CN.md b/README_CN.md index 04a2d959..7e4b9f92 100644 --- a/README_CN.md +++ b/README_CN.md @@ -11,11 +11,11 @@ [英文](./README.md) | [中文](./README_CN.md) -## Introduction +## 介绍 如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Groq、Cloudflare、DeepBricks、OpenRouter 等。 -## Features +## ✨ 特性 - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 - 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 @@ -33,16 +33,37 @@ - 支持多个标准 OpenAI 格式的接口：`/v1/chat/completions`，`/v1/images/generations`，`/v1/audio/transcriptions`，`/v1/moderations`，`/v1/models`。 - 支持 OpenAI moderation 道德审查，可以对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。降低后台 API 被提供商封禁的风险。 -## Configuration +## 使用方法 -使用 api.yaml 配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： +启动 uni-api 必须使用配置文件，有两种方式可以启动配置文件： + +1. 第一种是使用 `CONFIG_URL` 环境变量填写配置文件 URL，uni-api启动时会自动下载。 +2. 第二种就是挂载名为 `api.yaml` 的配置文件到容器内。 + +### 方法一：挂载 `api.yaml` 配置文件启动 uni-api + +必须事先填写完成配置文件才能启动 `uni-api`，必须使用名为 `api.yaml` 的配置文件才能启动 `uni-api`，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是最小可运行的 `api.yaml` 配置文件的示例： + +```yaml +providers: + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 + base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 + api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填，自动使用 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 + # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个 API Key 可以配置多个模型。 +api_keys: + - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key，用户请求 uni-api 需要 API key，必填 + model: # 该 API Key 可以使用的模型，必填。默认开启渠道级轮询负载均衡，每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 + - all # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。如果你不想在 `api_keys` 里面给每个 `api` 一个个设置可用渠道，`uni-api` 支持将 `api key` 设置为可以使用 providers 下面所有渠道的所有模型。 +``` + +`api.yaml` 详细的高级配置： ```yaml providers: - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 - model: # 至少填一个模型 + model: # 选填，如果不配置 model，会自动通过 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 - gpt-4o # 可以使用的模型名称，必填 - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 - dall-e-3 @@ -97,7 +118,7 @@ providers: api_keys: - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 - model: # 该 API Key 可以使用的模型，必填 + model: # 该 API Key 可以使用的模型，必填。默认开启渠道级轮询负载均衡，每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 @@ -109,7 +130,8 @@ api_keys: - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 - openai-test/text-moderation-latest # 当开启消息道德审查后，可以使用名为 openai-test 渠道下的 text-moderation-latest 模型进行道德审查。 preferences: - USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true。开启轮训后每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 + SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。修改默认开启的渠道轮询负载均衡。SCHEDULING_ALGORITHM 可选值为：fixed_priority，weighted_round_robin, lottery, random。 + # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 @@ -122,19 +144,26 @@ api_keys: - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 preferences: - USE_ROUND_ROBIN: true # 当 USE_ROUND_ROBIN 必须为 true 并且上面的渠道后面没有权重时，会按照原始的渠道顺序请求，如果有权重，会按照加权后的顺序请求。 + SCHEDULING_ALGORITHM: weighted_round_robin # 仅当 SCHEDULING_ALGORITHM 为 weighted_round_robin 并且上面的渠道如果有权重，会按照加权后的顺序请求。使用加权轮训负载均衡，按照权重顺序请求拥有请求的模型的渠道。当 SCHEDULING_ALGORITHM 为 lottery 时，使用抽奖轮训负载均衡，按照权重随机请求拥有请求的模型的渠道。 AUTO_RETRY: true ``` -如果你不想在 `api_keys` 里面给每个 `api` 一个个设置可用渠道，`uni-api` 支持将 `api key` 设置为可以使用所有模型，配置如下： +挂载配置文件并启动 uni-api docker 容器： -```yaml -# ... providers 配置不变 ... -api_keys: - - api: sk-LjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户请求 uni-api 需要 API key，必填 - model: # 该 API Key 可以使用的模型，必填 - - all # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 -# ... 其他配置不变 ... +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +``` + +### 方法二：使用 `CONFIG_URL` 环境变量启动 uni-api + +按照方法一写完配置文件后，上传到云端硬盘，获取文件的直链，然后使用 `CONFIG_URL` 环境变量启动 uni-api docker 容器： + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-e CONFIG_URL=http://file_url/api.yaml \ +yym68686/uni-api:latest ``` ## 环境变量 @@ -158,7 +187,7 @@ api_keys: 还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 -## Docker Local Deployment +## Docker 本地部署 Start the container @@ -226,8 +255,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` - -## Star History +## ⭐ Star History Star History Chart diff --git a/main.py b/main.py index dcadffc5..e5509746 100644 --- a/main.py +++ b/main.py @@ -128,7 +128,6 @@ async def http_exception_handler(request: Request, exc: HTTPException): ) import uuid -import json import asyncio import contextvars request_info = contextvars.ContextVar('request_info', default={}) @@ -602,6 +601,21 @@ def weighted_round_robin(weights): return weighted_provider_list +import random + +def lottery_scheduling(weights): + total_tickets = sum(weights.values()) + selections = [] + for _ in range(total_tickets): + ticket = random.randint(1, total_tickets) + cumulative = 0 + for provider, weight in weights.items(): + cumulative += weight + if ticket <= cumulative: + selections.append(provider) + break + return selections + import asyncio class ModelRequestHandler: def __init__(self): @@ -683,25 +697,25 @@ def get_matching_providers(self, model_name, token): # model_dict = get_model_dict(provider) # if model_name in model_dict.keys(): # provider_list.append(provider) - if is_debug: - for provider in provider_list: - logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): config = app.state.config - # api_keys_db = app.state.api_keys_db api_list = app.state.api_list + api_index = api_list.index(token) model_name = request.model matching_providers = self.get_matching_providers(model_name, token) - # import json - # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False)) + num_matching_providers = len(matching_providers) + if not matching_providers: raise HTTPException(status_code=404, detail="No matching model found") - # exit(0) + # 检查是否启用轮询 - api_index = api_list.index(token) + scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") + if scheduling_algorithm == "random": + matching_providers = random.sample(matching_providers, num_matching_providers) + weights = safe_get(config, 'api_keys', api_index, "weights") if weights: # 步骤 1: 提取 matching_providers 中的所有 provider 值 @@ -711,7 +725,14 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # 步骤 3: 计算交集 intersection = providers.intersection(weight_keys) weights = dict(filter(lambda item: item[0] in intersection, weights.items())) - weighted_provider_name_list = weighted_round_robin(weights) + + if scheduling_algorithm == "weighted_round_robin": + weighted_provider_name_list = weighted_round_robin(weights) + elif scheduling_algorithm == "lottery": + weighted_provider_name_list = lottery_scheduling(weights) + else: + weighted_provider_name_list = list(weights.keys()) + new_matching_providers = [] for provider_name in weighted_provider_name_list: for provider in matching_providers: @@ -719,34 +740,24 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques new_matching_providers.append(provider) matching_providers = new_matching_providers - # import json - # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False, default=circular_list_encoder)) - use_round_robin = True - auto_retry = True - if safe_get(config, 'api_keys', api_index, "preferences", "USE_ROUND_ROBIN") == False: - use_round_robin = False - if safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY") == False: - auto_retry = False - - return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token) + if is_debug: + for provider in matching_providers: + logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) - # 在 try_all_providers 函数中处理失败的情况 - async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None): status_code = 500 error_message = None - num_providers = len(providers) - model_name = request.model - if use_round_robin: + start_index = 0 + if scheduling_algorithm != "fixed_priority": async with self.locks[model_name]: - self.last_provider_indices[model_name] = (self.last_provider_indices[model_name] + 1) % num_providers + self.last_provider_indices[model_name] = (self.last_provider_indices[model_name] + 1) % num_matching_providers start_index = self.last_provider_indices[model_name] - else: - start_index = 0 - for i in range(num_providers + 1): - current_index = (start_index + i) % num_providers - provider = providers[current_index] + auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) + + for i in range(num_matching_providers + 1): + current_index = (start_index + i) % num_matching_providers + provider = matching_providers[current_index] try: response = await process_request(request, provider, endpoint, token) return response diff --git a/utils.py b/utils.py index 6e62a402..fac30cc7 100644 --- a/utils.py +++ b/utils.py @@ -100,7 +100,8 @@ def update_config(config_data): models.append(key) if isinstance(model, str): models.append(model) - config_data['api_keys'][index]['weights'] = weights_dict + if weights_dict: + config_data['api_keys'][index]['weights'] = weights_dict config_data['api_keys'][index]['model'] = models api_keys_db[index]['model'] = models From 5fc94c05c94d54bdaa14136609ba1e4089786f19 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 18 Oct 2024 21:18:11 +0000 Subject: [PATCH 135/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.32?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d788d433..78bae5bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.31 +0.0.32 From deb4a8860e0c0492a8403630369660221bf6d405 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 19 Oct 2024 06:11:26 +0800 Subject: [PATCH 136/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20vercel=20deployment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + README.md | 4 ++++ README_CN.md | 4 ++++ vercel.json | 14 ++++++++++++++ 4 files changed, 23 insertions(+) create mode 100644 vercel.json diff --git a/.gitignore b/.gitignore index d6dcf7c0..66e5bb64 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ node_modules .pytest_cache *.jpg *.json +!vercel.json *.png *.db .aider* diff --git a/README.md b/README.md index 17574152..ebe7531c 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,10 @@ The `hours` parameter in `/stats?hours=48` allows you to control how many hours There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed content moderation, the text content of each request, the API key for each request, the number of input tokens, and the number of output tokens for each request. +## Vercel Deployment + +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&project-name=uni-api-vercel&repository-name=uni-api-vercel) + ## Docker local deployment Start the container diff --git a/README_CN.md b/README_CN.md index 7e4b9f92..eb0717f6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -187,6 +187,10 @@ yym68686/uni-api:latest 还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 +## Vercel 部署 + +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&project-name=uni-api-vercel&repository-name=uni-api-vercel) + ## Docker 本地部署 Start the container diff --git a/vercel.json b/vercel.json new file mode 100644 index 00000000..bc1bf397 --- /dev/null +++ b/vercel.json @@ -0,0 +1,14 @@ +{ + "builds": [ + { + "src": "main.py", + "use": "@vercel/python" + } + ], + "routes": [ + { + "src": "/(.*)", + "dest": "main.py" + } + ] + } \ No newline at end of file From b4f47632158bae54052b1246f5c94f67491bde32 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 19 Oct 2024 06:30:45 +0800 Subject: [PATCH 137/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20completely=20disabling=20the=20database.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- README_CN.md | 5 ++++- main.py | 38 +++++++++++++++++++++++++++----------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ebe7531c..8a05fac8 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ yym68686/uni-api:latest - CONFIG_URL: The download address of the configuration file, which can be a local file or a remote file, optional - TIMEOUT: Request timeout, default is 100 seconds. The timeout can control the time needed to switch to the next channel when one channel does not respond. Optional +- DISABLE_DATABASE: Whether to disable the database, default is false, optional ## Get statistical data @@ -189,7 +190,7 @@ There are other statistical data that you can query yourself by writing SQL in t ## Vercel Deployment -[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&project-name=uni-api-vercel&repository-name=uni-api-vercel) +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) ## Docker local deployment diff --git a/README_CN.md b/README_CN.md index eb0717f6..1fb899d2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -170,6 +170,7 @@ yym68686/uni-api:latest - CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 - TIMEOUT: 请求超时时间，默认为 100 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +- DISABLE_DATABASE: 是否禁用数据库，默认为 false，选填 ## 获取统计数据 @@ -189,7 +190,9 @@ yym68686/uni-api:latest ## Vercel 部署 -[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&project-name=uni-api-vercel&repository-name=uni-api-vercel) +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) + +点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链，然后点击 Create 创建项目。 ## Docker 本地部署 diff --git a/main.py b/main.py index e5509746..91512a62 100644 --- a/main.py +++ b/main.py @@ -34,8 +34,13 @@ from sqlalchemy import inspect, text from sqlalchemy.sql import sqltypes +# 添加新的环境变量检查 +DISABLE_DATABASE = os.getenv("DISABLE_DATABASE", "false").lower() == "true" + async def create_tables(): - async with engine.begin() as conn: + if DISABLE_DATABASE: + return + async with db_engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) # 检查并添加缺失的列 @@ -85,7 +90,8 @@ async def lifespan(app: FastAPI): # for route in frontend_router.routes: # print(f"Route: {route.path}, methods: {route.methods}") # 启动时的代码 - await create_tables() + if not DISABLE_DATABASE: + await create_tables() TIMEOUT = float(os.getenv("TIMEOUT", 100)) timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) @@ -178,17 +184,19 @@ class ChannelStat(Base): success = Column(Boolean, default=False) timestamp = Column(DateTime(timezone=True), server_default=func.now()) -# 获取数据库路径 -db_path = os.getenv('DB_PATH', './data/stats.db') -# 确保 data 目录存在 -data_dir = os.path.dirname(db_path) -os.makedirs(data_dir, exist_ok=True) +if not DISABLE_DATABASE: + # 获取数据库路径 + db_path = os.getenv('DB_PATH', './data/stats.db') + + # 确保 data 目录存在 + data_dir = os.path.dirname(db_path) + os.makedirs(data_dir, exist_ok=True) -# 创建异步引擎和会话 -# engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=False) -engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) -async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + # 创建异步引擎和会话 + # db_engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=False) + db_engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) + async_session = sessionmaker(db_engine, class_=AsyncSession, expire_on_commit=False) from starlette.types import Scope, Receive, Send from starlette.responses import Response @@ -260,6 +268,8 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: async def update_stats(self): # 这里添加更新数据库的逻辑 # print("current_info2") + if DISABLE_DATABASE: + return async with async_session() as session: async with session.begin(): try: @@ -426,6 +436,8 @@ async def dispatch(self, request: Request, call_next): request_info.reset(current_request_info) async def update_stats(self, current_info): + if DISABLE_DATABASE: + return # 这里添加更新数据库的逻辑 async with async_session() as session: async with session.begin(): @@ -440,6 +452,8 @@ async def update_stats(self, current_info): logger.error(f"Error updating stats: {str(e)}") async def update_channel_stats(self, request_id, provider, model, api_key, success): + if DISABLE_DATABASE: + return async with async_session() as session: async with session.begin(): try: @@ -958,6 +972,8 @@ async def get_stats( token: str = Depends(verify_admin_api_key), hours: int = Query(default=24, ge=1, le=720, description="Number of hours to look back for stats (1-720)") ): + if DISABLE_DATABASE: + return JSONResponse(content={"stats": {}}) async with async_session() as session: # 计算指定时间范围的开始时间 start_time = datetime.now(timezone.utc) - timedelta(hours=hours) From 6719e81a8ddeb23685efecb2b71f04f29e00b1d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 18 Oct 2024 22:31:03 +0000 Subject: [PATCH 138/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.33?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 78bae5bb..cd9d21e6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.32 +0.0.33 From 2c0a34801b0c2f49541f1f8023ccf105d9785e15 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 19 Oct 2024 21:21:44 +0800 Subject: [PATCH 139/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20vercel=20cannot=20set=20app.state.config.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 + README_CN.md | 2 +- main.py | 125 ++++++++++++++++++++++++--------------------------- 3 files changed, 61 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 8a05fac8..2ed330a2 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ There are other statistical data that you can query yourself by writing SQL in t [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) +After clicking the one-click deployment button, set the environment variable `CONFIG_URL` to the direct link of the configuration file, and set `DISABLE_DATABASE` to true, then click Create to create the project. + ## Docker local deployment Start the container diff --git a/README_CN.md b/README_CN.md index 1fb899d2..f066d230 100644 --- a/README_CN.md +++ b/README_CN.md @@ -192,7 +192,7 @@ yym68686/uni-api:latest [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) -点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链，然后点击 Create 创建项目。 +点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。 ## Docker 本地部署 diff --git a/main.py b/main.py index 91512a62..a1d53d66 100644 --- a/main.py +++ b/main.py @@ -106,17 +106,6 @@ async def lifespan(app: FastAPI): verify=True, # 保持 SSL 验证（如需禁用，设为 False，但不建议） follow_redirects=True, # 自动跟随重定向 ) - # app.state.client = httpx.AsyncClient(timeout=timeout) - app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) - - for item in app.state.api_keys_db: - if item.get("role") == "admin": - app.state.admin_api_key = item.get("api") - if not hasattr(app.state, "admin_api_key"): - if len(app.state.api_keys_db) >= 1: - app.state.admin_api_key = app.state.api_keys_db[0].get("api") - else: - raise Exception("No admin API key found") yield # 关闭时的代码 @@ -224,6 +213,41 @@ def calculate_cost(model: str, input_tokens: int, output_tokens: int) -> Decimal # 返回精确到15位小数的结果 return total_cost.quantize(Decimal('0.000000000000001')) +async def update_stats(current_info): + if DISABLE_DATABASE: + return + # 这里添加更新数据库的逻辑 + async with async_session() as session: + async with session.begin(): + try: + columns = [column.key for column in RequestStat.__table__.columns] + filtered_info = {k: v for k, v in current_info.items() if k in columns} + new_request_stat = RequestStat(**filtered_info) + session.add(new_request_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating stats: {str(e)}") + +async def update_channel_stats(request_id, provider, model, api_key, success): + if DISABLE_DATABASE: + return + async with async_session() as session: + async with session.begin(): + try: + channel_stat = ChannelStat( + request_id=request_id, + provider=provider, + model=model, + api_key=api_key, + success=success, + ) + session.add(channel_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating channel stats: {str(e)}") + class LoggingStreamingResponse(Response): def __init__(self, content, status_code=200, headers=None, media_type=None, current_info=None): super().__init__(content=None, status_code=status_code, headers=headers, media_type=media_type) @@ -263,31 +287,14 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: process_time = time() - self.current_info["start_time"] self.current_info["process_time"] = process_time - await self.update_stats() - - async def update_stats(self): - # 这里添加更新数据库的逻辑 - # print("current_info2") - if DISABLE_DATABASE: - return - async with async_session() as session: - async with session.begin(): - try: - columns = [column.key for column in RequestStat.__table__.columns] - filtered_info = {k: v for k, v in self.current_info.items() if k in columns} - new_request_stat = RequestStat(**filtered_info) - session.add(new_request_stat) - await session.commit() - except Exception as e: - await session.rollback() - logger.error(f"Error updating stats: {str(e)}") + await update_stats(self.current_info) async def _logging_iterator(self): try: async for chunk in self.body_iterator: if isinstance(chunk, str): chunk = chunk.encode('utf-8') - line = chunk.decode() + line = chunk.decode('utf-8') if is_debug: logger.info(f"{line}") if line.startswith("data:"): @@ -435,41 +442,6 @@ async def dispatch(self, request: Request, call_next): # print("current_request_info", current_request_info) request_info.reset(current_request_info) - async def update_stats(self, current_info): - if DISABLE_DATABASE: - return - # 这里添加更新数据库的逻辑 - async with async_session() as session: - async with session.begin(): - try: - columns = [column.key for column in RequestStat.__table__.columns] - filtered_info = {k: v for k, v in current_info.items() if k in columns} - new_request_stat = RequestStat(**filtered_info) - session.add(new_request_stat) - await session.commit() - except Exception as e: - await session.rollback() - logger.error(f"Error updating stats: {str(e)}") - - async def update_channel_stats(self, request_id, provider, model, api_key, success): - if DISABLE_DATABASE: - return - async with async_session() as session: - async with session.begin(): - try: - channel_stat = ChannelStat( - request_id=request_id, - provider=provider, - model=model, - api_key=api_key, - success=success, - ) - session.add(channel_stat) - await session.commit() - except Exception as e: - await session.rollback() - logger.error(f"Error updating channel stats: {str(e)}") - async def moderate_content(self, content, token): moderation_request = ModerationRequest(input=content) @@ -500,6 +472,23 @@ async def moderate_content(self, content, token): app.add_middleware(StatsMiddleware) +@app.middleware("http") +async def ensure_config(request: Request, call_next): + if not hasattr(app.state, 'config'): + logger.warning("Config not found, attempting to reload") + app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) + + for item in app.state.api_keys_db: + if item.get("role") == "admin": + app.state.admin_api_key = item.get("api") + if not hasattr(app.state, "admin_api_key"): + if len(app.state.api_keys_db) >= 1: + app.state.admin_api_key = app.state.api_keys_db[0].get("api") + else: + raise Exception("No admin API key found") + + return await call_next(request) + # 在 process_request 函数中更新成功和失败计数 async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], provider: Dict, endpoint=None, token=None): url = provider['base_url'] @@ -581,14 +570,16 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A # response = JSONResponse(first_element) # 更新成功计数和首次响应时间 - await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) current_info["first_response_time"] = first_response_time current_info["success"] = True current_info["provider"] = provider['provider'] return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: - await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) + await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) + # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) raise e From 6caeb3f6b2eb342bf5dc239779d9a9c9e725fc61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 19 Oct 2024 13:22:19 +0000 Subject: [PATCH 140/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.34?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cd9d21e6..bb951c88 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.33 +0.0.34 From c35d947c833851e061befa9f6c6d6a63fa93216b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 20 Oct 2024 03:26:35 +0800 Subject: [PATCH 141/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Add=20documentat?= =?UTF-8?q?ion=20for=20serv00=20deployment=20steps.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++- README_CN.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ main.py | 15 +------------- utils.py | 33 ++++++++++++++++++++++-------- 4 files changed, 137 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 2ed330a2..af060f3b 100644 --- a/README.md +++ b/README.md @@ -188,12 +188,67 @@ The `hours` parameter in `/stats?hours=48` allows you to control how many hours There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed content moderation, the text content of each request, the API key for each request, the number of input tokens, and the number of output tokens for each request. -## Vercel Deployment +## Vercel remote deployment [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) After clicking the one-click deployment button, set the environment variable `CONFIG_URL` to the direct link of the configuration file, and set `DISABLE_DATABASE` to true, then click Create to create the project. +## Serv00 remote deployment + +First, log in to the panel, in Additional services click on the tab Run your own applications to enable the option to run your own programs, then go to the panel Port reservation to randomly open a port. + +If you don't have your own domain name, go to the panel WWW websites and delete the default domain name provided. Then create a new domain with the Domain being the one you just deleted. After clicking Advanced settings, set the Website type to Proxy domain, and the Proxy port should point to the port you just opened. Do not select Use HTTPS. + +ssh login to the serv00 server, execute the following command: + +```bash +git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git +cd uni-api +python -m venv uni-api +tmux new -s uni-api +source uni-api/bin/activate +export CFLAGS="-I/usr/local/include" +export CXXFLAGS="-I/usr/local/include" +export CC=gcc +export CXX=g++ +export MAX_CONCURRENCY=1 +export CPUCOUNT=1 +export MAKEFLAGS="-j1" +CMAKE_BUILD_PARALLEL_LEVEL=1 cpuset -l 0 pip install -vv -r requirements.txt +cpuset -l 0 pip install -r -vv requirements.txt +``` + +ctrl+b d to exit tmux, wait a few hours for the installation to complete, and after the installation is complete, execute the following command: + +```bash +tmux attach -t uni-api +source uni-api/bin/activate +export CONFIG_URL=http://file_url/api.yaml +export DISABLE_DATABASE=true +# Modify the port, xxx is the port, modify it yourself, corresponding to the port opened in the panel Port reservation +sed -i '' 's/port=8000/port=xxx/' main.py +sed -i '' 's/reload=True/reload=False/' main.py +python main.py +``` + +Use ctrl+b d to exit tmux, allowing the program to run in the background. At this point, you can use uni-api in other chat clients. curl test script: + +```bash +curl -X POST https://xxx.serv00.net/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-xxx' \ +-d '{"model": "gpt-4o","messages": [{"role": "user","content": "Hello"}]}' +``` + +Reference document: + +https://docs.serv00.com/Python/ + +https://linux.do/t/topic/201181 + +https://linux.do/t/topic/218738 + ## Docker local deployment Start the container diff --git a/README_CN.md b/README_CN.md index f066d230..156c9417 100644 --- a/README_CN.md +++ b/README_CN.md @@ -194,6 +194,61 @@ yym68686/uni-api:latest 点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。 +## serv00 远程部署 + +首先登录面板，Additional services 里面点击选项卡 Run your own applications 开启允许运行自己的程序，然后到面板 Port reservation 去随便开一个端口。 + +如果没有自己的域名，去面板 WWW websites 删掉默认给的域名，再新建一个域名 Domain 为刚才删掉的域名，点击 Advanced settings 后设置 Website type 为 Proxy 域名，Proxy port 指向你刚才开的端口，不要选中 Use HTTPS。 + +ssh 登陆到 serv00 服务器，执行下面的命令： + +```bash +git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git +cd uni-api +python -m venv uni-api +tmux new -s uni-api +source uni-api/bin/activate +export CFLAGS="-I/usr/local/include" +export CXXFLAGS="-I/usr/local/include" +export CC=gcc +export CXX=g++ +export MAX_CONCURRENCY=1 +export CPUCOUNT=1 +export MAKEFLAGS="-j1" +CMAKE_BUILD_PARALLEL_LEVEL=1 cpuset -l 0 pip install -vv -r requirements.txt +cpuset -l 0 pip install -r -vv requirements.txt +``` + +ctrl+b d 退出 tmux 等待几个小时安装完成，安装完成后执行下面的命令： + +```bash +tmux attach -t uni-api +source uni-api/bin/activate +export CONFIG_URL=http://file_url/api.yaml +export DISABLE_DATABASE=true +# 修改端口，xxx 为端口，自行修改，对应刚刚在面板 Port reservation 开的端口 +sed -i '' 's/port=8000/port=xxx/' main.py +sed -i '' 's/reload=True/reload=False/' main.py +python main.py +``` + +使用 ctrl+b d 退出 tmux，即可让程序后台运行。此时就可以在其他聊天客户端使用 uni-api 了。curl 测试脚本： + +```bash +curl -X POST https://xxx.serv00.net/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-xxx' \ +-d '{"model": "gpt-4o","messages": [{"role": "user","content": "你好"}]}' +``` + +参考文档： + +https://docs.serv00.com/Python/ + +https://linux.do/t/topic/201181 + +https://linux.do/t/topic/218738 + ## Docker 本地部署 Start the container diff --git a/main.py b/main.py index a1d53d66..a2e5a9dc 100644 --- a/main.py +++ b/main.py @@ -89,24 +89,11 @@ async def lifespan(app: FastAPI): # print("\nFrontend router routes:") # for route in frontend_router.routes: # print(f"Route: {route.path}, methods: {route.methods}") + # 启动时的代码 if not DISABLE_DATABASE: await create_tables() - TIMEOUT = float(os.getenv("TIMEOUT", 100)) - timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) - default_headers = { - "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent - "Accept": "*/*", # curl 的默认 Accept 头 - } - app.state.client = httpx.AsyncClient( - timeout=timeout, - headers=default_headers, - http2=True, # 禁用 HTTP/2 - verify=True, # 保持 SSL 验证（如需禁用，设为 False，但不建议） - follow_redirects=True, # 自动跟随重定向 - ) - yield # 关闭时的代码 await app.state.client.aclose() diff --git a/utils.py b/utils.py index fac30cc7..02402634 100644 --- a/utils.py +++ b/utils.py @@ -111,6 +111,23 @@ def update_config(config_data): # 读取YAML配置文件 async def load_config(app=None): + + if app and not hasattr(app.state, 'client'): + import os + TIMEOUT = float(os.getenv("TIMEOUT", 100)) + timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) + default_headers = { + "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent + "Accept": "*/*", # curl 的默认 Accept 头 + } + app.state.client = httpx.AsyncClient( + timeout=timeout, + headers=default_headers, + http2=True, # 禁用 HTTP/2 + verify=True, # 保持 SSL 验证（如需禁用，设为 False，但不建议） + follow_redirects=True, # 自动跟随重定向 + ) + from ruamel.yaml import YAML, YAMLError yaml = YAML() yaml.preserve_quotes = True @@ -122,19 +139,19 @@ async def load_config(app=None): if conf: config, api_keys_db, api_list = update_config(conf) else: - # logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") - config, api_keys_db, api_list = [], [], [] + logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") + config, api_keys_db, api_list = {}, {}, [] except FileNotFoundError: logger.error("'api.yaml' not found. Please check the file path.") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] except YAMLError as e: logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。%s", e) - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] except OSError as e: logger.error(f"open 'api.yaml' failed: {e}") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] - if config != []: + if config != {}: return config, api_keys_db, api_list import os @@ -152,10 +169,10 @@ async def load_config(app=None): config, api_keys_db, api_list = update_config(config_data) else: logger.error(f"Error fetching or parsing config from {config_url}") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] except Exception as e: logger.error(f"Error fetching or parsing config from {config_url}: {str(e)}") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] return config, api_keys_db, api_list def ensure_string(item): From bf80b6aa2f5a5fa0518f43e25090aae1a4d60b96 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 19 Oct 2024 19:26:56 +0000 Subject: [PATCH 142/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.35?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bb951c88..155069a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.34 +0.0.35 From 73319d158477ada5d5c842d3b33c1102afd3907a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 05:33:54 +0800 Subject: [PATCH 143/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20Gemini=20cannot=20use=20non-streaming=20output.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 7 +++-- response.py | 73 ++++++++++++++++++++++++++++++++++++++++++++--- test/test_json.py | 26 +++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 test/test_json.py diff --git a/main.py b/main.py index a2e5a9dc..34a9c5cc 100644 --- a/main.py +++ b/main.py @@ -531,7 +531,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if provider.get("engine"): engine = provider["engine"] - logger.info(f"provider: {provider['provider']:<10} model: {request.model:<10} engine: {engine}") + logger.info(f"provider: {provider['provider']:<11} model: {request.model:<22} engine: {engine}") url, headers, payload = await get_payload(request, engine, provider) if is_debug: @@ -542,16 +542,17 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) current_info = request_info.get() try: + model = model_dict[request.model] if request.stream: - model = model_dict[request.model] generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: - generator = fetch_response(app.state.client, url, headers, payload) + generator = fetch_response(app.state.client, url, headers, payload, engine, model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") + print("first_element", first_element) first_element = json.loads(first_element) response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") # response = JSONResponse(first_element) diff --git a/response.py b/response.py index c0603fc6..41c2b1a1 100644 --- a/response.py +++ b/response.py @@ -4,6 +4,8 @@ from log_config import logger +from utils import safe_get + # end_of_line = "\n\r\n" # end_of_line = "\r\n" # end_of_line = "\n\r" @@ -17,7 +19,6 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f "object": "chat.completion.chunk", "created": timestamp, "model": model, - "system_fingerprint": "fp_d576307f90", "choices": [ { "index": 0, @@ -26,7 +27,8 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f "finish_reason": None } ], - "usage": None + "usage": None, + "system_fingerprint": "fp_d576307f90", } if function_call_content: sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]} @@ -46,6 +48,34 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f return sse_response +async def generate_no_stream_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + sample_data = { + "id": "chatcmpl-ALGS9hpJBb8xVAe62DRriY2SpoT4L", + "object": "chat.completion", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": role, + "content": content, + "refusal": None + }, + "logprobs": None, + "finish_reason": "stop" + } + ], + "usage": None, + "system_fingerprint": "fp_a7d06e42a7" + } + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + json_data = json.dumps(sample_data, ensure_ascii=False) + + return json_data + async def check_response(response, error_log): if response and response.status_code != 200: error_message = await response.aread() @@ -274,7 +304,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model): yield sse_string yield "data: [DONE]" + end_of_line -async def fetch_response(client, url, headers, payload): +async def fetch_response(client, url, headers, payload, engine, model): response = None if payload.get("file"): file = payload.pop("file") @@ -285,7 +315,42 @@ async def fetch_response(client, url, headers, payload): if error_message: yield error_message return - yield response.json() + response_json = response.json() + if engine == "gemini" or engine == "vertex-gemini": + + if isinstance(response_json, str): + import ast + parsed_data = ast.literal_eval(str(response_json)) + elif isinstance(response_json, list): + parsed_data = response_json + else: + logger.error(f"error fetch_response: Unknown response_json type: {type(response_json)}") + parsed_data = response_json + + content = "" + for item in parsed_data: + chunk = safe_get(item, "candidates", 0, "content", "parts", 0, "text") + # logger.info(f"chunk: {repr(chunk)}") + if chunk: + content += chunk + + usage_metadata = safe_get(parsed_data, -1, "usageMetadata") + prompt_tokens = usage_metadata.get("promptTokenCount", 0) + candidates_tokens = usage_metadata.get("candidatesTokenCount", 0) + total_tokens = usage_metadata.get("totalTokenCount", 0) + + role = safe_get(parsed_data, -1, "candidates", 0, "content", "role") + if role == "model": + role = "assistant" + else: + logger.error(f"Unknown role: {role}") + role = "assistant" + + timestamp = int(datetime.timestamp(datetime.now())) + yield await generate_no_stream_response(timestamp, model, content=content, tools_id=None, function_call_name=None, function_call_content=None, role=role, total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=candidates_tokens) + + else: + yield response_json async def fetch_response_stream(client, url, headers, payload, engine, model): try: diff --git a/test/test_json.py b/test/test_json.py new file mode 100644 index 00000000..f52fe154 --- /dev/null +++ b/test/test_json.py @@ -0,0 +1,26 @@ +import ast +import json + +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils import safe_get +# 读取文件内容 +with open('test/states.json', 'r', encoding='utf-8') as file: + content = file.read() + +# 使用ast.literal_eval解析非标准JSON +parsed_data = ast.literal_eval(content) + +for item in parsed_data: + print(safe_get(item, "candidates", 0, "content", "parts", 0, "text")) + print(safe_get(item, "candidates", 0, "content", "role")) + +# 将解析后的数据转换为标准JSON +standard_json = json.dumps(parsed_data, ensure_ascii=False, indent=2) + +# 将标准JSON写入新文件 +with open('test/standard_states.json', 'w', encoding='utf-8') as file: + file.write(standard_json) + +print("转换完成，标准JSON已保存到 'test/standard_states.json'") \ No newline at end of file From 5b2a3fa26a6b7764220db6864fa97a1633ff92b8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 22 Oct 2024 21:34:15 +0000 Subject: [PATCH 144/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.36?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 155069a3..e85669f8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.35 +0.0.36 From b90ac28e7610b5a111f569210a455d83f2e1b6f3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 06:01:02 +0800 Subject: [PATCH 145/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20the=20feat?= =?UTF-8?q?ure=20of=20automatic=20repository=20synchronization.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/sync.yml diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 00000000..c23c4115 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,25 @@ +name: Sync Fork + +on: + schedule: + - cron: '0 0 * * *' # 每天凌晨执行 + # - cron: '0 */12 * * *' # 每12小时执行一次 + workflow_dispatch: # 支持手动触发 + +jobs: + sync: + runs-on: ubuntu-latest + if: | + secrets.UPSTREAM_REPO != '' && + secrets.GITHUB_TOKEN != '' + + steps: + - uses: actions/checkout@v3 + + - name: Sync Fork + uses: aormsby/Fork-Sync-With-Upstream-action@v3.4 + with: + target_repo_token: ${{ secrets.GITHUB_TOKEN }} + upstream_sync_repo: ${{ secrets.UPSTREAM_REPO }} + upstream_sync_branch: main + target_sync_branch: main \ No newline at end of file From 3c2b0d7bd89862e2b08093a9a528b24eb08cbaa8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 06:05:57 +0800 Subject: [PATCH 146/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20the=20feat?= =?UTF-8?q?ure=20of=20automatic=20repository=20synchronization.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c23c4115..baee2378 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -10,6 +10,7 @@ jobs: sync: runs-on: ubuntu-latest if: | + github.repository != 'yym68686/uni-api' && secrets.UPSTREAM_REPO != '' && secrets.GITHUB_TOKEN != '' @@ -20,6 +21,6 @@ jobs: uses: aormsby/Fork-Sync-With-Upstream-action@v3.4 with: target_repo_token: ${{ secrets.GITHUB_TOKEN }} - upstream_sync_repo: ${{ secrets.UPSTREAM_REPO }} + upstream_sync_repo: yym68686/uni-api upstream_sync_branch: main target_sync_branch: main \ No newline at end of file From 88b179434e54bd82b29bb473acc3fbc77eac13a4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 06:06:22 +0800 Subject: [PATCH 147/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20the=20feat?= =?UTF-8?q?ure=20of=20automatic=20repository=20synchronization.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index baee2378..fa8134e5 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -11,7 +11,6 @@ jobs: runs-on: ubuntu-latest if: | github.repository != 'yym68686/uni-api' && - secrets.UPSTREAM_REPO != '' && secrets.GITHUB_TOKEN != '' steps: From 7b8c2c54afab3b289d91086f3d8cc0a7a883e74f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 17:46:06 +0800 Subject: [PATCH 148/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20GitHub=20Action=20from=20running.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index fa8134e5..57c50f20 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -9,11 +9,18 @@ on: jobs: sync: runs-on: ubuntu-latest - if: | - github.repository != 'yym68686/uni-api' && - secrets.GITHUB_TOKEN != '' + if: github.repository != 'yym68686/uni-api' steps: + - name: Check if GITHUB_TOKEN exists + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -z "$GITHUB_TOKEN" ]; then + echo "GITHUB_TOKEN is not set" + exit 1 + fi + - uses: actions/checkout@v3 - name: Sync Fork From cd6dfcd6907633113abe6202e31bfa6b6849495c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 17:53:04 +0800 Subject: [PATCH 149/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Remove=20the=20r?= =?UTF-8?q?edundant=20GITHUB=5FTOKEN=20check=20step=20in=20GitHub=20action?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 57c50f20..2391ccff 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -12,15 +12,6 @@ jobs: if: github.repository != 'yym68686/uni-api' steps: - - name: Check if GITHUB_TOKEN exists - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if [ -z "$GITHUB_TOKEN" ]; then - echo "GITHUB_TOKEN is not set" - exit 1 - fi - - uses: actions/checkout@v3 - name: Sync Fork From f7f572eb6079edf4731c3e8c5dfa64df26ddb423 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 20:47:55 +0800 Subject: [PATCH 150/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20may=20cause=20GitHub=20action=20to=20not=20run.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 2391ccff..445a5edb 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -12,12 +12,23 @@ jobs: if: github.repository != 'yym68686/uni-api' steps: - - uses: actions/checkout@v3 + - name: Checkout target repo + uses: actions/checkout@v4.2.1 + with: + fetch-depth: 0 # 获取所有历史记录，以确保正确同步 - name: Sync Fork - uses: aormsby/Fork-Sync-With-Upstream-action@v3.4 + uses: aormsby/Fork-Sync-With-Upstream-action@v3.4.1 with: target_repo_token: ${{ secrets.GITHUB_TOKEN }} upstream_sync_repo: yym68686/uni-api upstream_sync_branch: main - target_sync_branch: main \ No newline at end of file + target_sync_branch: main + + - name: Check for new commits + if: steps.sync.outputs.has_new_commits == 'true' + run: echo "新的提交已同步。" + + - name: No new commits + if: steps.sync.outputs.has_new_commits == 'false' + run: echo "没有新的提交需要同步。" \ No newline at end of file From 9194b3b2956d5806754527063a6409f6763d6e37 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 21:39:36 +0800 Subject: [PATCH 151/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20may=20cause=20GitHub=20action=20to=20not=20run.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 445a5edb..de6dc23d 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -16,14 +16,18 @@ jobs: uses: actions/checkout@v4.2.1 with: fetch-depth: 0 # 获取所有历史记录，以确保正确同步 + token: ${{ secrets.PAT }} # 使用PAT替代GITHUB_TOKEN - name: Sync Fork uses: aormsby/Fork-Sync-With-Upstream-action@v3.4.1 with: - target_repo_token: ${{ secrets.GITHUB_TOKEN }} + target_repo_token: ${{ secrets.PAT }} upstream_sync_repo: yym68686/uni-api upstream_sync_branch: main target_sync_branch: main + upstream_pull_args: --allow-unrelated-histories --no-edit --strategy-option theirs + force_push: true + test_mode: false - name: Check for new commits if: steps.sync.outputs.has_new_commits == 'true' From 031b5175f4d037cf767bc7fc4e5a669620a9ce7a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 23 Oct 2024 21:57:20 +0800 Subject: [PATCH 152/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20delete=20Unexpect?= =?UTF-8?q?ed=20input(s)=20'force=5Fpush'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index de6dc23d..ff6aa8d7 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -26,7 +26,6 @@ jobs: upstream_sync_branch: main target_sync_branch: main upstream_pull_args: --allow-unrelated-histories --no-edit --strategy-option theirs - force_push: true test_mode: false - name: Check for new commits From 7fb5f96a42a45b48a8fe27046542a8ba6823c2fd Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 24 Oct 2024 21:52:40 +0800 Subject: [PATCH 153/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20model=20is=20not=20persisted=20to=20the=20file?= =?UTF-8?q?=20after=20being=20automatically=20retrieved.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 17 +++++++---------- utils.py | 29 ++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index 34a9c5cc..218430f0 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest from request import get_payload from response import fetch_response, fetch_response_stream -from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder, get_model_dict +from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder, get_model_dict, save_api_yaml from collections import defaultdict from typing import List, Dict, Union @@ -1120,8 +1120,6 @@ async def frontend_rate_limit_dependency(request: Request, x_api_key: str = Depe xue_initialize(tailwind=True) -API_YAML_PATH = "./api.yaml" - data_table_columns = [ # {"label": "Status", "value": "status", "sortable": True}, {"label": "Provider", "value": "provider", "sortable": True}, @@ -1500,10 +1498,6 @@ def update_row_data(row_id, updated_data): index = int(row_id) app.state.config["providers"][index] = updated_data -def save_api_yaml(): - with open(API_YAML_PATH, "w", encoding="utf-8") as f: - yaml.dump(app.state.config, f) - @frontend_router.post("/submit/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def submit_form( row_id: str, @@ -1551,7 +1545,8 @@ async def submit_form( update_row_data(row_id, updated_data) # 保存更新后的配置 - save_api_yaml() + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) return await root() @@ -1564,7 +1559,8 @@ async def duplicate_row(row_id: str): app.state.config["providers"].insert(index + 1, new_data) # 保存更新后的配置 - save_api_yaml() + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) return await root() @@ -1574,7 +1570,8 @@ async def delete_row(row_id: str): del app.state.config["providers"][index] # 保存更新后的配置 - save_api_yaml() + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) return await root() diff --git a/utils.py b/utils.py index 02402634..6bcac081 100644 --- a/utils.py +++ b/utils.py @@ -63,7 +63,18 @@ def update_initial_model(api_url, api): traceback.print_exc() return [] -def update_config(config_data): +from ruamel.yaml import YAML, YAMLError +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +API_YAML_PATH = "./api.yaml" + +def save_api_yaml(config_data): + with open(API_YAML_PATH, "w", encoding="utf-8") as f: + yaml.dump(config_data, f) + +def update_config(config_data, use_config_url=False): for index, provider in enumerate(config_data['providers']): if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' @@ -78,7 +89,11 @@ def update_config(config_data): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList(provider_api) if not provider.get("model"): - provider["model"] = update_initial_model(provider['base_url'], provider['api']) + model_list = update_initial_model(provider['base_url'], provider['api']) + if model_list: + provider["model"] = model_list + if not use_config_url: + save_api_yaml(config_data) if provider.get("tools") == None: provider["tools"] = True @@ -128,16 +143,12 @@ async def load_config(app=None): follow_redirects=True, # 自动跟随重定向 ) - from ruamel.yaml import YAML, YAMLError - yaml = YAML() - yaml.preserve_quotes = True - yaml.indent(mapping=2, sequence=4, offset=2) try: - with open('api.yaml', 'r', encoding='utf-8') as file: + with open(API_YAML_PATH, 'r', encoding='utf-8') as file: conf = yaml.load(file) if conf: - config, api_keys_db, api_list = update_config(conf) + config, api_keys_db, api_list = update_config(conf, use_config_url=False) else: logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") config, api_keys_db, api_list = {}, {}, [] @@ -166,7 +177,7 @@ async def load_config(app=None): # 更新配置 # logger.info(config_data) if config_data: - config, api_keys_db, api_list = update_config(config_data) + config, api_keys_db, api_list = update_config(config_data, use_config_url=True) else: logger.error(f"Error fetching or parsing config from {config_url}") config, api_keys_db, api_list = {}, {}, [] From 8c8e2940567dba6805f6ed8a2d616bcc30149f56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Oct 2024 13:53:00 +0000 Subject: [PATCH 154/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.37?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e85669f8..1435d6cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.36 +0.0.37 From 972208e84c3d16b98f94b6d1e96c8a9dd0fa4b57 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 24 Oct 2024 22:18:52 +0800 Subject: [PATCH 155/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20tool=20use=20is=20not=20compatible=20with=20enum=20f?= =?UTF-8?q?ields.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index 3a301c41..26842ade 100644 --- a/models.py +++ b/models.py @@ -5,7 +5,7 @@ class FunctionParameter(BaseModel): type: str - properties: Dict[str, Dict[str, Union[str, Dict[str, str]]]] + properties: Dict[str, Dict[str, Any]] required: List[str] class Function(BaseModel): From 0ebec228cbd13ef19f470cfadd8d9b08c96830a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Oct 2024 14:19:15 +0000 Subject: [PATCH 156/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.38?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1435d6cf..e4a35e8b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.37 +0.0.38 From 60014c4b0c5f55f06aefb8d9e02ad459686dad8c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 25 Oct 2024 04:02:05 +0800 Subject: [PATCH 157/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20support=20wildcard=20matching=20like=20gpt*=20to=20match=20m?= =?UTF-8?q?odels=20such=20as=20gpt-3.5=20and=20gpt-4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 56 +++++++++++++++++++++++++++-------------------- test/test_dict.py | 12 ++++++++++ 2 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 test/test_dict.py diff --git a/main.py b/main.py index 218430f0..22ac3a7e 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ from log_config import logger import re +import copy import httpx import secrets from time import time @@ -652,44 +653,51 @@ def get_matching_providers(self, model_name, token): # print("model_name", model_name) # print("model_name_split", model_name_split) # print("model", model) + + # api_keys 中 model 为 provider_name/* 时，表示所有模型都匹配 if model_name_split == "*": if model_name in models_list: provider_rules.append(provider_name) - elif model_name_split == model_name: - if model_name in models_list: - provider_rules.append(provider_name) + + # 如果请求模型名： gpt-4* ，则匹配所有以模型名开头且不以 * 结尾的模型 + for models_list_model in models_list: + if model_name.endswith("*") and models_list_model.startswith(model_name.rstrip("*")): + provider_rules.append(provider_name + "/" + models_list_model) + + # api_keys 中 model 为 provider_name/model_name 时，表示模型名完全匹配 + elif model_name_split == model_name \ + or (model_name.endswith("*") and model_name_split.startswith(model_name.rstrip("*"))): # api_keys 中 model 为 provider_name/model_name 时，请求模型名： model_name* + if model_name_split in models_list: + provider_rules.append(provider_name + "/" + model_name_split) + else: - for provider in config['providers']: + for provider in config["providers"]: model_dict = get_model_dict(provider) if model in model_dict.keys(): - provider_rules.append(provider['provider'] + "/" + model) + provider_rules.append(provider["provider"] + "/" + model) provider_list = [] # print("provider_rules", provider_rules) for item in provider_rules: for provider in config['providers']: # print("provider", provider, provider['provider'] == item, item) - if "/" in item: - if provider['provider'] == item.split("/")[0]: - model_dict = get_model_dict(provider) - if model_name in model_dict.keys() and "/".join(item.split("/")[1:]) == model_name: - provider_list.append(provider) - # 如果 item 不包含 /，则直接匹配 provider，说明整个渠道所有模型都能用 - elif provider['provider'] == item: + if provider['provider'] == item.split("/")[0]: + new_provider = copy.deepcopy(provider) model_dict = get_model_dict(provider) + # print("model_dict", model_dict) + model_name_split = "/".join(item.split("/")[1:]) if model_name in model_dict.keys(): - provider_list.append(provider) - else: - pass - - # if provider['provider'] == item: - # if "/" in item: - # if item.split("/")[1] == model_name: - # provider_list.append(provider) - # else: - # model_dict = get_model_dict(provider) - # if model_name in model_dict.keys(): - # provider_list.append(provider) + if "/" in item and model_name_split == model_name: + new_provider["model"] = [{model_dict[model_name]: model_name}] + # 如果 item 不包含 /，则直接匹配 provider，说明整个渠道所有模型都能用 + provider_list.append(new_provider) + + elif model_name.endswith("*") and "/" in item and model_name_split.startswith(model_name.rstrip("*")): + # old: new + new_provider["model"] = [{model_dict[model_name_split]: model_name}] + provider_list.append(new_provider) + + # print("provider_list", provider_list) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): diff --git a/test/test_dict.py b/test/test_dict.py new file mode 100644 index 00000000..2fea5fd3 --- /dev/null +++ b/test/test_dict.py @@ -0,0 +1,12 @@ +a = [ + {"a": 1, "b": 2, "c": 3}, + {"a": 4, "b": 5, "c": 6}, + {"a": 7, "b": 8, "c": 9} +] +import copy +for item in a: + new_item = copy.deepcopy(item) + new_item["a"] = 10 + del new_item["b"] + # print(item) +print(a) From d91f3fa9ab1b0a75db33882059471f7e8b4ff1b1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Oct 2024 20:02:29 +0000 Subject: [PATCH 158/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.39?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e4a35e8b..12a74d75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.38 +0.0.39 From c50b8cc6d472f689fa75747da93933b01846b342 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 25 Oct 2024 04:19:29 +0800 Subject: [PATCH 159/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20support=20?= =?UTF-8?q?for=20embeddings=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 17 ++++++++++++++--- models.py | 11 ++++++++++- request.py | 23 +++++++++++++++++++++++ utils.py | 1 + 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 22ac3a7e..eddbbf80 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.exceptions import RequestValidationError -from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest, EmbeddingRequest from request import get_payload from response import fetch_response, fetch_response_stream from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder, get_model_dict, save_api_yaml @@ -478,7 +478,7 @@ async def ensure_config(request: Request, call_next): return await call_next(request) # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], provider: Dict, endpoint=None, token=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, token=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -529,6 +529,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "moderation" request.stream = False + if endpoint == "/v1/embeddings": + engine = "embedding" + request.stream = False + if provider.get("engine"): engine = provider["engine"] @@ -700,7 +704,7 @@ def get_matching_providers(self, model_name, token): # print("provider_list", provider_list) return provider_list - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest], token: str, endpoint=None): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): config = app.state.config api_list = app.state.api_list api_index = api_list.index(token) @@ -904,6 +908,13 @@ async def images_generations( ): return await model_handler.request_model(request, token, endpoint="/v1/images/generations") +@app.post("/v1/embeddings", dependencies=[Depends(rate_limit_dependency)]) +async def embeddings( + request: EmbeddingRequest, + token: str = Depends(verify_api_key) +): + return await model_handler.request_model(request, token, endpoint="/v1/embeddings") + @app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) async def moderations( request: ModerationRequest, diff --git a/models.py b/models.py index 26842ade..c8d17777 100644 --- a/models.py +++ b/models.py @@ -111,6 +111,12 @@ class ImageGenerationRequest(BaseRequest): size: Optional[str] = "1024x1024" stream: bool = False +class EmbeddingRequest(BaseRequest): + input: str + model: str + encoding_format: Optional[str] = "float" + stream: bool = False + class AudioTranscriptionRequest(BaseRequest): file: Tuple[str, IOBase, str] model: str @@ -129,7 +135,7 @@ class ModerationRequest(BaseRequest): stream: bool = False class UnifiedRequest(BaseModel): - data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest] + data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest] @model_validator(mode='before') @classmethod @@ -147,6 +153,9 @@ def set_request_type(cls, values): elif "input" in values: values["data"] = ModerationRequest(**values) values["data"].request_type = "moderation" + elif "input" in values: + values["data"] = EmbeddingRequest(**values) + values["data"].request_type = "embedding" else: raise ValueError("无法确定请求类型") return values \ No newline at end of file diff --git a/request.py b/request.py index a2f371e4..241c2344 100644 --- a/request.py +++ b/request.py @@ -1125,6 +1125,27 @@ async def get_moderation_payload(request, engine, provider): return url, headers, payload +async def get_embedding_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + url = provider['base_url'] + url = BaseAPI(url).embeddings + + payload = { + "input": request.input, + "model": model, + } + + if request.encoding_format: + payload["encoding_format"] = request.encoding_format + + return url, headers, payload + async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) @@ -1150,5 +1171,7 @@ async def get_payload(request: RequestModel, engine, provider): return await get_whisper_payload(request, engine, provider) elif engine == "moderation": return await get_moderation_payload(request, engine, provider) + elif engine == "embedding": + return await get_embedding_payload(request, engine, provider) else: raise ValueError("Unknown payload") \ No newline at end of file diff --git a/utils.py b/utils.py index 6bcac081..371b8a03 100644 --- a/utils.py +++ b/utils.py @@ -377,6 +377,7 @@ def __init__( self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/moderations",) + ("",) * 3) + self.embeddings: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/embeddings",) + ("",) * 3) def safe_get(data, *keys, default=None): for key in keys: From 501b1f1712e0f3e7b1f4496db99fa9d06128f9bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 24 Oct 2024 20:19:50 +0000 Subject: [PATCH 160/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.40?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 12a74d75..4fe2fe84 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.39 +0.0.40 From 141a8b83f257bf4e82b9ff8e820433f3abe27a67 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 26 Oct 2024 17:30:24 +0800 Subject: [PATCH 161/476] Default to all models if model field is not set --- For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/yym68686/uni-api?shareId=XXXX-XXXX-XXXX-XXXX). --- utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 371b8a03..e859b643 100644 --- a/utils.py +++ b/utils.py @@ -119,6 +119,10 @@ def update_config(config_data, use_config_url=False): config_data['api_keys'][index]['weights'] = weights_dict config_data['api_keys'][index]['model'] = models api_keys_db[index]['model'] = models + else: + # Default to all models if 'model' field is not set + config_data['api_keys'][index]['model'] = ["all"] + api_keys_db[index]['model'] = ["all"] api_list = [item["api"] for item in api_keys_db] # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False)) @@ -385,4 +389,4 @@ def safe_get(data, *keys, default=None): data = data[key] if isinstance(data, (dict, list)) else data.get(key) except (KeyError, IndexError, AttributeError, TypeError): return default - return data \ No newline at end of file + return data From 9bf0345c6fb303e734a2c0d8c58568e9e12fd4ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 26 Oct 2024 09:31:38 +0000 Subject: [PATCH 162/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4fe2fe84..89a67e8c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.40 +0.0.41 From d2ba74a9451b6fd311d12309656ad46e8cc8dde4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 26 Oct 2024 18:23:02 +0800 Subject: [PATCH 163/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++---- README_CN.md | 3 +-- main.py | 19 +++++++------------ 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index af060f3b..a4addc9f 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,10 @@ providers: - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required, automatically uses base_url and api to get all available models through the /v1/models endpoint. - # Multiple providers can be configured here, each provider can have multiple API Keys, and each API Key can have multiple models configured. + # Multiple providers can be configured here, each provider can configure multiple API Keys, and each API Key can configure multiple models. api_keys: - - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key, required for user requests to uni-api - model: # Models that can be used by this API Key, required. Channel-level round-robin load balancing is enabled by default, and each request to the model follows the order configured in model. It is independent of the original channel order in providers. Therefore, you can set a different request order for each API key. - - all # Can use all models from all channels set under providers, no need to add available channels one by one. If you don't want to set available channels for each api in api_keys, uni-api supports setting the api key to use all models from all channels under providers. + - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key, user request uni-api requires API key, required + # This API Key can use all models, that is, it can use all models in all channels set under providers, without needing to add available channels one by one. ``` Detailed advanced configuration of `api.yaml`: diff --git a/README_CN.md b/README_CN.md index 156c9417..63ac1f7b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -52,8 +52,7 @@ providers: # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个 API Key 可以配置多个模型。 api_keys: - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key，用户请求 uni-api 需要 API key，必填 - model: # 该 API Key 可以使用的模型，必填。默认开启渠道级轮询负载均衡，每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 - - all # 可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。如果你不想在 `api_keys` 里面给每个 `api` 一个个设置可用渠道，`uni-api` 支持将 `api key` 设置为可以使用 providers 下面所有渠道的所有模型。 + # 该 API Key 可以使用所有模型，即可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 ``` `api.yaml` 详细的高级配置： diff --git a/main.py b/main.py index eddbbf80..d82bd869 100644 --- a/main.py +++ b/main.py @@ -631,7 +631,7 @@ def get_matching_providers(self, model_name, token): for model in config['api_keys'][api_index]['model']: if model == "all": - # 如果模型名为 *，则返回所有模型 + # 如果模型名为 all，则返回所有模型 for provider in config["providers"]: model_dict = get_model_dict(provider) for model in model_dict.keys(): @@ -661,7 +661,7 @@ def get_matching_providers(self, model_name, token): # api_keys 中 model 为 provider_name/* 时，表示所有模型都匹配 if model_name_split == "*": if model_name in models_list: - provider_rules.append(provider_name) + provider_rules.append(provider_name + "/" + model_name) # 如果请求模型名： gpt-4* ，则匹配所有以模型名开头且不以 * 结尾的模型 for models_list_model in models_list: @@ -684,21 +684,16 @@ def get_matching_providers(self, model_name, token): # print("provider_rules", provider_rules) for item in provider_rules: for provider in config['providers']: - # print("provider", provider, provider['provider'] == item, item) - if provider['provider'] == item.split("/")[0]: + if "/" in item and provider['provider'] == item.split("/")[0]: new_provider = copy.deepcopy(provider) model_dict = get_model_dict(provider) - # print("model_dict", model_dict) model_name_split = "/".join(item.split("/")[1:]) - if model_name in model_dict.keys(): - if "/" in item and model_name_split == model_name: - new_provider["model"] = [{model_dict[model_name]: model_name}] - # 如果 item 不包含 /，则直接匹配 provider，说明整个渠道所有模型都能用 + # old: new + new_provider["model"] = [{model_dict[model_name_split]: model_name}] + if model_name in model_dict.keys() and model_name_split == model_name: provider_list.append(new_provider) - elif model_name.endswith("*") and "/" in item and model_name_split.startswith(model_name.rstrip("*")): - # old: new - new_provider["model"] = [{model_dict[model_name_split]: model_name}] + elif model_name.endswith("*") and model_name_split.startswith(model_name.rstrip("*")): provider_list.append(new_provider) # print("provider_list", provider_list) From a55b91d34d17348e433150cfd66fd7fc65d5d6ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 26 Oct 2024 10:23:25 +0000 Subject: [PATCH 164/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.42?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 89a67e8c..54a00221 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.41 +0.0.42 From b2991c2cf790c49d8b287532ea296a9433600df5 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 03:13:27 +0800 Subject: [PATCH 165/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20weight=20load=20balancing=20round-robin.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index d82bd869..08408d46 100644 --- a/main.py +++ b/main.py @@ -717,13 +717,14 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques matching_providers = random.sample(matching_providers, num_matching_providers) weights = safe_get(config, 'api_keys', api_index, "weights") - if weights: - # 步骤 1: 提取 matching_providers 中的所有 provider 值 - providers = set(provider['provider'] for provider in matching_providers) - weight_keys = set(weights.keys()) - # 步骤 3: 计算交集 - intersection = providers.intersection(weight_keys) + # 步骤 1: 提取 matching_providers 中的所有 provider 值 + all_providers = set(provider['provider'] for provider in matching_providers) + weight_keys = set(weights.keys()) + # 步骤 3: 计算交集 + intersection = all_providers.intersection(weight_keys) + + if weights and intersection: weights = dict(filter(lambda item: item[0] in intersection, weights.items())) if scheduling_algorithm == "weighted_round_robin": From d088195f2a1e2a82b3312ca1be69b18d46526650 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 27 Oct 2024 19:13:51 +0000 Subject: [PATCH 166/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.43?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 54a00221..5c49468d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.42 +0.0.43 From c9427f326325895b4534b91a4543d89251e05e65 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 07:51:49 +0800 Subject: [PATCH 167/476] =?UTF-8?q?=F0=9F=AA=9E=20Frontend:=20Added=20side?= =?UTF-8?q?bar=20support=20to=20the=20front=20end,=20removed=20the=20menu?= =?UTF-8?q?=20bar.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 📖 Docs: Update documentation --- README.md | 65 +++++++++++++++++----------------- README_CN.md | 11 +++--- main.py | 99 ++++++++++++++++++++++++++++------------------------ 3 files changed, 92 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index a4addc9f..01cf9c4c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ For personal use, one/new-api is too complex with many commercial features that - Support four types of load balancing. 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. - 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. + 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. - Support automatic retry, when an API channel response fails, automatically retry the next API channel. - Support fine-grained permission control. Support using wildcards to set specific models available for API key channels. @@ -59,36 +59,36 @@ Detailed advanced configuration of `api.yaml`: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name, required - base_url: https://api.your.com/v1/chat/completions # Backend service API address, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name can be given, required + base_url: https://api.your.com/v1/chat/completions # API address of the backend service, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # Optional, if model is not configured, all available models will be automatically retrieved through base_url and api via the /v1/models endpoint. - - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional + model: # Optional, if model is not configured, all available models will be automatically obtained via base_url and api through the /v1/models endpoint. + - gpt-4o # Model name that can be used, required + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Renamed model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional - tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Renamed model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional + tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the following line to use the original name + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON-formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON-formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, or can be obtained by viewing service account details in the "IAM & Admin" section of the Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector in Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in Google Cloud Console, generate a JSON format key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, can also be obtained by viewing the service account details in the "IAM and Admin" section of Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -97,14 +97,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # Can include the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes to avoid YAML syntax error, llama-3.1-8b is the renamed name, you can use a simpler name instead of the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes to avoid YAML syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Renamed model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, the model name must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a simple name to replace the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # The model name must be enclosed in quotes, otherwise yaml syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -113,37 +113,38 @@ providers: - causallm-35b-beta2ep-q6k: causallm-35b - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # Force use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional + engine: openrouter # Force to use a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service - model: # The models that this API Key can use, required. Channel-level round-robin load balancing is enabled by default, and each request model is requested in the order configured in the model. It is unrelated to the original channel order in providers. Therefore, you can set different request orders for each API key. - - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers - - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required + model: # The model that this API Key can use, required. Channel-level polling load balancing is enabled by default, each request model is requested in the order configured in the model. It is unrelated to the original channel order in providers. Therefore, you can set different request orders for each API key. + - gpt-4o # Model name that can be used, can use the gpt-4o model provided by all providers + - claude-3-5-sonnet # Model name that can be used, can use the claude-3-5-sonnet model provided by all providers + - gemini/* # Model name that can be used, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models role: admin - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models named claude-3-5-sonnet from other providers cannot be used. This notation will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - - # By adding angle brackets around the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but instead treat the entire anthropic/claude-3-5-sonnet as the model name. This notation can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. - - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for message moderation. + - anthropic/claude-3-5-sonnet # Model name that can be used, can only use the claude-3-5-sonnet model provided by the provider named anthropic. The claude-3-5-sonnet model from other providers cannot be used. This way of writing will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will use the entire anthropic/claude-3-5-sonnet as the model name. This way of writing can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moral review. preferences: - SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, fixed priority scheduling is used, always executing the channel of the first model with a request. Modify the default channel round-robin load balancing. SCHEDULING_ALGORITHM options are: fixed_priority, weighted_round_robin, lottery, random. - # When SCHEDULING_ALGORITHM is random, random round-robin load balancing is used, randomly requesting the channel of the model with a request. + SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the first channel with a request. Enabled by default, the default value of SCHEDULING_ALGORITHM is fixed_priority. Optional values for SCHEDULING_ALGORITHM are: fixed_priority, round_robin, weighted_round_robin, lottery, random. + # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel with the requested model. + # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the user's model in order. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true - RATE_LIMIT: 2/min # Supports rate limiting, the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional - ENABLE_MODERATION: true # Whether to enable message moderation, true to enable, false to disable, default is false, when enabled, user messages will be moderated, and if inappropriate messages are found, an error message will be returned. + RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, messages will be morally reviewed, if inappropriate messages are found, an error message will be returned. # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - gcp1/*: 5 # The number after the colon is the weight, weight only supports positive integers. - - gcp2/*: 3 # The larger the number, the greater the probability of the request. - - gcp3/*: 2 # In this example, there are a total of 10 weights across all channels, and out of 10 requests, 5 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights across all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and if the above channels have weights, requests will be made according to the weighted order. Use weighted round-robin load balancing, request the channel of the model with a request according to the weight order. When SCHEDULING_ALGORITHM is lottery, use lottery round-robin load balancing, request the channel of the model with a request according to the weight randomly. + SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the channels above have weights, requests will be made in the weighted order. Use weighted polling load balancing, request the channel of the model with the request in weight order. When SCHEDULING_ALGORITHM is lottery, use lottery polling load balancing, randomly request the channel of the model with the request according to weight. Channels without weights automatically fall back to round_robin polling load balancing. AUTO_RETRY: true ``` diff --git a/README_CN.md b/README_CN.md index 63ac1f7b..c322eed4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -25,7 +25,7 @@ - 支持四种负载均衡。 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 - 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。自动开启不需要额外配置。 + 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。默认不开启，需要配置 `SCHEDULING_ALGORITHM` 为 `round_robin`。 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 @@ -87,8 +87,8 @@ providers: - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个JSON格式的字符串，包含服务账号的私钥信息。获取方式： 在Google Cloud Console中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在Google Cloud Console的"IAM与管理"部分查看服务账号详情获得。 + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个 JSON 格式的字符串，包含服务账号的私钥信息。获取方式： 在 Google Cloud Console 中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI 服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在 Google Cloud Console 的"IAM与管理"部分查看服务账号详情获得。 model: - gemini-1.5-pro - gemini-1.5-flash @@ -129,8 +129,9 @@ api_keys: - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 - openai-test/text-moderation-latest # 当开启消息道德审查后，可以使用名为 openai-test 渠道下的 text-moderation-latest 模型进行道德审查。 preferences: - SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。修改默认开启的渠道轮询负载均衡。SCHEDULING_ALGORITHM 可选值为：fixed_priority，weighted_round_robin, lottery, random。 + SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。默认开启，SCHEDULING_ALGORITHM 缺省值为 fixed_priority。SCHEDULING_ALGORITHM 可选值有：fixed_priority，round_robin，weighted_round_robin, lottery, random。 # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 + # 当 SCHEDULING_ALGORITHM 为 round_robin 时，使用轮训负载均衡，按照顺序请求用户使用的模型的渠道。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 @@ -143,7 +144,7 @@ api_keys: - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 preferences: - SCHEDULING_ALGORITHM: weighted_round_robin # 仅当 SCHEDULING_ALGORITHM 为 weighted_round_robin 并且上面的渠道如果有权重，会按照加权后的顺序请求。使用加权轮训负载均衡，按照权重顺序请求拥有请求的模型的渠道。当 SCHEDULING_ALGORITHM 为 lottery 时，使用抽奖轮训负载均衡，按照权重随机请求拥有请求的模型的渠道。 + SCHEDULING_ALGORITHM: weighted_round_robin # 仅当 SCHEDULING_ALGORITHM 为 weighted_round_robin 并且上面的渠道如果有权重，会按照加权后的顺序请求。使用加权轮训负载均衡，按照权重顺序请求拥有请求的模型的渠道。当 SCHEDULING_ALGORITHM 为 lottery 时，使用抽奖轮训负载均衡，按照权重随机请求拥有请求的模型的渠道。没设置权重的渠道自动回退到 round_robin 轮训负载均衡。 AUTO_RETRY: true ``` diff --git a/main.py b/main.py index 08408d46..a910da5c 100644 --- a/main.py +++ b/main.py @@ -1077,8 +1077,7 @@ async def get_stats( Menubar, MenubarMenu, MenubarTrigger, MenubarContent, MenubarItem, MenubarSeparator ) -from xue.components import input -from xue.components import dropdown, sheet, form, button, checkbox +from xue.components import input, dropdown, sheet, form, button, checkbox, sidebar from xue.components.model_config_row import model_config_row # import sys # import os @@ -1184,6 +1183,38 @@ async def verify_api_key(x_api_key: str = FastapiForm(...)): else: return Div("无效的API密钥", class_="text-red-500").render() +# 添加侧边栏配置 +sidebar_items = [ + { + "icon": "layout-dashboard", + # "label": "仪表盘", + "label": "Dashboard", + "value": "dashboard", + "hx": {"get": "/dashboard", "target": "#main-content"} + }, + # { + # "icon": "settings", + # # "label": "设置", + # "label": "Settings", + # "value": "settings", + # "hx": {"get": "/settings", "target": "#main-content"} + # }, + # { + # "icon": "database", + # # "label": "数据", + # "label": "Data", + # "value": "data", + # "hx": {"get": "/data", "target": "#main-content"} + # }, + # { + # "icon": "scroll-text", + # # "label": "日志", + # "label": "Logs", + # "value": "logs", + # "hx": {"get": "/logs", "target": "#main-content"} + # } +] + @frontend_router.get("/", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def root(x_api_key: str = Depends(get_api_key)): if not x_api_key: @@ -1200,56 +1231,22 @@ async def root(x_api_key: str = Depends(get_api_key)): }); }); """), - title="Menubar Example" + title="uni-api" ), Body( Div( - Menubar( - MenubarMenu( - MenubarTrigger("File", "file-menu"), - MenubarContent( - MenubarItem("New Tab", shortcut="⌘T"), - MenubarItem("New Window", shortcut="⌘N"), - MenubarItem("New Incognito Window", disabled=True), - MenubarSeparator(), - MenubarItem("Print...", shortcut="⌘P"), - ), - id="file-menu" - ), - MenubarMenu( - MenubarTrigger("Edit", "edit-menu"), - MenubarContent( - MenubarItem("Undo", shortcut="⌘Z"), - MenubarItem("Redo", shortcut="⇧⌘Z"), - MenubarSeparator(), - MenubarItem("Cut"), - MenubarItem("Copy"), - MenubarItem("Paste"), - ), - id="edit-menu" - ), - MenubarMenu( - MenubarTrigger("View", "view-menu"), - MenubarContent( - MenubarItem("Always Show Bookmarks Bar"), - MenubarItem("Always Show Full URLs"), - MenubarSeparator(), - MenubarItem("Reload", shortcut="⌘R"), - MenubarItem("Force Reload", shortcut="⇧⌘R", disabled=True), - MenubarSeparator(), - MenubarItem("Toggle Fullscreen"), - MenubarItem("Hide Sidebar"), - ), - id="view-menu" + sidebar.Sidebar("zap", "uni-api", sidebar_items, is_collapsed=False, active_item="dashboard"), + Div( + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" ), + Div(id="sheet-container"), # sheet加载位置 + id="main-content", + class_="ml-[240px] p-6 transition-[margin] duration-200 ease-in-out" ), - class_="p-4" - ), - Div( - data_table(data_table_columns, app.state.config["providers"], "users-table"), - class_="p-4" + class_="flex" ), - Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 class_="container mx-auto", id="body" ) @@ -1257,6 +1254,16 @@ async def root(x_api_key: str = Depends(get_api_key)): # print(result) return result +@frontend_router.get("/sidebar/toggle", response_class=HTMLResponse) +async def toggle_sidebar(is_collapsed: bool = False): + return sidebar.Sidebar( + "zap", + "uni-api", + sidebar_items, + is_collapsed=not is_collapsed, + active_item="dashboard" + ).render() + @frontend_router.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def get_columns_menu(menu_id: str, row_id: str): columns = [ From 53a76c58328452cd76cf8b1321b946fb31467e7f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 27 Oct 2024 23:52:09 +0000 Subject: [PATCH 168/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.44?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5c49468d..2aa91601 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.43 +0.0.44 From 54d814cdde46e1190f701e29853edc9d27636cab Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 09:41:37 +0800 Subject: [PATCH 169/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20did=20not=20check=20if=20weights=20were=20None.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index a910da5c..7ccd7383 100644 --- a/main.py +++ b/main.py @@ -720,9 +720,12 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # 步骤 1: 提取 matching_providers 中的所有 provider 值 all_providers = set(provider['provider'] for provider in matching_providers) - weight_keys = set(weights.keys()) - # 步骤 3: 计算交集 - intersection = all_providers.intersection(weight_keys) + + intersection = None + if weights and all_providers: + weight_keys = set(weights.keys()) + # 步骤 3: 计算交集 + intersection = all_providers.intersection(weight_keys) if weights and intersection: weights = dict(filter(lambda item: item[0] in intersection, weights.items())) From 9addb238505616f3f77f108bd350e22f74f05398 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Oct 2024 01:41:56 +0000 Subject: [PATCH 170/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.45?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2aa91601..31c01c92 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.44 +0.0.45 From ee53a1b1ac3e839acfc860868cbedb7854075253 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 09:57:53 +0800 Subject: [PATCH 171/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 ---------------- README_CN.md | 16 ---------------- main.py | 17 +++++++++++++++++ 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 01cf9c4c..78fa5a42 100644 --- a/README.md +++ b/README.md @@ -172,22 +172,6 @@ yym68686/uni-api:latest - TIMEOUT: Request timeout, default is 100 seconds. The timeout can control the time needed to switch to the next channel when one channel does not respond. Optional - DISABLE_DATABASE: Whether to disable the database, default is false, optional -## Get statistical data - -Use `/stats` to get the usage statistics of each channel for the past 24 hours. Also include your uni-api admin API key. - -Data includes: - -1. The success rate of each model under each channel, sorted from high to low. -2. The overall success rate of each channel, sorted from high to low. -3. The total number of requests for each model across all channels. -4. The number of requests for each endpoint. -5. The number of requests per IP. - -The `hours` parameter in `/stats?hours=48` allows you to control how many hours of recent data statistics to return. If the `hours` parameter is not provided, it defaults to statistics for the last 24 hours. - -There are other statistical data that you can query yourself by writing SQL in the database. Other data includes: first token time, total processing time for each request, whether each request was successful, whether each request passed content moderation, the text content of each request, the API key for each request, the number of input tokens, and the number of output tokens for each request. - ## Vercel remote deployment [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) diff --git a/README_CN.md b/README_CN.md index c322eed4..ae8b8b6f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -172,22 +172,6 @@ yym68686/uni-api:latest - TIMEOUT: 请求超时时间，默认为 100 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 - DISABLE_DATABASE: 是否禁用数据库，默认为 false，选填 -## 获取统计数据 - -使用 `/stats` 获取最近 24 小时各个渠道的使用情况统计。同时带上 自己的 uni-api 的 admin API key。 - -数据包括： - -1. 每个渠道下面每个模型的成功率，成功率从高到低排序。 -2. 每个渠道总的成功率，成功率从高到低排序。 -3. 每个模型在所有渠道总的请求次数。 -4. 每个端点的请求次数。 -5. 每个ip请求的次数。 - -`/stats?hours=48` 参数 `hours` 可以控制返回最近多少小时的数据统计，不传 `hours` 这个参数，默认统计最近 24 小时的统计数据。 - -还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 - ## Vercel 部署 [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) diff --git a/main.py b/main.py index 7ccd7383..f15e4ec2 100644 --- a/main.py +++ b/main.py @@ -969,6 +969,23 @@ async def get_stats( token: str = Depends(verify_admin_api_key), hours: int = Query(default=24, ge=1, le=720, description="Number of hours to look back for stats (1-720)") ): + ''' + ## 获取统计数据 + + 使用 `/v1/stats` 获取最近 24 小时各个渠道的使用情况统计。同时带上 自己的 uni-api 的 admin API key。 + + 数据包括： + + 1. 每个渠道下面每个模型的成功率，成功率从高到低排序。 + 2. 每个渠道总的成功率，成功率从高到低排序。 + 3. 每个模型在所有渠道总的请求次数。 + 4. 每个端点的请求次数。 + 5. 每个ip请求的次数。 + + `/v1/stats?hours=48` 参数 `hours` 可以控制返回最近多少小时的数据统计，不传 `hours` 这个参数，默认统计最近 24 小时的统计数据。 + + 还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 + ''' if DISABLE_DATABASE: return JSONResponse(content={"stats": {}}) async with async_session() as session: From 2dfadd4ce5a746f1aa65a8ee79db2c1ca8d33e24 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Oct 2024 01:58:24 +0000 Subject: [PATCH 172/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.46?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 31c01c92..1df5b7ec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.45 +0.0.46 From 84daa54bb9b767fb519cf51d70c643483b697977 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 10:08:23 +0800 Subject: [PATCH 173/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Add=20/docs/mark?= =?UTF-8?q?down=20endpoint=20to=20output=20markdown=20API=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/main.py b/main.py index f15e4ec2..4ab3a673 100644 --- a/main.py +++ b/main.py @@ -101,6 +101,39 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan, debug=is_debug) +def generate_markdown_docs(): + openapi_schema = app.openapi() + + markdown = f"# {openapi_schema['info']['title']}\n\n" + markdown += f"Version: {openapi_schema['info']['version']}\n\n" + markdown += f"{openapi_schema['info'].get('description', '')}\n\n" + + markdown += "## API Endpoints\n\n" + + paths = openapi_schema['paths'] + for path, path_info in paths.items(): + for method, operation in path_info.items(): + markdown += f"### {method.upper()} {path}\n\n" + markdown += f"{operation.get('summary', '')}\n\n" + markdown += f"{operation.get('description', '')}\n\n" + + if 'parameters' in operation: + markdown += "Parameters:\n" + for param in operation['parameters']: + markdown += f"- {param['name']} ({param['in']}): {param.get('description', '')}\n" + + markdown += "\n---\n\n" + + return markdown + +@app.get("/docs/markdown") +async def get_markdown_docs(): + markdown = generate_markdown_docs() + return Response( + content=markdown, + media_type="text/markdown" + ) + @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): if exc.status_code == 404: From 7416300bebf2a6a2105b5e34e220b9dc469984a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Oct 2024 02:08:46 +0000 Subject: [PATCH 174/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.47?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1df5b7ec..fad9d0a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.46 +0.0.47 From 5b1ad673f726bb5d6f55de0f2e088389abb55860 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 28 Oct 2024 13:18:06 +0800 Subject: [PATCH 175/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20weight=20polling=20did=20not=20check=20if=20the=20we?= =?UTF-8?q?ight=20channel=20conforms=20to=20the=20request=20model.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 181 ++++++++++++++++++++++++++++++------------------------- utils.py | 4 +- 2 files changed, 100 insertions(+), 85 deletions(-) diff --git a/main.py b/main.py index 4ab3a673..3256b8f8 100644 --- a/main.py +++ b/main.py @@ -647,98 +647,105 @@ def lottery_scheduling(weights): break return selections +def get_provider_rules(model_rule, config, request_model): + provider_rules = [] + if model_rule == "all": + # 如果模型名为 all，则返回所有模型 + for provider in config["providers"]: + model_dict = get_model_dict(provider) + for model in model_dict.keys(): + provider_rules.append(provider["provider"] + "/" + model) + + elif "/" in model_rule: + if model_rule.startswith("<") and model_rule.endswith(">"): + model_rule = model_rule[1:-1] + # 处理带斜杠的模型名 + for provider in config['providers']: + model_dict = get_model_dict(provider) + if model_rule in model_dict.keys(): + provider_rules.append(provider['provider'] + "/" + model_rule) + else: + provider_name = model_rule.split("/")[0] + model_name_split = "/".join(model_rule.split("/")[1:]) + models_list = [] + for provider in config['providers']: + model_dict = get_model_dict(provider) + if provider['provider'] == provider_name: + models_list.extend(list(model_dict.keys())) + # print("models_list", models_list) + # print("model_name", model_name) + # print("model_name_split", model_name_split) + # print("model", model) + + # api_keys 中 model 为 provider_name/* 时，表示所有模型都匹配 + if model_name_split == "*": + if request_model in models_list: + provider_rules.append(provider_name + "/" + request_model) + + # 如果请求模型名： gpt-4* ，则匹配所有以模型名开头且不以 * 结尾的模型 + for models_list_model in models_list: + if request_model.endswith("*") and models_list_model.startswith(request_model.rstrip("*")): + provider_rules.append(provider_name + "/" + models_list_model) + + # api_keys 中 model 为 provider_name/model_name 时，表示模型名完全匹配 + elif model_name_split == request_model \ + or (request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*"))): # api_keys 中 model 为 provider_name/model_name 时，请求模型名： model_name* + if model_name_split in models_list: + provider_rules.append(provider_name + "/" + model_name_split) + + else: + for provider in config["providers"]: + model_dict = get_model_dict(provider) + if model_rule in model_dict.keys(): + provider_rules.append(provider["provider"] + "/" + model_rule) + + return provider_rules + +def get_provider_list(provider_rules, config, request_model): + provider_list = [] + # print("provider_rules", provider_rules) + for item in provider_rules: + for provider in config['providers']: + if "/" in item and provider['provider'] == item.split("/")[0]: + new_provider = copy.deepcopy(provider) + model_dict = get_model_dict(provider) + model_name_split = "/".join(item.split("/")[1:]) + # old: new + new_provider["model"] = [{model_dict[model_name_split]: request_model}] + if request_model in model_dict.keys() and model_name_split == request_model: + provider_list.append(new_provider) + + elif request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*")): + provider_list.append(new_provider) + return provider_list + +def get_matching_providers(request_model, config, api_index): + provider_rules = [] + + for model_rule in config['api_keys'][api_index]['model']: + provider_rules.extend(get_provider_rules(model_rule, config, request_model)) + + provider_list = get_provider_list(provider_rules, config, request_model) + + # print("provider_list", provider_list) + return provider_list + import asyncio class ModelRequestHandler: def __init__(self): self.last_provider_indices = defaultdict(lambda: -1) self.locks = defaultdict(asyncio.Lock) - def get_matching_providers(self, model_name, token): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): config = app.state.config - # api_keys_db = app.state.api_keys_db api_list = app.state.api_list api_index = api_list.index(token) + if not safe_get(config, 'api_keys', api_index, 'model'): raise HTTPException(status_code=404, detail="No matching model found") - provider_rules = [] - - for model in config['api_keys'][api_index]['model']: - if model == "all": - # 如果模型名为 all，则返回所有模型 - for provider in config["providers"]: - model_dict = get_model_dict(provider) - for model in model_dict.keys(): - provider_rules.append(provider["provider"] + "/" + model) - break - if "/" in model: - if model.startswith("<") and model.endswith(">"): - model = model[1:-1] - # 处理带斜杠的模型名 - for provider in config['providers']: - model_dict = get_model_dict(provider) - if model in model_dict.keys(): - provider_rules.append(provider['provider'] + "/" + model) - else: - provider_name = model.split("/")[0] - model_name_split = "/".join(model.split("/")[1:]) - models_list = [] - for provider in config['providers']: - model_dict = get_model_dict(provider) - if provider['provider'] == provider_name: - models_list.extend(list(model_dict.keys())) - # print("models_list", models_list) - # print("model_name", model_name) - # print("model_name_split", model_name_split) - # print("model", model) - - # api_keys 中 model 为 provider_name/* 时，表示所有模型都匹配 - if model_name_split == "*": - if model_name in models_list: - provider_rules.append(provider_name + "/" + model_name) - - # 如果请求模型名： gpt-4* ，则匹配所有以模型名开头且不以 * 结尾的模型 - for models_list_model in models_list: - if model_name.endswith("*") and models_list_model.startswith(model_name.rstrip("*")): - provider_rules.append(provider_name + "/" + models_list_model) - - # api_keys 中 model 为 provider_name/model_name 时，表示模型名完全匹配 - elif model_name_split == model_name \ - or (model_name.endswith("*") and model_name_split.startswith(model_name.rstrip("*"))): # api_keys 中 model 为 provider_name/model_name 时，请求模型名： model_name* - if model_name_split in models_list: - provider_rules.append(provider_name + "/" + model_name_split) - - else: - for provider in config["providers"]: - model_dict = get_model_dict(provider) - if model in model_dict.keys(): - provider_rules.append(provider["provider"] + "/" + model) - - provider_list = [] - # print("provider_rules", provider_rules) - for item in provider_rules: - for provider in config['providers']: - if "/" in item and provider['provider'] == item.split("/")[0]: - new_provider = copy.deepcopy(provider) - model_dict = get_model_dict(provider) - model_name_split = "/".join(item.split("/")[1:]) - # old: new - new_provider["model"] = [{model_dict[model_name_split]: model_name}] - if model_name in model_dict.keys() and model_name_split == model_name: - provider_list.append(new_provider) - - elif model_name.endswith("*") and model_name_split.startswith(model_name.rstrip("*")): - provider_list.append(new_provider) - - # print("provider_list", provider_list) - return provider_list - - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): - config = app.state.config - api_list = app.state.api_list - api_index = api_list.index(token) - model_name = request.model - matching_providers = self.get_matching_providers(model_name, token) + request_model = request.model + matching_providers = get_matching_providers(request_model, config, api_index) num_matching_providers = len(matching_providers) if not matching_providers: @@ -757,6 +764,13 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques intersection = None if weights and all_providers: weight_keys = set(weights.keys()) + provider_rules = [] + for model_rule in weight_keys: + provider_rules.extend(get_provider_rules(model_rule, config, request_model)) + provider_list = get_provider_list(provider_rules, config, request_model) + weight_keys = set([provider['provider'] for provider in provider_list]) + # print("all_providers", all_providers) + # print("weights", weight_keys) # 步骤 3: 计算交集 intersection = all_providers.intersection(weight_keys) @@ -769,6 +783,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques weighted_provider_name_list = lottery_scheduling(weights) else: weighted_provider_name_list = list(weights.keys()) + # print("weighted_provider_name_list", weighted_provider_name_list) new_matching_providers = [] for provider_name in weighted_provider_name_list: @@ -786,9 +801,9 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques start_index = 0 if scheduling_algorithm != "fixed_priority": - async with self.locks[model_name]: - self.last_provider_indices[model_name] = (self.last_provider_indices[model_name] + 1) % num_matching_providers - start_index = self.last_provider_indices[model_name] + async with self.locks[request_model]: + self.last_provider_indices[request_model] = (self.last_provider_indices[request_model] + 1) % num_matching_providers + start_index = self.last_provider_indices[request_model] auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) diff --git a/utils.py b/utils.py index e859b643..7ac7bf49 100644 --- a/utils.py +++ b/utils.py @@ -109,9 +109,9 @@ def update_config(config_data, use_config_url=False): for model in api_key.get('model'): if isinstance(model, dict): key, value = list(model.items())[0] - provider_name = key.split("/")[0] + # provider_name = key.split("/")[0] if "/" in key: - weights_dict.update({provider_name: int(value)}) + weights_dict.update({key: int(value)}) models.append(key) if isinstance(model, str): models.append(model) From 44cd6f61ba15d5b5fe937d17c15029b8aba888bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Oct 2024 05:18:30 +0000 Subject: [PATCH 176/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.48?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fad9d0a8..a24809ad 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.47 +0.0.48 From 8583e535fa21594705e5439e421c9902a974044f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 30 Oct 2024 02:05:43 +0800 Subject: [PATCH 177/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20weight=20polling=20cannot=20match=20the=20model.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 3256b8f8..439784ce 100644 --- a/main.py +++ b/main.py @@ -759,7 +759,9 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques weights = safe_get(config, 'api_keys', api_index, "weights") # 步骤 1: 提取 matching_providers 中的所有 provider 值 - all_providers = set(provider['provider'] for provider in matching_providers) + # print("matching_providers", matching_providers) + # print(type(matching_providers[0]['model'][0].keys()), list(matching_providers[0]['model'][0].keys())[0], matching_providers[0]['model'][0].keys()) + all_providers = set(provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in matching_providers) intersection = None if weights and all_providers: @@ -768,21 +770,25 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques for model_rule in weight_keys: provider_rules.extend(get_provider_rules(model_rule, config, request_model)) provider_list = get_provider_list(provider_rules, config, request_model) - weight_keys = set([provider['provider'] for provider in provider_list]) + weight_keys = set([provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in provider_list]) # print("all_providers", all_providers) - # print("weights", weight_keys) + # print("weights", weights) + # print("weight_keys", weight_keys) + # 步骤 3: 计算交集 intersection = all_providers.intersection(weight_keys) + # print("intersection", intersection) if weights and intersection: - weights = dict(filter(lambda item: item[0] in intersection, weights.items())) + filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k in intersection} + # print("filtered_weights", filtered_weights) if scheduling_algorithm == "weighted_round_robin": - weighted_provider_name_list = weighted_round_robin(weights) + weighted_provider_name_list = weighted_round_robin(filtered_weights) elif scheduling_algorithm == "lottery": - weighted_provider_name_list = lottery_scheduling(weights) + weighted_provider_name_list = lottery_scheduling(filtered_weights) else: - weighted_provider_name_list = list(weights.keys()) + weighted_provider_name_list = list(filtered_weights.keys()) # print("weighted_provider_name_list", weighted_provider_name_list) new_matching_providers = [] From ffacd7425bad45250ad6c107ce5e2d6afc9a7294 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 29 Oct 2024 18:06:14 +0000 Subject: [PATCH 178/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.49?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a24809ad..50f402a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.48 +0.0.49 From 2f5446239c486afc45ee5050e5802a8a6bbeba92 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 30 Oct 2024 02:14:53 +0800 Subject: [PATCH 179/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20model=20cannot=20be=20found.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 439784ce..c499aecd 100644 --- a/main.py +++ b/main.py @@ -706,11 +706,15 @@ def get_provider_list(provider_rules, config, request_model): # print("provider_rules", provider_rules) for item in provider_rules: for provider in config['providers']: - if "/" in item and provider['provider'] == item.split("/")[0]: + model_dict = get_model_dict(provider) + model_name_split = "/".join(item.split("/")[1:]) + if "/" in item and provider['provider'] == item.split("/")[0] and model_name_split in model_dict.keys(): new_provider = copy.deepcopy(provider) - model_dict = get_model_dict(provider) - model_name_split = "/".join(item.split("/")[1:]) # old: new + # print("item", item) + # print("model_dict", model_dict) + # print("model_name_split", model_name_split) + # print("request_model", request_model) new_provider["model"] = [{model_dict[model_name_split]: request_model}] if request_model in model_dict.keys() and model_name_split == request_model: provider_list.append(new_provider) From 4d2d046225fc0eb7d732caf93533e450f34665a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 29 Oct 2024 18:15:21 +0000 Subject: [PATCH 180/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.50?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 50f402a4..85ab4c64 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.49 +0.0.50 From 25f3a29647ea86461338ab8fdc2cafebb14edd63 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 30 Oct 2024 19:02:59 +0800 Subject: [PATCH 181/476] =?UTF-8?q?=F0=9F=92=B0=20Sponsors:=20Thanks=20to?= =?UTF-8?q?=20@PowerHunter=20for=20the=20CNY=20200=20sponsorship,=20sponso?= =?UTF-8?q?rship=20information=20has=20been=20added=20to=20the=20README.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💻 Code: Delete log display of non-streaming output. --- README.md | 28 ++++++++++++++++++++++++---- README_CN.md | 22 +++++++++++++++++++++- main.py | 4 ++-- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 78fa5a42..7c34469b 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,10 @@ For personal use, one/new-api is too complex with many commercial features that - Support OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Support OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Support four types of load balancing. - 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. - 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. - 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. - 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. +1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. +2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. +3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. +4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. - Support automatic retry, when an API channel response fails, automatically retry the next API channel. - Support fine-grained permission control. Support using wildcards to set specific models available for API key channels. - Support rate limiting, you can set the maximum number of requests per minute as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. @@ -301,6 +301,26 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` +## Sponsors + +We thank the following sponsors for their support: + +- @PowerHunter: ¥200 + +## How to sponsor us + +If you would like to support our project, you can sponsor us in the following ways: + +1. [PayPal](https://www.paypal.me/yym68686) + +2. [USDT-TRC20](https://pb.yym68686.top/~USDT-TRC20), USDT-TRC20 wallet address: `TLFbqSv5pDu5he43mVmK1dNx7yBMFeN7d8` + +3. [WeChat](https://pb.yym68686.top/~wechat) + +4. [Alipay](https://pb.yym68686.top/~alipay) + +Thank you for your support! + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index ae8b8b6f..6d7d4dfc 100644 --- a/README_CN.md +++ b/README_CN.md @@ -301,7 +301,27 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` -## ⭐ Star History +## 赞助商 + +我们感谢以下赞助商的支持： + +- @PowerHunter：¥200 + +## 如何赞助我们 + +如果您想支持我们的项目，您可以通过以下方式赞助我们： + +1. [PayPal](https://www.paypal.me/yym68686) + +2. [USDT-TRC20](https://pb.yym68686.top/~USDT-TRC20)，USDT-TRC20 钱包地址：`TLFbqSv5pDu5he43mVmK1dNx7yBMFeN7d8` + +3. [微信](https://pb.yym68686.top/~wechat) + +4. [支付宝](https://pb.yym68686.top/~alipay) + +感谢您的支持！ + +## ⭐ Star 历史 Star History Chart diff --git a/main.py b/main.py index c499aecd..da2f1b44 100644 --- a/main.py +++ b/main.py @@ -317,7 +317,7 @@ async def _logging_iterator(self): chunk = chunk.encode('utf-8') line = chunk.decode('utf-8') if is_debug: - logger.info(f"{line}") + logger.info(f"{line.encode('utf-8').decode('unicode_escape')}") if line.startswith("data:"): line = line.lstrip("data: ") if not line.startswith("[DONE]") and not line.startswith("OK"): @@ -590,7 +590,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A wrapped_generator, first_response_time = await error_handling_wrapper(generator) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") - print("first_element", first_element) + # print("first_element", first_element) first_element = json.loads(first_element) response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") # response = JSONResponse(first_element) From 1e88a6629568c73e2ee096827c3584bbaf47210a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Oct 2024 11:03:24 +0000 Subject: [PATCH 182/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.51?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 85ab4c64..c4132bc6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.50 +0.0.51 From 4f0c23d237c3f5094e00e896d3ba36607352fb6d Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 30 Oct 2024 22:50:58 +0800 Subject: [PATCH 183/476] =?UTF-8?q?=F0=9F=AA=9E=20Frontend:=20Add=20reques?= =?UTF-8?q?t=20model=20statistics=20bar=20chart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 Bug: Fix the bug where the error reason is not displayed due to a timeout. ✨ Feature: Add feature: support custom timeout for different models --- README.md | 72 +++++++------- README_CN.md | 8 ++ main.py | 266 ++++++++++++++++++++++++++++++++++++++++++--------- utils.py | 17 ---- 4 files changed, 270 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 7c34469b..d0a549f4 100644 --- a/README.md +++ b/README.md @@ -59,36 +59,36 @@ Detailed advanced configuration of `api.yaml`: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name can be given, required - base_url: https://api.your.com/v1/chat/completions # API address of the backend service, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name is fine, required + base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # Optional, if model is not configured, all available models will be automatically obtained via base_url and api through the /v1/models endpoint. - - gpt-4o # Model name that can be used, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Renamed model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional + model: # Optional, if the model is not configured, all available models will be automatically obtained through the /v1/models endpoint via base_url and api. + - gpt-4o # Usable model name, required + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, a simpler name can replace the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Renamed model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional - tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, a simpler name can replace the original complex name, optional + tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name - - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the following line to use the original name. + - gemini-1.5-flash-exp-0827 # Adding this line allows both gemini-1.5-flash-exp-0827 and gemini-1.5-flash to be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector in Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in Google Cloud Console, generate a JSON format key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, can also be obtained by viewing the service account details in the "IAM and Admin" section of Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: A string usually consisting of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the service account's private key information. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, or can be found in the "IAM & Admin" section of the Google Cloud Console to view service account details. model: - gemini-1.5-pro - gemini-1.5-flash @@ -97,14 +97,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # Can include the provider's website, remarks, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Renamed model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, the model name must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a simple name to replace the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # The model name must be enclosed in quotes, otherwise yaml syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes to avoid YAML syntax error, llama-3.1-8b is the renamed name, a simpler name can replace the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes to avoid YAML syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -113,39 +113,47 @@ providers: - causallm-35b-beta2ep-q6k: causallm-35b - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # Force to use a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional + engine: openrouter # Force using a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required - model: # The model that this API Key can use, required. Channel-level polling load balancing is enabled by default, each request model is requested in the order configured in the model. It is unrelated to the original channel order in providers. Therefore, you can set different request orders for each API key. - - gpt-4o # Model name that can be used, can use the gpt-4o model provided by all providers - - claude-3-5-sonnet # Model name that can be used, can use the claude-3-5-sonnet model provided by all providers - - gemini/* # Model name that can be used, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models + model: # Models that can be used with this API Key, required. By default, channel-level round-robin load balancing is enabled, and each request is made in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request orders for each API key. + - gpt-4o # Usable model name, can use all gpt-4o models provided by providers + - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers + - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models role: admin - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Model name that can be used, can only use the claude-3-5-sonnet model provided by the provider named anthropic. The claude-3-5-sonnet model from other providers cannot be used. This way of writing will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - - # By adding angle brackets on both sides of the model name, it will not look for the claude-3-5-sonnet model under the channel named anthropic, but will use the entire anthropic/claude-3-5-sonnet as the model name. This way of writing can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. - - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moral review. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models named claude-3-5-sonnet from other providers cannot be used. This syntax will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets around the model name, it will not search for the claude-3-5-sonnet model under the channel named anthropic, but will treat the entire anthropic/claude-3-5-sonnet as the model name. This syntax can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moderation. preferences: - SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the first channel with a request. Enabled by default, the default value of SCHEDULING_ALGORITHM is fixed_priority. Optional values for SCHEDULING_ALGORITHM are: fixed_priority, round_robin, weighted_round_robin, lottery, random. - # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel with the requested model. - # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the user's model in order. + SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, fixed priority scheduling is used, always executing the channel of the first requested model. Enabled by default, the default value of SCHEDULING_ALGORITHM is fixed_priority. Optional values for SCHEDULING_ALGORITHM are: fixed_priority, round_robin, weighted_round_robin, lottery, random. + # When SCHEDULING_ALGORITHM is random, random round-robin load balancing is used, randomly requesting the channel of the requested model. + # When SCHEDULING_ALGORITHM is round_robin, round-robin load balancing is used, requesting the user's model channels in order. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true - RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional - ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, messages will be morally reviewed, if inappropriate messages are found, an error message will be returned. + RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default 60/min, optional + ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, messages will be moderated, and inappropriate messages will return an error. # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, weight only supports positive integers. - - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of the request. + - gcp1/*: 5 # The number after the colon is the weight, weights only support positive integers. + - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of request. - gcp3/*: 2 # In this example, there are a total of 10 weights across all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the channels above have weights, requests will be made in the weighted order. Use weighted polling load balancing, request the channel of the model with the request in weight order. When SCHEDULING_ALGORITHM is lottery, use lottery polling load balancing, randomly request the channel of the model with the request according to weight. Channels without weights automatically fall back to round_robin polling load balancing. + SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the above channels have weights, requests will be made in the weighted order. Using weighted round-robin load balancing, requests are made in the order of weight for the channel of the requested model. When SCHEDULING_ALGORITHM is lottery, lottery round-robin load balancing is used, randomly requesting the channel of the requested model according to weight. Channels without weights automatically fall back to round_robin round-robin load balancing. AUTO_RETRY: true + +preferences: # Global configuration + model_timeout: # Model timeout, in seconds, default 100 seconds, optional + gpt-4o: 10 # Model gpt-4o timeout is 10 seconds, gpt-4o is the model name, when requesting models like gpt-4o-2024-08-06, the timeout is also 10 seconds + claude-3-5-sonnet: 10 # Model claude-3-5-sonnet timeout is 10 seconds, when requesting models like claude-3-5-sonnet-20240620, the timeout is also 10 seconds + default: 10 # If the model does not have a timeout set, the default timeout of 10 seconds is used, when requesting models not in model_timeout, the default timeout is 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, which is 100 seconds + o1-mini: 30 # Model o1-mini timeout is 30 seconds, when requesting models with names starting with o1-mini, the timeout is 30 seconds + o1-preview: 100 # Model o1-preview timeout is 100 seconds, when requesting models with names starting with o1-preview, the timeout is 100 seconds ``` Mount the configuration file and start the uni-api docker container: diff --git a/README_CN.md b/README_CN.md index 6d7d4dfc..58a65d4b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -146,6 +146,14 @@ api_keys: preferences: SCHEDULING_ALGORITHM: weighted_round_robin # 仅当 SCHEDULING_ALGORITHM 为 weighted_round_robin 并且上面的渠道如果有权重，会按照加权后的顺序请求。使用加权轮训负载均衡，按照权重顺序请求拥有请求的模型的渠道。当 SCHEDULING_ALGORITHM 为 lottery 时，使用抽奖轮训负载均衡，按照权重随机请求拥有请求的模型的渠道。没设置权重的渠道自动回退到 round_robin 轮训负载均衡。 AUTO_RETRY: true + +preferences: # 全局配置 + model_timeout: # 模型超时时间，单位为秒，默认 100 秒，选填 + gpt-4o: 10 # 模型 gpt-4o 的超时时间为 10 秒,gpt-4o 是模型名称，当请求 gpt-4o-2024-08-06 等模型时，超时时间也是 10 秒 + claude-3-5-sonnet: 10 # 模型 claude-3-5-sonnet 的超时时间为 10 秒，当请求 claude-3-5-sonnet-20240620 等模型时，超时时间也是 10 秒 + default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用 环境变量 TIMEOUT 设置的默认超时时间，默认超时时间是 100 秒 + o1-mini: 30 # 模型 o1-mini 的超时时间为 30 秒，当请求名字是 o1-mini 开头的模型时，超时时间是 30 秒 + o1-preview: 100 # 模型 o1-preview 的超时时间为 100 秒，当请求名字是 o1-preview 开头的模型时，超时时间是 100 秒 ``` 挂载配置文件并启动 uni-api docker 容器： diff --git a/main.py b/main.py index da2f1b44..afb4441c 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,7 @@ import string import json +DEFAULT_TIMEOUT = float(os.getenv("TIMEOUT", 100)) is_debug = bool(os.getenv("DEBUG", False)) # is_debug = False @@ -97,7 +98,9 @@ async def lifespan(app: FastAPI): yield # 关闭时的代码 - await app.state.client.aclose() + # await app.state.client.aclose() + if hasattr(app.state, 'client_manager'): + await app.state.client_manager.close() app = FastAPI(lifespan=lifespan, debug=is_debug) @@ -493,6 +496,49 @@ async def moderate_content(self, content, token): app.add_middleware(StatsMiddleware) +class ClientManager: + def __init__(self, pool_size=100): + self.pool_size = pool_size + self.clients = {} # {timeout_value: AsyncClient} + self.locks = {} # {timeout_value: Lock} + + async def init(self, default_config): + self.default_config = default_config + + @asynccontextmanager + async def get_client(self, timeout_value): + # 对同一超时值的客户端加锁 + if timeout_value not in self.locks: + self.locks[timeout_value] = asyncio.Lock() + + async with self.locks[timeout_value]: + # 获取或创建指定超时值的客户端 + if timeout_value not in self.clients: + timeout = httpx.Timeout( + connect=15.0, + read=timeout_value, + write=30.0, + pool=self.pool_size + ) + self.clients[timeout_value] = httpx.AsyncClient( + timeout=timeout, + limits=httpx.Limits(max_connections=self.pool_size), + **self.default_config + ) + + try: + yield self.clients[timeout_value] + except Exception as e: + # 如果客户端出现问题，关闭并重新创建 + await self.clients[timeout_value].aclose() + del self.clients[timeout_value] + raise e + + async def close(self): + for client in self.clients.values(): + await client.aclose() + self.clients.clear() + @app.middleware("http") async def ensure_config(request: Request, call_next): if not hasattr(app.state, 'config'): @@ -508,6 +554,32 @@ async def ensure_config(request: Request, call_next): else: raise Exception("No admin API key found") + if app and not hasattr(app.state, 'client_manager'): + + default_config = { + "headers": { + "User-Agent": "curl/7.68.0", + "Accept": "*/*", + }, + "http2": True, + "verify": True, + "follow_redirects": True + } + + # 初始化客户端管理器 + app.state.client_manager = ClientManager(pool_size=200) + await app.state.client_manager.init(default_config) + + # 存储超时配置 + app.state.timeouts = {} + if app.state.config and 'preferences' in app.state.config: + for model_name, timeout_value in app.state.config['preferences'].get('model_timeout', {}).items(): + app.state.timeouts[model_name] = timeout_value + if "default" not in app.state.config['preferences'].get('model_timeout', {}): + app.state.timeouts["default"] = DEFAULT_TIMEOUT + + print("app.state.timeouts", app.state.timeouts) + return await call_next(request) # 在 process_request 函数中更新成功和失败计数 @@ -578,32 +650,51 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A pass else: logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) + current_info = request_info.get() + model = model_dict[request.model] + + timeout_value = None + # 先尝试精确匹配 + + if model in app.state.timeouts: + timeout_value = app.state.timeouts[model] + else: + # 如果没有精确匹配，尝试模糊匹配 + for timeout_model in app.state.timeouts: + if timeout_model in model: + timeout_value = app.state.timeouts[timeout_model] + break + + # 如果都没匹配到，使用默认值 + if timeout_value is None: + timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) + try: - model = model_dict[request.model] - if request.stream: - generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator) - response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") - else: - generator = fetch_response(app.state.client, url, headers, payload, engine, model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator) - first_element = await anext(wrapped_generator) - first_element = first_element.lstrip("data: ") - # print("first_element", first_element) - first_element = json.loads(first_element) - response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") - # response = JSONResponse(first_element) - - # 更新成功计数和首次响应时间 - await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) - # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) - current_info["first_response_time"] = first_response_time - current_info["success"] = True - current_info["provider"] = provider['provider'] + async with app.state.client_manager.get_client(timeout_value) as client: + if request.stream: + generator = fetch_response_stream(client, url, headers, payload, engine, model) + wrapped_generator, first_response_time = await error_handling_wrapper(generator) + response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") + else: + generator = fetch_response(client, url, headers, payload, engine, model) + wrapped_generator, first_response_time = await error_handling_wrapper(generator) + first_element = await anext(wrapped_generator) + first_element = first_element.lstrip("data: ") + # print("first_element", first_element) + first_element = json.loads(first_element) + response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") + # response = JSONResponse(first_element) + + # 更新成功计数和首次响应时间 + await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + current_info["first_response_time"] = first_response_time + current_info["success"] = True + current_info["provider"] = provider['provider'] + return response - return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) @@ -823,25 +914,36 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques try: response = await process_request(request, provider, endpoint, token) return response - except HTTPException as e: - logger.error(f"Error with provider {provider['provider']}: {str(e)}") - status_code = e.status_code - error_message = e.detail - - if auto_retry: - continue + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: + + # 根据异常类型设置状态码和错误消息 + if isinstance(e, httpx.ReadTimeout): + status_code = 504 # Gateway Timeout + error_message = "Request timed out" + elif isinstance(e, httpx.ReadError): + status_code = 502 # Bad Gateway + error_message = "Network read error" + elif isinstance(e, httpx.RemoteProtocolError): + status_code = 502 # Bad Gateway + error_message = "Remote protocol error" + elif isinstance(e, asyncio.CancelledError): + status_code = 499 # Client Closed Request + error_message = "Request was cancelled" + elif isinstance(e, HTTPException): + status_code = e.status_code + error_message = str(e.detail) else: - raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") - except (Exception, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e: - logger.error(f"Error with provider {provider['provider']}: {str(e)}") + status_code = 500 # Internal Server Error + error_message = str(e) or f"Unknown error: {e.__class__.__name__}" + + logger.error(f"Error {status_code} with provider {provider['provider']}: {error_message}") if is_debug: import traceback traceback.print_exc() - error_message = str(e) if auto_retry: continue else: - raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}") + raise HTTPException(status_code=status_code, detail=f"Error: Current provider response failed: {error_message}") current_info = request_info.get() current_info["first_response_time"] = -1 @@ -1155,7 +1257,7 @@ async def get_stats( Menubar, MenubarMenu, MenubarTrigger, MenubarContent, MenubarItem, MenubarSeparator ) -from xue.components import input, dropdown, sheet, form, button, checkbox, sidebar +from xue.components import input, dropdown, sheet, form, button, checkbox, sidebar, chart from xue.components.model_config_row import model_config_row # import sys # import os @@ -1277,13 +1379,13 @@ async def verify_api_key(x_api_key: str = FastapiForm(...)): # "value": "settings", # "hx": {"get": "/settings", "target": "#main-content"} # }, - # { - # "icon": "database", - # # "label": "数据", - # "label": "Data", - # "value": "data", - # "hx": {"get": "/data", "target": "#main-content"} - # }, + { + "icon": "database", + # "label": "数据", + "label": "Data", + "value": "data", + "hx": {"get": "/data", "target": "#main-content"} + }, # { # "icon": "scroll-text", # # "label": "日志", @@ -1342,6 +1444,82 @@ async def toggle_sidebar(is_collapsed: bool = False): active_item="dashboard" ).render() +@frontend_router.get("/data", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def data_page(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + if DISABLE_DATABASE: + return HTMLResponse("数据库已禁用") + + async with async_session() as session: + # 计算过去24小时的开始时间 + start_time = datetime.now(timezone.utc) - timedelta(hours=24) + + # 获取每个模型的请求数据 + model_stats = await session.execute( + select( + RequestStat.model, + RequestStat.provider, + func.count().label('count') + ) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.model, RequestStat.provider) + .order_by(desc('count')) + ) + model_stats = model_stats.fetchall() + + # 处理数据以适配图表格式 + chart_data = [] + providers = list(set(stat.provider for stat in model_stats)) + models = list(set(stat.model for stat in model_stats)) + + for model in models: + data_point = {"model": model} + for provider in providers: + count = next( + (stat.count for stat in model_stats + if stat.model == model and stat.provider == provider), + 0 + ) + data_point[provider] = count + chart_data.append(data_point) + + # 定义图表系列 + series = [ + {"name": provider, "data_key": provider} + for provider in providers + ] + + # 图表配置 + chart_config = { + "stacked": True, # 堆叠柱状图 + "horizontal": False, + "colors": [f"hsl({i * 360 / len(providers)}, 70%, 50%)" for i in range(len(providers))], # 生成不同的颜色 + "grid": True, + "legend": True, + "tooltip": True + } + + result = HTML( + Head(title="数据统计"), + Body( + Div( + Div( + "模型使用统计 (24小时)", + class_="text-2xl font-bold mb-4" + ), + Div( + chart.bar_chart("model-usage-chart", chart_data, "model", series, chart_config), + class_="h-[600px]" # 设置图表高度 + ), + class_="container mx-auto p-4" + ) + ) + ).render() + + return result + @frontend_router.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def get_columns_menu(menu_id: str, row_id: str): columns = [ diff --git a/utils.py b/utils.py index 7ac7bf49..42dab548 100644 --- a/utils.py +++ b/utils.py @@ -130,23 +130,6 @@ def update_config(config_data, use_config_url=False): # 读取YAML配置文件 async def load_config(app=None): - - if app and not hasattr(app.state, 'client'): - import os - TIMEOUT = float(os.getenv("TIMEOUT", 100)) - timeout = httpx.Timeout(connect=15.0, read=TIMEOUT, write=30.0, pool=30.0) - default_headers = { - "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent - "Accept": "*/*", # curl 的默认 Accept 头 - } - app.state.client = httpx.AsyncClient( - timeout=timeout, - headers=default_headers, - http2=True, # 禁用 HTTP/2 - verify=True, # 保持 SSL 验证（如需禁用，设为 False，但不建议） - follow_redirects=True, # 自动跟随重定向 - ) - try: with open(API_YAML_PATH, 'r', encoding='utf-8') as file: conf = yaml.load(file) From c084ed123e9c5f15ccecedd207995daec7d31275 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Oct 2024 14:51:18 +0000 Subject: [PATCH 184/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.52?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c4132bc6..62077419 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.51 +0.0.52 From a20c06f87e586b1dc5f0acca6b9500c922bac611 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 31 Oct 2024 03:08:18 +0800 Subject: [PATCH 185/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20moral=20check=20database=20update=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index afb4441c..98b852f2 100644 --- a/main.py +++ b/main.py @@ -429,7 +429,7 @@ async def dispatch(self, request: Request, call_next): process_time = time() - start_time current_info["process_time"] = process_time current_info["is_flagged"] = is_flagged - await self.update_stats(current_info) + await update_stats(current_info) return JSONResponse( status_code=400, content={"error": "Content did not pass the moral check, please modify and try again."} From a6c11f18aeee5ce2618825c0cf9d56ccd15c5cdb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Oct 2024 19:08:41 +0000 Subject: [PATCH 186/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.53?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 62077419..f2e654c1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.52 +0.0.53 From 3035b10f121c8a682e01c087eaa3fcb354b3e633 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 31 Oct 2024 16:54:47 +0800 Subject: [PATCH 187/476] =?UTF-8?q?=F0=9F=AA=9E=20Frontend:=20Fixed=20the?= =?UTF-8?q?=20bug=20where=20the=20chart=20size=20was=20displayed=20incorre?= =?UTF-8?q?ctly=20when=20there=20was=20no=20data.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 Bug: Fix the bug where the model cannot be retrieved in weight polling after renaming the provider model. --- main.py | 71 ++++++++++++++++++++++++++++++++++++++++---------------- utils.py | 15 +++++++++--- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/main.py b/main.py index 98b852f2..bfda9afa 100644 --- a/main.py +++ b/main.py @@ -1253,10 +1253,6 @@ async def get_stats( from typing import Optional, List from xue import HTML, Head, Body, Div, xue_initialize, Script, Ul, Li -from xue.components.menubar import ( - Menubar, MenubarMenu, MenubarTrigger, MenubarContent, - MenubarItem, MenubarSeparator -) from xue.components import input, dropdown, sheet, form, button, checkbox, sidebar, chart from xue.components.model_config_row import model_config_row # import sys @@ -1423,7 +1419,7 @@ async def root(x_api_key: str = Depends(get_api_key)): ), Div(id="sheet-container"), # sheet加载位置 id="main-content", - class_="ml-[240px] p-6 transition-[margin] duration-200 ease-in-out" + class_="ml-[200px] p-6 transition-[margin] duration-200 ease-in-out" ), class_="flex" ), @@ -1444,6 +1440,33 @@ async def toggle_sidebar(is_collapsed: bool = False): active_item="dashboard" ).render() +@app.get("/sidebar/update/{active_item}", response_class=HTMLResponse) +async def update_sidebar(active_item: str): + return sidebar.Sidebar( + "zap", + "uni-api", + sidebar_items, + is_collapsed=False, + active_item=active_item + ).render() + +@frontend_router.get("/dashboard", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def data_page(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = Div( + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # sheet加载位置 + id="main-content", + class_="ml-[200px] p-6 transition-[margin] duration-200 ease-in-out" + ).render() + + return result + @frontend_router.get("/data", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) async def data_page(x_api_key: str = Depends(get_api_key)): if not x_api_key: @@ -1500,22 +1523,30 @@ async def data_page(x_api_key: str = Depends(get_api_key)): "legend": True, "tooltip": True } + chart_config = { + "stacked": False, + "horizontal": False, + "colors": ["#2563eb", "#60a5fa"], + "grid": True, # 隐藏网格 + "legend": True, # 显示图例 + "tooltip": True # 启用工具提示 + } + print(chart_data) + print(series) - result = HTML( - Head(title="数据统计"), - Body( - Div( - Div( - "模型使用统计 (24小时)", - class_="text-2xl font-bold mb-4" - ), - Div( - chart.bar_chart("model-usage-chart", chart_data, "model", series, chart_config), - class_="h-[600px]" # 设置图表高度 - ), - class_="container mx-auto p-4" - ) - ) + result = Div( + Div( + "模型使用统计 (24小时)", + class_="text-2xl font-bold mb-4" + ), + Div( + chart.bar_chart("basic-chart", chart_data, "month", series, chart_config), + # chart.bar_chart("model-usage-chart", chart_data, "model", series, chart_config), + class_="mb-8" # 设置图表高度 + ), + id="main-content", + class_="container ml-[200px] mx-auto p-4" + # class_="container ml-[200px] mx-auto p-4" ).render() return result diff --git a/utils.py b/utils.py index 42dab548..072743a1 100644 --- a/utils.py +++ b/utils.py @@ -109,9 +109,18 @@ def update_config(config_data, use_config_url=False): for model in api_key.get('model'): if isinstance(model, dict): key, value = list(model.items())[0] - # provider_name = key.split("/")[0] - if "/" in key: - weights_dict.update({key: int(value)}) + provider_name = key.split("/")[0] + model_name = key.split("/")[1] + + for provider_item in config_data["providers"]: + if provider_item['provider'] != provider_name: + continue + model_dict = get_model_dict(provider_item) + if model_name in model_dict.keys(): + weights_dict.update({provider_name + "/" + model_dict[model_name]: int(value)}) + elif model_name == "*": + weights_dict.update({provider_name + "/" + model_dict[model_item]: int(value) for model_item in model_dict.keys()}) + models.append(key) if isinstance(model, str): models.append(model) From c5cf73f6204e217eb493f325d9d1eb49098eaf55 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 08:55:06 +0000 Subject: [PATCH 188/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.54?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f2e654c1..c97e08fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.53 +0.0.54 From fc4b8260705024a461e56cb3507b1ada6e1312f5 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 31 Oct 2024 17:36:09 +0800 Subject: [PATCH 189/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20channel=20cooldown.=20When=20an=20API=20channel=20?= =?UTF-8?q?response=20fails,=20the=20channel=20will=20automatically=20be?= =?UTF-8?q?=20excluded=20and=20cooled=20down=20for=20a=20period=20of=20tim?= =?UTF-8?q?e,=20during=20which=20no=20requests=20will=20be=20made=20to=20t?= =?UTF-8?q?hat=20channel.=20After=20the=20cooldown=20period=20ends,=20the?= =?UTF-8?q?=20model=20will=20automatically=20be=20restored=20until=20it=20?= =?UTF-8?q?fails=20again,=20triggering=20another=20cooldown.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 69 +++++++++++++++++++++++++++------------------------- README_CN.md | 3 +++ main.py | 57 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d0a549f4..88fac9d6 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ For personal use, one/new-api is too complex with many commercial features that 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. - Support automatic retry, when an API channel response fails, automatically retry the next API channel. +- Support channel cooling: When an API channel response fails, the channel will automatically be excluded and cooled for a period of time, and requests to the channel will be stopped. After the cooling period ends, the model will automatically be restored until it fails again, at which point it will be cooled again. +- Support fine-grained model timeout settings, allowing different timeout durations for each model. - Support fine-grained permission control. Support using wildcards to set specific models available for API key channels. - Support rate limiting, you can set the maximum number of requests per minute as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. - Supports multiple standard OpenAI format interfaces: `/v1/chat/completions`, `/v1/images/generations`, `/v1/audio/transcriptions`, `/v1/moderations`, `/v1/models`. @@ -59,36 +61,36 @@ Detailed advanced configuration of `api.yaml`: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name is fine, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # Optional, if the model is not configured, all available models will be automatically obtained through the /v1/models endpoint via base_url and api. + model: # Optional, if model is not configured, all available models will be automatically obtained through base_url and api via the /v1/models endpoint. - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, a simpler name can replace the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, a simpler name can replace the original complex name, optional - tools: true # Whether to support tools, such as code generation, document generation, etc., default is true, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional + tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini - base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini models, required + base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini model use, required api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the following line to use the original name. - - gemini-1.5-flash-exp-0827 # Adding this line allows both gemini-1.5-flash-exp-0827 and gemini-1.5-flash to be requested + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name + - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: A string usually consisting of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the service account's private key information. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, or can be found in the "IAM & Admin" section of the Google Cloud Console to view service account details. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key for Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, or you can view the service account details in the "IAM and Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -97,14 +99,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # Can include the provider's website, remarks, official documentation, optional + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional - provider: cloudflare api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required model: - - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes to avoid YAML syntax error, llama-3.1-8b is the renamed name, a simpler name can replace the original complex name, optional - - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes to avoid YAML syntax error + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a simple name to replace the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes, otherwise yaml syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -113,47 +115,48 @@ providers: - causallm-35b-beta2ep-q6k: causallm-35b - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # Force using a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional + engine: openrouter # Force the use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, users need an API key to use this service, required - model: # Models that can be used with this API Key, required. By default, channel-level round-robin load balancing is enabled, and each request is made in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request orders for each API key. + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service + model: # Models that can be used by this API Key, required. Default channel-level polling load balancing is enabled, and each request model is requested in sequence according to the model configuration. It is not related to the original channel order in providers. Therefore, you can set different request sequences for each API key. - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers - - gemini/* # Usable model name, can only use all models provided by the provider named gemini, where gemini is the provider name, * represents all models + - gemini/* # Usable model name, can only use all models provided by providers named gemini, where gemini is the provider name, * represents all models role: admin - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models named claude-3-5-sonnet from other providers cannot be used. This syntax will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - - # By adding angle brackets around the model name, it will not search for the claude-3-5-sonnet model under the channel named anthropic, but will treat the entire anthropic/claude-3-5-sonnet as the model name. This syntax can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. This syntax will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets on both sides of the model name, it will not search for the claude-3-5-sonnet model under the channel named anthropic, but will take the entire anthropic/claude-3-5-sonnet as the model name. This syntax can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moderation. preferences: - SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, fixed priority scheduling is used, always executing the channel of the first requested model. Enabled by default, the default value of SCHEDULING_ALGORITHM is fixed_priority. Optional values for SCHEDULING_ALGORITHM are: fixed_priority, round_robin, weighted_round_robin, lottery, random. - # When SCHEDULING_ALGORITHM is random, random round-robin load balancing is used, randomly requesting the channel of the requested model. - # When SCHEDULING_ALGORITHM is round_robin, round-robin load balancing is used, requesting the user's model channels in order. + SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the channel of the first model with a request. Default is enabled, SCHEDULING_ALGORITHM default value is fixed_priority. SCHEDULING_ALGORITHM optional values are: fixed_priority, round_robin, weighted_round_robin, lottery, random. + # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel of the model with a request. + # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the model used by the user in order. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true - RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default 60/min, optional - ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, messages will be moderated, and inappropriate messages will return an error. + RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, weights only support positive integers. - - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of request. - - gcp3/*: 2 # In this example, there are a total of 10 weights across all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp1/*: 5 # The number after the colon is the weight, weight only supports positive integers. + - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 10 requests will have 5 requests for the gcp1/* model, 2 requests for the gcp2/* model, and 3 requests for the gcp3/* model. preferences: - SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the above channels have weights, requests will be made in the weighted order. Using weighted round-robin load balancing, requests are made in the order of weight for the channel of the requested model. When SCHEDULING_ALGORITHM is lottery, lottery round-robin load balancing is used, randomly requesting the channel of the requested model according to weight. Channels without weights automatically fall back to round_robin round-robin load balancing. + SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the above channel has weights, it will request according to the weighted order. Use weighted polling load balancing, request the channel of the model with a request according to the weight order. When SCHEDULING_ALGORITHM is lottery, use lottery polling load balancing, request the channel of the model with a request according to the weight randomly. Channels without weights automatically fall back to round_robin polling load balancing. AUTO_RETRY: true preferences: # Global configuration model_timeout: # Model timeout, in seconds, default 100 seconds, optional gpt-4o: 10 # Model gpt-4o timeout is 10 seconds, gpt-4o is the model name, when requesting models like gpt-4o-2024-08-06, the timeout is also 10 seconds claude-3-5-sonnet: 10 # Model claude-3-5-sonnet timeout is 10 seconds, when requesting models like claude-3-5-sonnet-20240620, the timeout is also 10 seconds - default: 10 # If the model does not have a timeout set, the default timeout of 10 seconds is used, when requesting models not in model_timeout, the default timeout is 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, which is 100 seconds - o1-mini: 30 # Model o1-mini timeout is 30 seconds, when requesting models with names starting with o1-mini, the timeout is 30 seconds - o1-preview: 100 # Model o1-preview timeout is 100 seconds, when requesting models with names starting with o1-preview, the timeout is 100 seconds + default: 10 # Model does not have a timeout set, use the default timeout of 10 seconds, when requesting a model not in model_timeout, the default timeout is 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, the default timeout is 100 seconds + o1-mini: 30 # Model o1-mini timeout is 30 seconds, when requesting models starting with o1-mini, the timeout is 30 seconds + o1-preview: 100 # Model o1-preview timeout is 100 seconds, when requesting models starting with o1-preview, the timeout is 100 seconds + cooldown_period: 300 # Channel cooldown time, in seconds, default 300 seconds, optional. When a model request fails, the channel will be automatically excluded and cooled down for a period of time, and will not request the channel again. After the cooldown time ends, the model will be automatically restored until the request fails again, and it will be cooled down again. When cooldown_period is set to 0, the cooling mechanism is not enabled. ``` Mount the configuration file and start the uni-api docker container: diff --git a/README_CN.md b/README_CN.md index 58a65d4b..9ed05dca 100644 --- a/README_CN.md +++ b/README_CN.md @@ -28,6 +28,8 @@ 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。默认不开启，需要配置 `SCHEDULING_ALGORITHM` 为 `round_robin`。 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 - 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 +- 支持渠道冷却，当一个 API 渠道响应失败时，会自动将该渠道排除冷却一段时间，不再请求该渠道，冷却时间结束后，会自动将该模型恢复，直到再次请求失败，会重新冷却。 +- 支持细粒度的模型超时时间设置，可以为每个模型设置不同的超时时间。 - 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 - 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 - 支持多个标准 OpenAI 格式的接口：`/v1/chat/completions`，`/v1/images/generations`，`/v1/audio/transcriptions`，`/v1/moderations`，`/v1/models`。 @@ -154,6 +156,7 @@ preferences: # 全局配置 default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用 环境变量 TIMEOUT 设置的默认超时时间，默认超时时间是 100 秒 o1-mini: 30 # 模型 o1-mini 的超时时间为 30 秒，当请求名字是 o1-mini 开头的模型时，超时时间是 30 秒 o1-preview: 100 # 模型 o1-preview 的超时时间为 100 秒，当请求名字是 o1-preview 开头的模型时，超时时间是 100 秒 + cooldown_period: 300 # 渠道冷却时间，单位为秒，默认 300 秒，选填。当模型请求失败时，会自动将该渠道排除冷却一段时间，不再请求该渠道，冷却时间结束后，会自动将该模型恢复，直到再次请求失败，会重新冷却。当 cooldown_period 设置为 0 时，不启用冷却机制。 ``` 挂载配置文件并启动 uni-api docker 容器： diff --git a/main.py b/main.py index bfda9afa..51069ed7 100644 --- a/main.py +++ b/main.py @@ -159,6 +159,39 @@ async def parse_request_body(request: Request): return None return None +class ChannelManager: + def __init__(self, cooldown_period: int = 300): # 默认冷却时间5分钟 + self._excluded_channels: Dict[str, datetime] = {} + self._lock = asyncio.Lock() + self.cooldown_period = cooldown_period + + async def exclude_channel(self, channel_id: str): + """将渠道添加到排除列表""" + async with self._lock: + self._excluded_channels[channel_id] = datetime.now() + + async def is_channel_excluded(self, channel_id: str) -> bool: + """检查渠道是否被排除""" + async with self._lock: + if channel_id not in self._excluded_channels: + return False + + excluded_time = self._excluded_channels[channel_id] + if datetime.now() - excluded_time > timedelta(seconds=self.cooldown_period): + # 已超过冷却时间，移除限制 + del self._excluded_channels[channel_id] + return False + return True + + async def get_available_providers(self, providers: list) -> list: + """过滤出可用的providers""" + available_providers = [] + for provider in providers: + channel_id = f"{provider['provider']}" + if not await self.is_channel_excluded(channel_id): + available_providers.append(provider) + return available_providers + from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession from sqlalchemy.orm import declarative_base, sessionmaker from sqlalchemy import Column, Integer, String, Float, DateTime, select, Boolean, Text @@ -541,7 +574,7 @@ async def close(self): @app.middleware("http") async def ensure_config(request: Request, call_next): - if not hasattr(app.state, 'config'): + if app and not hasattr(app.state, 'config'): logger.warning("Config not found, attempting to reload") app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) @@ -580,6 +613,14 @@ async def ensure_config(request: Request, call_next): print("app.state.timeouts", app.state.timeouts) + if app and not hasattr(app.state, "channel_manager"): + if app.state.config and 'preferences' in app.state.config: + COOLDOWN_PERIOD = app.state.config['preferences'].get('cooldown_period', 300) + else: + COOLDOWN_PERIOD = 300 + + app.state.channel_manager = ChannelManager(cooldown_period=COOLDOWN_PERIOD) + return await call_next(request) # 在 process_request 函数中更新成功和失败计数 @@ -841,11 +882,18 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques request_model = request.model matching_providers = get_matching_providers(request_model, config, api_index) - num_matching_providers = len(matching_providers) if not matching_providers: raise HTTPException(status_code=404, detail="No matching model found") + if app.state.channel_manager.cooldown_period > 0: + matching_providers = await app.state.channel_manager.get_available_providers(matching_providers) + if not matching_providers: + raise HTTPException(status_code=503, detail="No available providers at the moment") + + num_matching_providers = len(matching_providers) + + # 检查是否启用轮询 scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") if scheduling_algorithm == "random": @@ -936,7 +984,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques status_code = 500 # Internal Server Error error_message = str(e) or f"Unknown error: {e.__class__.__name__}" - logger.error(f"Error {status_code} with provider {provider['provider']}: {error_message}") + channel_id = f"{provider['provider']}" + if app.state.channel_manager.cooldown_period > 0: + await app.state.channel_manager.exclude_channel(channel_id) + logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") if is_debug: import traceback traceback.print_exc() From 5ab6b69b7fcdac050aa9cd106176a2e4073112ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 09:36:30 +0000 Subject: [PATCH 190/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.55?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c97e08fd..8e0e3bc2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.54 +0.0.55 From bdb98ffc796fb12225609f26d983cdf8af6591ca Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 31 Oct 2024 19:20:53 +0800 Subject: [PATCH 191/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fixed=20the=20bug?= =?UTF-8?q?=20where=20the=20cooling=20model=20could=20not=20take=20effect?= =?UTF-8?q?=20in=20this=20request.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 90 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/main.py b/main.py index 51069ed7..79d611b4 100644 --- a/main.py +++ b/main.py @@ -866,48 +866,28 @@ def get_matching_providers(request_model, config, api_index): # print("provider_list", provider_list) return provider_list -import asyncio -class ModelRequestHandler: - def __init__(self): - self.last_provider_indices = defaultdict(lambda: -1) - self.locks = defaultdict(asyncio.Lock) - - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): - config = app.state.config - api_list = app.state.api_list - api_index = api_list.index(token) +async def get_right_order_providers(request_model, config, api_index, scheduling_algorithm): + matching_providers = get_matching_providers(request_model, config, api_index) - if not safe_get(config, 'api_keys', api_index, 'model'): - raise HTTPException(status_code=404, detail="No matching model found") - - request_model = request.model - matching_providers = get_matching_providers(request_model, config, api_index) + if not matching_providers: + raise HTTPException(status_code=404, detail="No matching model found") + if app.state.channel_manager.cooldown_period > 0: + matching_providers = await app.state.channel_manager.get_available_providers(matching_providers) if not matching_providers: - raise HTTPException(status_code=404, detail="No matching model found") - - if app.state.channel_manager.cooldown_period > 0: - matching_providers = await app.state.channel_manager.get_available_providers(matching_providers) - if not matching_providers: - raise HTTPException(status_code=503, detail="No available providers at the moment") + raise HTTPException(status_code=503, detail="No available providers at the moment") + # 检查是否启用轮询 + if scheduling_algorithm == "random": num_matching_providers = len(matching_providers) + matching_providers = random.sample(matching_providers, num_matching_providers) + weights = safe_get(config, 'api_keys', api_index, "weights") - # 检查是否启用轮询 - scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") - if scheduling_algorithm == "random": - matching_providers = random.sample(matching_providers, num_matching_providers) - - weights = safe_get(config, 'api_keys', api_index, "weights") - - # 步骤 1: 提取 matching_providers 中的所有 provider 值 - # print("matching_providers", matching_providers) - # print(type(matching_providers[0]['model'][0].keys()), list(matching_providers[0]['model'][0].keys())[0], matching_providers[0]['model'][0].keys()) - all_providers = set(provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in matching_providers) - + if weights: intersection = None - if weights and all_providers: + all_providers = set(provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in matching_providers) + if all_providers: weight_keys = set(weights.keys()) provider_rules = [] for model_rule in weight_keys: @@ -922,7 +902,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques intersection = all_providers.intersection(weight_keys) # print("intersection", intersection) - if weights and intersection: + if intersection: filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k in intersection} # print("filtered_weights", filtered_weights) @@ -941,9 +921,31 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques new_matching_providers.append(provider) matching_providers = new_matching_providers - if is_debug: - for provider in matching_providers: - logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) + if is_debug: + for provider in matching_providers: + logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) + + return matching_providers + +import asyncio +class ModelRequestHandler: + def __init__(self): + self.last_provider_indices = defaultdict(lambda: -1) + self.locks = defaultdict(asyncio.Lock) + + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): + config = app.state.config + api_list = app.state.api_list + api_index = api_list.index(token) + + if not safe_get(config, 'api_keys', api_index, 'model'): + raise HTTPException(status_code=404, detail="No matching model found") + + request_model = request.model + scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") + + matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) + num_matching_providers = len(matching_providers) status_code = 500 error_message = None @@ -956,8 +958,12 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) - for i in range(num_matching_providers + 1): - current_index = (start_index + i) % num_matching_providers + index = 0 + while True: + if index >= num_matching_providers: + break + current_index = (start_index + index) % num_matching_providers + index += 1 provider = matching_providers[current_index] try: response = await process_request(request, provider, endpoint, token) @@ -987,6 +993,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques channel_id = f"{provider['provider']}" if app.state.channel_manager.cooldown_period > 0: await app.state.channel_manager.exclude_channel(channel_id) + matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) + num_matching_providers = len(matching_providers) + index = 0 + logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") if is_debug: import traceback From 69f5c7c6dfc9b0a27dfebae9d0f7b52d63c77a0e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 11:21:19 +0000 Subject: [PATCH 192/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.56?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8e0e3bc2..ea7f0308 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.55 +0.0.56 From ec63db6d7c2b8e94d881e3246619466aaa24e808 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 03:29:02 +0800 Subject: [PATCH 193/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20chart=20display.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💰 Sponsors: Thanks to @PowerHunter for the ¥600 sponsorship, sponsorship information has been added to the README. --- README.md | 4 +-- README_CN.md | 4 +-- main.py | 99 +++++++++++++++++++++++++++------------------------- 3 files changed, 56 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 88fac9d6..39ee9f7a 100644 --- a/README.md +++ b/README.md @@ -315,8 +315,8 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ ## Sponsors We thank the following sponsors for their support: - -- @PowerHunter: ¥200 + +- @PowerHunter: ¥600 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index 9ed05dca..da7c6cb2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -315,8 +315,8 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ ## 赞助商 我们感谢以下赞助商的支持： - -- @PowerHunter：¥200 + +- @PowerHunter：¥600 ## 如何赞助我们 diff --git a/main.py b/main.py index 79d611b4..a5be183b 100644 --- a/main.py +++ b/main.py @@ -1540,75 +1540,80 @@ async def data_page(x_api_key: str = Depends(get_api_key)): # 计算过去24小时的开始时间 start_time = datetime.now(timezone.utc) - timedelta(hours=24) - # 获取每个模型的请求数据 + # 按小时统计每个模型的请求数据 model_stats = await session.execute( select( + func.strftime('%H', RequestStat.timestamp).label('hour'), RequestStat.model, - RequestStat.provider, func.count().label('count') ) .where(RequestStat.timestamp >= start_time) - .group_by(RequestStat.model, RequestStat.provider) - .order_by(desc('count')) + .group_by('hour', RequestStat.model) + .order_by('hour') ) model_stats = model_stats.fetchall() - # 处理数据以适配图表格式 - chart_data = [] - providers = list(set(stat.provider for stat in model_stats)) + # 获取所有唯一的模型名称 models = list(set(stat.model for stat in model_stats)) - for model in models: - data_point = {"model": model} - for provider in providers: + # 生成24小时的数据点 + chart_data = [] + current_hour = datetime.now().hour + + for i in range(24): + # 计算小时标签(从当前小时往前推24小时) + hour = (current_hour - i) % 24 + hour_str = f"{hour:02d}" + + # 创建该小时的数据点 + data_point = {"label": hour_str} + + # 添加每个模型在该小时的请求数 + for model in models: count = next( (stat.count for stat in model_stats - if stat.model == model and stat.provider == provider), + if stat.hour == f"{hour:02d}" and stat.model == model), 0 ) - data_point[provider] = count + data_point[model] = count + chart_data.append(data_point) - # 定义图表系列 - series = [ - {"name": provider, "data_key": provider} - for provider in providers - ] + # 反转数据点顺序使其按时间正序显示 + chart_data.reverse() - # 图表配置 - chart_config = { - "stacked": True, # 堆叠柱状图 - "horizontal": False, - "colors": [f"hsl({i * 360 / len(providers)}, 70%, 50%)" for i in range(len(providers))], # 生成不同的颜色 - "grid": True, - "legend": True, - "tooltip": True - } + # 为每个模型配置显示属性 chart_config = { - "stacked": False, - "horizontal": False, - "colors": ["#2563eb", "#60a5fa"], - "grid": True, # 隐藏网格 - "legend": True, # 显示图例 - "tooltip": True # 启用工具提示 + model: { + "label": model, + "color": f"hsl({i * 360 / len(models)}, 70%, 50%)" # 为每个模型生成不同的颜色 + } + for i, model in enumerate(models) } - print(chart_data) - print(series) - result = Div( - Div( - "模型使用统计 (24小时)", - class_="text-2xl font-bold mb-4" - ), - Div( - chart.bar_chart("basic-chart", chart_data, "month", series, chart_config), - # chart.bar_chart("model-usage-chart", chart_data, "model", series, chart_config), - class_="mb-8" # 设置图表高度 - ), - id="main-content", - class_="container ml-[200px] mx-auto p-4" - # class_="container ml-[200px] mx-auto p-4" + result = HTML( + Head(title="数据统计"), + Body( + Div( + # 堆叠柱状图 + Div( + "模型使用统计 (24小时) - 按小时统计", + class_="text-2xl font-bold mb-4" + ), + Div( + chart.chart( + chart_data, + chart_config, + stacked=True, + ), + class_="mb-8 h-[400px]" # 添加固定高度 + ), + id="main-content", + class_="container ml-[200px] mx-auto p-4" + ) + ) ).render() + print(result) return result From fd03c8172d296090eaf315a452e40844320430b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 19:29:29 +0000 Subject: [PATCH 194/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.57?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ea7f0308..a758e3a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.56 +0.0.57 From ab6e0580dcb2425de1605ad61c7cd1e04305d3de Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 04:16:38 +0800 Subject: [PATCH 195/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20GitHub=20Action?= =?UTF-8?q?=20Docker=20image=20build=20adds=20support=20for=20manual=20tri?= =?UTF-8?q?gger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7fa3c869..b2df4b11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,7 @@ on: - requirements.txt - docker-compose.yml - .github/workflows/main.yml + workflow_dispatch: jobs: build-and-push: From 3165e149d212d38624c07e68646b7c1679ff66db Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 20:17:12 +0000 Subject: [PATCH 196/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.58?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a758e3a8..9ebecc78 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.57 +0.0.58 From 656d0b550a417d4246896f40736d9be53fbebdf8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Oct 2024 20:20:30 +0000 Subject: [PATCH 197/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.59?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9ebecc78..1e7c7744 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.58 +0.0.59 From f2794a657ec483f2e463dab72a6db5ea682c6af4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 17:31:44 +0800 Subject: [PATCH 198/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20model-level=20cooling.=20When=20a=20model=20under?= =?UTF-8?q?=20a=20channel=20reports=20an=20error,=20it=20does=20not=20affe?= =?UTF-8?q?ct=20other=20models=20under=20the=20same=20channel,=20only=20th?= =?UTF-8?q?e=20model=20that=20reported=20the=20error=20is=20cooled.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index a5be183b..1038cc60 100644 --- a/main.py +++ b/main.py @@ -161,35 +161,43 @@ async def parse_request_body(request: Request): class ChannelManager: def __init__(self, cooldown_period: int = 300): # 默认冷却时间5分钟 - self._excluded_channels: Dict[str, datetime] = {} + self._excluded_models: Dict[str, datetime] = {} self._lock = asyncio.Lock() self.cooldown_period = cooldown_period - async def exclude_channel(self, channel_id: str): - """将渠道添加到排除列表""" + async def exclude_model(self, provider: str, model: str): + """将特定渠道下的特定模型添加到排除列表""" async with self._lock: - self._excluded_channels[channel_id] = datetime.now() + model_key = f"{provider}/{model}" + self._excluded_models[model_key] = datetime.now() - async def is_channel_excluded(self, channel_id: str) -> bool: - """检查渠道是否被排除""" + async def is_model_excluded(self, provider: str, model: str) -> bool: + """检查特定渠道下的特定模型是否被排除""" async with self._lock: - if channel_id not in self._excluded_channels: + model_key = f"{provider}/{model}" + if model_key not in self._excluded_models: return False - excluded_time = self._excluded_channels[channel_id] + excluded_time = self._excluded_models[model_key] if datetime.now() - excluded_time > timedelta(seconds=self.cooldown_period): # 已超过冷却时间，移除限制 - del self._excluded_channels[channel_id] + del self._excluded_models[model_key] return False return True async def get_available_providers(self, providers: list) -> list: - """过滤出可用的providers""" + """过滤出可用的providers，仅排除不可用的模型""" available_providers = [] for provider in providers: - channel_id = f"{provider['provider']}" - if not await self.is_channel_excluded(channel_id): + provider_name = provider['provider'] + model_dict = provider['model'][0] # 获取唯一的模型字典 + source_model = list(model_dict.keys())[0] # 源模型名称 + # target_model = list(model_dict.values())[0] # 目标模型名称 + + # 检查该模型是否被排除 + if not await self.is_model_excluded(provider_name, source_model): available_providers.append(provider) + return available_providers from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession @@ -992,7 +1000,9 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques channel_id = f"{provider['provider']}" if app.state.channel_manager.cooldown_period > 0: - await app.state.channel_manager.exclude_channel(channel_id) + # 获取源模型名称（实际配置的模型名） + source_model = list(provider['model'][0].keys())[0] + await app.state.channel_manager.exclude_model(channel_id, source_model) matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) num_matching_providers = len(matching_providers) index = 0 From f01693fe94acd978cbd6c768d69d74fcd79ff73b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 09:32:05 +0000 Subject: [PATCH 199/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.60?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1e7c7744..b4ae2bd0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.59 +0.0.60 From f2a60ffc1bcaa4f2c1f2c8c38733ab16f86735d7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 20:28:55 +0800 Subject: [PATCH 200/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20incorrect=20weight=20allocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 14 +++++++------- utils.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 1038cc60..ea26c991 100644 --- a/main.py +++ b/main.py @@ -191,11 +191,11 @@ async def get_available_providers(self, providers: list) -> list: for provider in providers: provider_name = provider['provider'] model_dict = provider['model'][0] # 获取唯一的模型字典 - source_model = list(model_dict.keys())[0] # 源模型名称 - # target_model = list(model_dict.values())[0] # 目标模型名称 + # source_model = list(model_dict.keys())[0] # 源模型名称 + target_model = list(model_dict.values())[0] # 目标模型名称 # 检查该模型是否被排除 - if not await self.is_model_excluded(provider_name, source_model): + if not await self.is_model_excluded(provider_name, target_model): available_providers.append(provider) return available_providers @@ -894,14 +894,14 @@ async def get_right_order_providers(request_model, config, api_index, scheduling if weights: intersection = None - all_providers = set(provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in matching_providers) + all_providers = set(provider['provider'] + "/" + request_model for provider in matching_providers) if all_providers: weight_keys = set(weights.keys()) provider_rules = [] for model_rule in weight_keys: provider_rules.extend(get_provider_rules(model_rule, config, request_model)) provider_list = get_provider_list(provider_rules, config, request_model) - weight_keys = set([provider['provider'] + "/" + list(provider['model'][0].keys())[0] for provider in provider_list]) + weight_keys = set([provider['provider'] + "/" + request_model for provider in provider_list]) # print("all_providers", all_providers) # print("weights", weights) # print("weight_keys", weight_keys) @@ -1001,8 +1001,8 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques channel_id = f"{provider['provider']}" if app.state.channel_manager.cooldown_period > 0: # 获取源模型名称（实际配置的模型名） - source_model = list(provider['model'][0].keys())[0] - await app.state.channel_manager.exclude_model(channel_id, source_model) + # source_model = list(provider['model'][0].keys())[0] + await app.state.channel_manager.exclude_model(channel_id, request_model) matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) num_matching_providers = len(matching_providers) index = 0 diff --git a/utils.py b/utils.py index 072743a1..4db8a7fd 100644 --- a/utils.py +++ b/utils.py @@ -117,9 +117,9 @@ def update_config(config_data, use_config_url=False): continue model_dict = get_model_dict(provider_item) if model_name in model_dict.keys(): - weights_dict.update({provider_name + "/" + model_dict[model_name]: int(value)}) + weights_dict.update({provider_name + "/" + model_name: int(value)}) elif model_name == "*": - weights_dict.update({provider_name + "/" + model_dict[model_item]: int(value) for model_item in model_dict.keys()}) + weights_dict.update({provider_name + "/" + model_name: int(value) for model_item in model_dict.keys()}) models.append(key) if isinstance(model, str): From 55564a6eb12e67734c276d272b86842d739da1f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 12:29:25 +0000 Subject: [PATCH 201/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.61?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b4ae2bd0..72189def 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.60 +0.0.61 From 52d3f47a574386e2c36b241bc7c1faebb7971342 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 20:53:13 +0800 Subject: [PATCH 202/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20Vercel=20fails=20to=20start=20successfully.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.py b/main.py index ea26c991..19670cc2 100644 --- a/main.py +++ b/main.py @@ -1348,6 +1348,9 @@ async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_k # print(f"Header x_api_key: {x_api_key}") # 添加此行 # logger.info(f"x_api_key: {x_api_key} {x_api_key == 'your_admin_api_key'}") + if not hasattr(app.state, 'config'): + await ensure_config(request, lambda: None) + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 return x_api_key else: From aa9bf5723e60394eab759b2c109f483d75e62180 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 12:53:48 +0000 Subject: [PATCH 203/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.62?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 72189def..7eb36658 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.61 +0.0.62 From db534e04cb2231f75316f81eb757786c582143c6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 21:01:12 +0800 Subject: [PATCH 204/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20Vercel=20fails=20to=20start=20successfully.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 4db8a7fd..59774aff 100644 --- a/utils.py +++ b/utils.py @@ -166,7 +166,8 @@ async def load_config(app=None): config_url = os.environ.get('CONFIG_URL') if config_url: try: - response = await app.state.client.get(config_url) + client = app.state.client_manager.get_client(100) + response = await client.get(config_url) # logger.info(f"Fetching config from {response.text}") response.raise_for_status() config_data = yaml.load(response.text) From 8e8b08a1d91ee2f78eee34a6379b59741ddc8efd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 13:01:39 +0000 Subject: [PATCH 205/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.63?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7eb36658..8d056f1a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.62 +0.0.63 From 467f2562efbd0b8e4dfb9986ae4f3af1a4547467 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 1 Nov 2024 21:10:32 +0800 Subject: [PATCH 206/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20Vercel=20fails=20to=20start=20successfully.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 ++- utils.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 19670cc2..e96fbe94 100644 --- a/main.py +++ b/main.py @@ -582,8 +582,9 @@ async def close(self): @app.middleware("http") async def ensure_config(request: Request, call_next): + if app and not hasattr(app.state, 'config'): - logger.warning("Config not found, attempting to reload") + # logger.warning("Config not found, attempting to reload") app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) for item in app.state.api_keys_db: diff --git a/utils.py b/utils.py index 59774aff..37f3bc26 100644 --- a/utils.py +++ b/utils.py @@ -166,7 +166,26 @@ async def load_config(app=None): config_url = os.environ.get('CONFIG_URL') if config_url: try: - client = app.state.client_manager.get_client(100) + default_config = { + "headers": { + "User-Agent": "curl/7.68.0", + "Accept": "*/*", + }, + "http2": True, + "verify": True, + "follow_redirects": True + } + # 初始化客户端管理器 + timeout = httpx.Timeout( + connect=15.0, + read=100, + write=30.0, + pool=200 + ) + client = httpx.AsyncClient( + timeout=timeout, + **default_config + ) response = await client.get(config_url) # logger.info(f"Fetching config from {response.text}") response.raise_for_status() From bc315915f93eba81a019e9fd3c173e31ed02aec8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 13:10:56 +0000 Subject: [PATCH 207/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8d056f1a..60483109 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.63 +0.0.64 From eefdfa1fd23e4270d07ad10a837dc03b8524ffa3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 2 Nov 2024 20:48:18 +0800 Subject: [PATCH 208/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fixed=20the=20bug?= =?UTF-8?q?=20with=20limited=20concurrency=20and=20removed=20unnecessary?= =?UTF-8?q?=20asynchronous=20mutex=20locks.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💰 Sponsors: Thanks to @PowerHunter for the ¥1000 sponsorship, sponsorship information has been added to the README. --- README.md | 4 +-- README_CN.md | 4 +-- main.py | 79 ++++++++++++++++++++++------------------------------ 3 files changed, 37 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 39ee9f7a..35e0b62d 100644 --- a/README.md +++ b/README.md @@ -315,8 +315,8 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ ## Sponsors We thank the following sponsors for their support: - -- @PowerHunter: ¥600 + +- @PowerHunter: ¥1000 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index da7c6cb2..2776c62a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -315,8 +315,8 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ ## 赞助商 我们感谢以下赞助商的支持： - -- @PowerHunter：¥600 + +- @PowerHunter：¥1000 ## 如何赞助我们 diff --git a/main.py b/main.py index e96fbe94..811f3445 100644 --- a/main.py +++ b/main.py @@ -160,30 +160,24 @@ async def parse_request_body(request: Request): return None class ChannelManager: - def __init__(self, cooldown_period: int = 300): # 默认冷却时间5分钟 - self._excluded_models: Dict[str, datetime] = {} - self._lock = asyncio.Lock() + def __init__(self, cooldown_period=300): + self._excluded_models = defaultdict(lambda: None) self.cooldown_period = cooldown_period async def exclude_model(self, provider: str, model: str): - """将特定渠道下的特定模型添加到排除列表""" - async with self._lock: - model_key = f"{provider}/{model}" - self._excluded_models[model_key] = datetime.now() + model_key = f"{provider}/{model}" + self._excluded_models[model_key] = datetime.now() async def is_model_excluded(self, provider: str, model: str) -> bool: - """检查特定渠道下的特定模型是否被排除""" - async with self._lock: - model_key = f"{provider}/{model}" - if model_key not in self._excluded_models: - return False - - excluded_time = self._excluded_models[model_key] - if datetime.now() - excluded_time > timedelta(seconds=self.cooldown_period): - # 已超过冷却时间，移除限制 - del self._excluded_models[model_key] - return False - return True + model_key = f"{provider}/{model}" + excluded_time = self._excluded_models[model_key] + if not excluded_time: + return False + + if datetime.now() - excluded_time > timedelta(seconds=self.cooldown_period): + del self._excluded_models[model_key] + return False + return True async def get_available_providers(self, providers: list) -> list: """过滤出可用的providers，仅排除不可用的模型""" @@ -541,39 +535,32 @@ class ClientManager: def __init__(self, pool_size=100): self.pool_size = pool_size self.clients = {} # {timeout_value: AsyncClient} - self.locks = {} # {timeout_value: Lock} async def init(self, default_config): self.default_config = default_config @asynccontextmanager async def get_client(self, timeout_value): - # 对同一超时值的客户端加锁 - if timeout_value not in self.locks: - self.locks[timeout_value] = asyncio.Lock() - - async with self.locks[timeout_value]: - # 获取或创建指定超时值的客户端 - if timeout_value not in self.clients: - timeout = httpx.Timeout( - connect=15.0, - read=timeout_value, - write=30.0, - pool=self.pool_size - ) - self.clients[timeout_value] = httpx.AsyncClient( - timeout=timeout, - limits=httpx.Limits(max_connections=self.pool_size), - **self.default_config - ) + # 直接获取或创建客户端,不使用锁 + if timeout_value not in self.clients: + timeout = httpx.Timeout( + connect=15.0, + read=timeout_value, + write=30.0, + pool=self.pool_size + ) + self.clients[timeout_value] = httpx.AsyncClient( + timeout=timeout, + limits=httpx.Limits(max_connections=self.pool_size), + **self.default_config + ) - try: - yield self.clients[timeout_value] - except Exception as e: - # 如果客户端出现问题，关闭并重新创建 - await self.clients[timeout_value].aclose() - del self.clients[timeout_value] - raise e + try: + yield self.clients[timeout_value] + except Exception as e: + await self.clients[timeout_value].aclose() + del self.clients[timeout_value] + raise e async def close(self): for client in self.clients.values(): @@ -791,7 +778,7 @@ def lottery_scheduling(weights): def get_provider_rules(model_rule, config, request_model): provider_rules = [] if model_rule == "all": - # 如果模型名为 all，则返回所有模型 + # 如���模型名为 all，则返回所有模型 for provider in config["providers"]: model_dict = get_model_dict(provider) for model in model_dict.keys(): From ba79e57172e595b65ae0d94ce6bb805e6e594028 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 2 Nov 2024 12:48:59 +0000 Subject: [PATCH 209/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 60483109..3df20f59 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.64 +0.0.65 From 44f121d58ad19909655f18bcccf402c7658c2bbb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 3 Nov 2024 01:38:38 +0800 Subject: [PATCH 210/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20When=20there=20i?= =?UTF-8?q?s=20only=20one=20optional=20provider,=20cooling=20will=20not=20?= =?UTF-8?q?take=20effect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 811f3445..fdb54165 100644 --- a/main.py +++ b/main.py @@ -868,14 +868,14 @@ async def get_right_order_providers(request_model, config, api_index, scheduling if not matching_providers: raise HTTPException(status_code=404, detail="No matching model found") - if app.state.channel_manager.cooldown_period > 0: + num_matching_providers = len(matching_providers) + if app.state.channel_manager.cooldown_period > 0 and num_matching_providers > 1: matching_providers = await app.state.channel_manager.get_available_providers(matching_providers) if not matching_providers: raise HTTPException(status_code=503, detail="No available providers at the moment") # 检查是否启用轮询 if scheduling_algorithm == "random": - num_matching_providers = len(matching_providers) matching_providers = random.sample(matching_providers, num_matching_providers) weights = safe_get(config, 'api_keys', api_index, "weights") @@ -987,7 +987,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques error_message = str(e) or f"Unknown error: {e.__class__.__name__}" channel_id = f"{provider['provider']}" - if app.state.channel_manager.cooldown_period > 0: + if app.state.channel_manager.cooldown_period > 0 and num_matching_providers > 1: # 获取源模型名称（实际配置的模型名） # source_model = list(provider['model'][0].keys())[0] await app.state.channel_manager.exclude_model(channel_id, request_model) From fc338674ed2dda19e4c4080ab569c76fa684f329 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 2 Nov 2024 17:39:18 +0000 Subject: [PATCH 211/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.66?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3df20f59..ff8026fa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.65 +0.0.66 From ec66323c0bcdf6667a22a0d828439d2d142b2970 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 4 Nov 2024 22:11:13 +0800 Subject: [PATCH 212/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++++++ README_CN.md | 6 ++++++ main.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 35e0b62d..2bbf1189 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,12 @@ If you would like to support our project, you can sponsor us in the following wa Thank you for your support! +## FAQ + +- Why does the error `Error processing request or performing moral check: 404: No matching model found` always appear? + +Setting ENABLE_MODERATION to false will fix this issue. When ENABLE_MODERATION is true, the API must be able to use the text-moderation-latest model, and if you have not provided text-moderation-latest in the provider model settings, an error will occur indicating that the model cannot be found. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index 2776c62a..fd8ebc9e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -332,6 +332,12 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ 感谢您的支持！ +## 常见问题 + +- 为什么总是出现 `Error processing request or performing moral check: 404: No matching model found` 错误？ + +将 ENABLE_MODERATION 设置为 false 将修复这个问题。当 ENABLE_MODERATION 为 true 时，API 必须能够使用 text-moderation-latest 模型，如果你没有在提供商模型设置里面提供 text-moderation-latest，将会报错找不到模型。 + ## ⭐ Star 历史 diff --git a/main.py b/main.py index fdb54165..3d4ce56a 100644 --- a/main.py +++ b/main.py @@ -477,7 +477,7 @@ async def dispatch(self, request: Request, call_next): import traceback traceback.print_exc() - logger.error(f"处理请求或进行道德检查时出错: {str(e)}") + logger.error(f"Error processing request or performing moral check: {str(e)}") try: response = await call_next(request) From 5f2aeb026157bf67a3cc3fcde92b171fe47355d8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 Nov 2024 14:11:46 +0000 Subject: [PATCH 213/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.67?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ff8026fa..9c3f756d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.66 +0.0.67 From 76819d61dce53a023f547e98eb88180b9c5881f6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 04:35:35 +0800 Subject: [PATCH 214/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20the=20closed=20client=20cannot=20be=20found=20whe?= =?UTF-8?q?n=20closing=20the=20request=20client.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where, when there is only one provider but multiple API keys, an error prevents switching to the next API key. ✨ Feature: Add feature: Add support for API key request rate limiting, add support for automatically cooling down API key upon receiving a 429 status code. 📖 Docs: Update documentation --- .github/workflows/main.yml | 1 + README.md | 43 ++++++++++++- README_CN.md | 43 ++++++++++++- main.py | 87 ++++++++----------------- utils.py | 129 +++++++++++++++++++++++++++++++++++-- 5 files changed, 237 insertions(+), 66 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2df4b11..e6dcef5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,7 @@ on: jobs: build-and-push: runs-on: ubuntu-latest + if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} steps: - name: Checkout repository diff --git a/README.md b/README.md index 2bbf1189..3e5da90a 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,18 @@ providers: - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini model use, required - api: AIzaSyAN2k6IRdgw + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + - AIzaSyAN2k6IRdgw123 + - AIzaSyAN2k6IRdgw456 + - AIzaSyAN2k6IRdgw789 model: - gemini-1.5-pro - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true + preferences: + API_KEY_RATE_LIMIT: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. + API_KEY_COOLDOWN_PERIOD: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 60 seconds. - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. @@ -338,6 +344,41 @@ Thank you for your support! Setting ENABLE_MODERATION to false will fix this issue. When ENABLE_MODERATION is true, the API must be able to use the text-moderation-latest model, and if you have not provided text-moderation-latest in the provider model settings, an error will occur indicating that the model cannot be found. +- How to prioritize requests for a specific channel, how to set the priority of a channel? + +Directly set the channel order in the api_keys. No other settings are required. Sample configuration file: + +```yaml +providers: + - provider: ai1 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + + - provider: ai2 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + +api_keys: + - api: sk-1234 + model: + - ai2/* + - ai1/* +``` + +In this way, request ai2 first, and if it fails, request ai1. + +- What is the behavior behind various scheduling algorithms? For example, fixed_priority, weighted_round_robin, lottery, random, round_robin? + +All scheduling algorithms need to be enabled by setting api_keys.(api).preferences.SCHEDULING_ALGORITHM in the configuration file to any of the values: fixed_priority, weighted_round_robin, lottery, random, round_robin. + +1. fixed_priority: Fixed priority scheduling. All requests are always executed by the channel of the model that first has a user request. In case of an error, it will switch to the next channel. This is the default scheduling algorithm. + +2. weighted_round_robin: Weighted round-robin load balancing, requests channels with the user's requested model according to the weight order set in the configuration file api_keys.(api).model. + +3. lottery: Draw round-robin load balancing, randomly request the channel of the model with user requests according to the weight set in the configuration file api_keys.(api).model. + +4. round_robin: Round-robin load balancing, requests the channel that owns the model requested by the user according to the configuration order in the configuration file api_keys.(api).model. You can check the previous question on how to set the priority of channels. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index fd8ebc9e..3b7200e0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -80,12 +80,18 @@ providers: - provider: gemini base_url: https://generativelanguage.googleapis.com/v1beta # base_url 支持 v1beta/v1, 仅供 Gemini 模型使用，必填 - api: AIzaSyAN2k6IRdgw + api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + - AIzaSyAN2k6IRdgw123 + - AIzaSyAN2k6IRdgw456 + - AIzaSyAN2k6IRdgw789 model: - gemini-1.5-pro - gemini-1.5-flash-exp-0827: gemini-1.5-flash # 重命名后，原来的模型名字 gemini-1.5-flash-exp-0827 无法使用，如果要使用原来的名字，可以在 model 中添加原来的名字，只要加上下面一行就可以使用原来的名字了 - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 tools: true + preferences: + API_KEY_RATE_LIMIT: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min + API_KEY_COOLDOWN_PERIOD: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 60 秒 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 @@ -338,6 +344,41 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ 将 ENABLE_MODERATION 设置为 false 将修复这个问题。当 ENABLE_MODERATION 为 true 时，API 必须能够使用 text-moderation-latest 模型，如果你没有在提供商模型设置里面提供 text-moderation-latest，将会报错找不到模型。 +- 怎么优先请求某个渠道，怎么设置渠道的优先级？ + +直接在api_keys里面通过设置渠道顺序即可。不需要做其他设置，示例配置文件： + +```yaml +providers: + - provider: ai1 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + + - provider: ai2 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + +api_keys: + - api: sk-1234 + model: + - ai2/* + - ai1/* +``` + +这样设置则先请求 ai2，失败后请求 ai1。 + +- 各种调度算法背后的行为是怎样的？比如 fixed_priority，weighted_round_robin，lottery，random，round_robin？ + +所有调度算法需要通过在配置文件的 api_keys.(api).preferences.SCHEDULING_ALGORITHM 设置为 fixed_priority，weighted_round_robin，lottery，random，round_robin 中的任意值来开启。 + +1. fixed_priority：固定优先级调度。所有请求永远执行第一个拥有用户请求的模型的渠道。报错时，会切换下一个渠道。这是默认的调度算法。 + +2. weighted_round_robin：加权轮训负载均衡，按照配置文件 api_keys.(api).model 设定的权重顺序请求拥有用户请求的模型的渠道。 + +3. lottery：抽奖轮训负载均衡，按照配置文件 api_keys.(api).model 设置的权重随机请求拥有用户请求的模型的渠道。 + +4. round_robin：轮训负载均衡，按照配置文件 api_keys.(api).model 的配置顺序请求拥有用户请求的模型的渠道。可以查看上一个问题，如何设置渠道的优先级。 + ## ⭐ Star 历史 diff --git a/main.py b/main.py index 3d4ce56a..2ba7c0b0 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ from log_config import logger -import re import copy import httpx import secrets @@ -19,7 +18,18 @@ from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest, EmbeddingRequest from request import get_payload from response import fetch_response, fetch_response_stream -from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder, get_model_dict, save_api_yaml +from utils import ( + safe_get, + load_config, + save_api_yaml, + get_model_dict, + post_all_models, + get_user_rate_limit, + circular_list_encoder, + error_handling_wrapper, + rate_limiter, + provider_api_circular_list, +) from collections import defaultdict from typing import List, Dict, Union @@ -542,6 +552,7 @@ async def init(self, default_config): @asynccontextmanager async def get_client(self, timeout_value): # 直接获取或创建客户端,不使用锁 + timeout_value = int(timeout_value) if timeout_value not in self.clients: timeout = httpx.Timeout( connect=15.0, @@ -558,8 +569,10 @@ async def get_client(self, timeout_value): try: yield self.clients[timeout_value] except Exception as e: - await self.clients[timeout_value].aclose() - del self.clients[timeout_value] + if timeout_value in self.clients: + tmp_client = self.clients[timeout_value] + del self.clients[timeout_value] # 先删除引用 + await tmp_client.aclose() # 然后关闭客户端 raise e async def close(self): @@ -955,8 +968,13 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) index = 0 + if num_matching_providers == 1 and (count := provider_api_circular_list[matching_providers[0]['provider']].get_items_count()) > 1: + retry_count = count + else: + retry_count = 0 + while True: - if index >= num_matching_providers: + if index >= num_matching_providers + retry_count: break current_index = (start_index + index) % num_matching_providers index += 1 @@ -995,6 +1013,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques num_matching_providers = len(matching_providers) index = 0 + if status_code == 429: + current_api = await provider_api_circular_list[channel_id].after_next_current() + await provider_api_circular_list[channel_id].set_cooling(current_api, cooldown_period=safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=60)) + logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") if is_debug: import traceback @@ -1012,59 +1034,6 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques model_handler = ModelRequestHandler() -def parse_rate_limit(limit_string): - # 定义时间单位到秒的映射 - time_units = { - 's': 1, 'sec': 1, 'second': 1, - 'm': 60, 'min': 60, 'minute': 60, - 'h': 3600, 'hr': 3600, 'hour': 3600, - 'd': 86400, 'day': 86400, - 'mo': 2592000, 'month': 2592000, - 'y': 31536000, 'year': 31536000 - } - - # 使用正则表达式匹配数字和单位 - match = re.match(r'^(\d+)/(\w+)$', limit_string) - if not match: - raise ValueError(f"Invalid rate limit format: {limit_string}") - - count, unit = match.groups() - count = int(count) - - # 转换单位到秒 - if unit not in time_units: - raise ValueError(f"Unknown time unit: {unit}") - - seconds = time_units[unit] - - return (count, seconds) - -class InMemoryRateLimiter: - def __init__(self): - self.requests = defaultdict(list) - - async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: - now = time() - self.requests[key] = [req for req in self.requests[key] if req > now - period] - if len(self.requests[key]) >= limit: - return True - self.requests[key].append(now) - return False - -rate_limiter = InMemoryRateLimiter() - -async def get_user_rate_limit(api_index: str = None): - # 这里应该实现根据 token 获取用户速率限制的逻辑 - # 示例： 返回 (次数， 秒数) - config = app.state.config - raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") - - if not api_index or not raw_rate_limit: - return (30, 60) - - rate_limit = parse_rate_limit(raw_rate_limit) - return rate_limit - security = HTTPBearer() async def rate_limit_dependency(request: Request, credentials: HTTPAuthorizationCredentials = Depends(security)): @@ -1076,7 +1045,7 @@ async def rate_limit_dependency(request: Request, credentials: HTTPAuthorization print("error: Invalid or missing API Key:", token) api_index = None token = None - limit, period = await get_user_rate_limit(api_index) + limit, period = await get_user_rate_limit(app, api_index) # 使用 IP 地址和 token（如果有）作为限制键 client_ip = request.client.host diff --git a/utils.py b/utils.py index 37f3bc26..b908b605 100644 --- a/utils.py +++ b/utils.py @@ -3,22 +3,135 @@ import httpx from log_config import logger + +import re +from time import time +def parse_rate_limit(limit_string): + # 定义时间单位到秒的映射 + time_units = { + 's': 1, 'sec': 1, 'second': 1, + 'm': 60, 'min': 60, 'minute': 60, + 'h': 3600, 'hr': 3600, 'hour': 3600, + 'd': 86400, 'day': 86400, + 'mo': 2592000, 'month': 2592000, + 'y': 31536000, 'year': 31536000 + } + + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit_string) + if not match: + raise ValueError(f"Invalid rate limit format: {limit_string}") + + count, unit = match.groups() + count = int(count) + + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") + + seconds = time_units[unit] + + return (count, seconds) + from collections import defaultdict +class InMemoryRateLimiter: + def __init__(self): + self.requests = defaultdict(list) + + async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: + now = time() + self.requests[key] = [req for req in self.requests[key] if req > now - period] + if len(self.requests[key]) >= limit: + return True + self.requests[key].append(now) + return False + +rate_limiter = InMemoryRateLimiter() + +async def get_user_rate_limit(app, api_index: str = None): + # 这里应该实现根据 token 获取用户速率限制的逻辑 + # 示例： 返回 (次数， 秒数) + config = app.state.config + raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") + # print("raw_rate_limit", raw_rate_limit) + # print("not api_index or not raw_rate_limit", api_index == None, not raw_rate_limit, api_index == None or not raw_rate_limit, api_index, raw_rate_limit) + + if api_index == None or not raw_rate_limit: + return (30, 60) + + rate_limit = parse_rate_limit(raw_rate_limit) + return rate_limit import asyncio class ThreadSafeCircularList: - def __init__(self, items): + def __init__(self, items, rate_limit="99999/min"): self.items = items self.index = 0 self.lock = asyncio.Lock() + self.requests = defaultdict(list) # 用于追踪每个 API key 的请求时间 + self.cooling_until = defaultdict(float) # 记录每个 item 的冷却结束时间 + count, period = parse_rate_limit(rate_limit) + self.rate_limit = count + self.period = period + + async def set_cooling(self, item: str, cooling_time: int = 60): + """设置某个 item 进入冷却状态 + + Args: + item: 需要冷却的 item + cooling_time: 冷却时间(秒)，默认60秒 + """ + now = time() + async with self.lock: + self.cooling_until[item] = now + cooling_time + # 清空该 item 的请求记录 + self.requests[item] = [] + logger.warning(f"API key {item} 已进入冷却状态，冷却时间 {cooling_time} 秒") + + async def is_rate_limited(self, item) -> bool: + now = time() + # 检查是否在冷却中 + if now < self.cooling_until[item]: + return True + + self.requests[item] = [req for req in self.requests[item] if req > now - self.period] + if len(self.requests[item]) >= self.rate_limit: + return True + self.requests[item].append(now) + return False async def next(self): async with self.lock: - item = self.items[self.index] - self.index = (self.index + 1) % len(self.items) + start_index = self.index + while True: + item = self.items[self.index] + self.index = (self.index + 1) % len(self.items) + + if not await self.is_rate_limited(item): + return item + + logger.warning(f"API key {item} 已达到速率限制 ({self.rate_limit}/{self.period}秒)") + + # 如果已经检查了所有的 API key 都被限制 + if self.index == start_index: + logger.warning(f"所有 API key 都已达到速率限制 ({self.rate_limit}/{self.period}秒)") + return None + + async def after_next_current(self): + # 返回当前取出的 API，因为已经调用了 next，所以当前API应该是上一个 + async with self.lock: + item = self.items[(self.index - 1) % len(self.items)] return item + def get_items_count(self) -> int: + """返回列表中的项目数量 + + Returns: + int: items列表的长度 + """ + return len(self.items) + def circular_list_encoder(obj): if isinstance(obj, ThreadSafeCircularList): return obj.to_dict() @@ -84,9 +197,15 @@ def update_config(config_data, use_config_url=False): provider_api = provider.get('api', None) if provider_api: if isinstance(provider_api, str): - provider_api_circular_list[provider['provider']] = ThreadSafeCircularList([provider_api]) + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( + [provider_api], + safe_get(provider, "preferences", "API_KEY_RATE_LIMIT", default="999999/min") + ) if isinstance(provider_api, list): - provider_api_circular_list[provider['provider']] = ThreadSafeCircularList(provider_api) + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( + provider_api, + safe_get(provider, "preferences", "API_KEY_RATE_LIMIT", default="999999/min") + ) if not provider.get("model"): model_list = update_initial_model(provider['base_url'], provider['api']) From e3bab2ccea60bcf340723c03e8a9dc082f2a5325 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 04:55:29 +0800 Subject: [PATCH 215/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20GitHub=20Action=20cannot=20run.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e6dcef5c..b2df4b11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,6 @@ on: jobs: build-and-push: runs-on: ubuntu-latest - if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} steps: - name: Checkout repository From c64a64ebf69c2265d6b4995a50ed8782b41a1f2e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 Nov 2024 20:56:01 +0000 Subject: [PATCH 216/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.68?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9c3f756d..fcae301f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.67 +0.0.68 From 68795da3c1b773f3da338705a2891d7f2a0548dc Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 05:40:48 +0800 Subject: [PATCH 217/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20incorrect=20function=20parameters=20entering=20the=20co?= =?UTF-8?q?oldown=20state.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add feature: Add support for multiple frequency constraints of API key --- README.md | 2 ++ README_CN.md | 2 ++ main.py | 10 +++---- utils.py | 75 +++++++++++++++++++++++++++++++++------------------- 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3e5da90a..265c9df9 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ providers: tools: true preferences: API_KEY_RATE_LIMIT: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. + # API_KEY_RATE_LIMIT: 15/min,10/day # Supports multiple frequency constraints API_KEY_COOLDOWN_PERIOD: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 60 seconds. - provider: vertex @@ -142,6 +143,7 @@ api_keys: # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the model used by the user in order. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional + # RATE_LIMIT: 2/min,10/day # Supports multiple frequency constraints ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. # Channel-level weighted load balancing configuration example diff --git a/README_CN.md b/README_CN.md index 3b7200e0..53b8cfac 100644 --- a/README_CN.md +++ b/README_CN.md @@ -91,6 +91,7 @@ providers: tools: true preferences: API_KEY_RATE_LIMIT: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min + # API_KEY_RATE_LIMIT: 15/min,10/day # 支持多个频率约束条件 API_KEY_COOLDOWN_PERIOD: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 60 秒 - provider: vertex @@ -142,6 +143,7 @@ api_keys: # 当 SCHEDULING_ALGORITHM 为 round_robin 时，使用轮训负载均衡，按照顺序请求用户使用的模型的渠道。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 + # RATE_LIMIT: 2/min,10/day 支持多个频率约束条件 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 # 渠道级加权负载均衡配置示例 diff --git a/main.py b/main.py index 2ba7c0b0..a60e8fc3 100644 --- a/main.py +++ b/main.py @@ -1015,7 +1015,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if status_code == 429: current_api = await provider_api_circular_list[channel_id].after_next_current() - await provider_api_circular_list[channel_id].set_cooling(current_api, cooldown_period=safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=60)) + await provider_api_circular_list[channel_id].set_cooling(current_api, cooling_time=safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=60)) logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") if is_debug: @@ -1045,13 +1045,13 @@ async def rate_limit_dependency(request: Request, credentials: HTTPAuthorization print("error: Invalid or missing API Key:", token) api_index = None token = None - limit, period = await get_user_rate_limit(app, api_index) # 使用 IP 地址和 token（如果有）作为限制键 client_ip = request.client.host rate_limit_key = f"{client_ip}:{token}" if token else client_ip - if await rate_limiter.is_rate_limited(rate_limit_key, limit, period): + limits = await get_user_rate_limit(app, api_index) + if await rate_limiter.is_rate_limited(rate_limit_key, limits): raise HTTPException(status_code=429, detail="Too many requests") def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): @@ -1315,13 +1315,13 @@ async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_k async def frontend_rate_limit_dependency(request: Request, x_api_key: str = Depends(get_api_key)): token = x_api_key if x_api_key else None - limit, period = 100, 60 # 使用 IP 地址和 token（如果有）作为限制键 client_ip = request.client.host rate_limit_key = f"{client_ip}:{token}" if token else client_ip - if await rate_limiter.is_rate_limited(rate_limit_key, limit, period): + limits = [(100, 60)] + if await rate_limiter.is_rate_limited(rate_limit_key, limits): raise HTTPException(status_code=429, detail="Too many requests") # def get_backend_router_api_list(): diff --git a/utils.py b/utils.py index b908b605..14c0e090 100644 --- a/utils.py +++ b/utils.py @@ -17,32 +17,47 @@ def parse_rate_limit(limit_string): 'y': 31536000, 'year': 31536000 } - # 使用正则表达式匹配数字和单位 - match = re.match(r'^(\d+)/(\w+)$', limit_string) - if not match: - raise ValueError(f"Invalid rate limit format: {limit_string}") + # 处理多个限制条件 + limits = [] + for limit in limit_string.split(','): + limit = limit.strip() + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit) + if not match: + raise ValueError(f"Invalid rate limit format: {limit}") - count, unit = match.groups() - count = int(count) + count, unit = match.groups() + count = int(count) - # 转换单位到秒 - if unit not in time_units: - raise ValueError(f"Unknown time unit: {unit}") + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") - seconds = time_units[unit] + seconds = time_units[unit] + limits.append((count, seconds)) - return (count, seconds) + return limits from collections import defaultdict class InMemoryRateLimiter: def __init__(self): self.requests = defaultdict(list) - async def is_rate_limited(self, key: str, limit: int, period: int) -> bool: + async def is_rate_limited(self, key: str, limits) -> bool: now = time() - self.requests[key] = [req for req in self.requests[key] if req > now - period] - if len(self.requests[key]) >= limit: - return True + + # 检查所有速率限制条件 + for limit, period in limits: + # 计算在当前时间窗口内的请求数量 + recent_requests = sum(1 for req in self.requests[key] if req > now - period) + if recent_requests >= limit: + return True + + # 清理太旧的请求记录（比最长时间窗口还要老的记录） + max_period = max(period for _, period in limits) + self.requests[key] = [req for req in self.requests[key] if req > now - max_period] + + # 记录新的请求 self.requests[key].append(now) return False @@ -70,10 +85,8 @@ def __init__(self, items, rate_limit="99999/min"): self.index = 0 self.lock = asyncio.Lock() self.requests = defaultdict(list) # 用于追踪每个 API key 的请求时间 - self.cooling_until = defaultdict(float) # 记录每个 item 的冷却结束时间 - count, period = parse_rate_limit(rate_limit) - self.rate_limit = count - self.period = period + self.cooling_until = defaultdict(float) + self.rate_limits = parse_rate_limit(rate_limit) # 现在返回一个限制条件列表 async def set_cooling(self, item: str, cooling_time: int = 60): """设置某个 item 进入冷却状态 @@ -86,7 +99,7 @@ async def set_cooling(self, item: str, cooling_time: int = 60): async with self.lock: self.cooling_until[item] = now + cooling_time # 清空该 item 的请求记录 - self.requests[item] = [] + # self.requests[item] = [] logger.warning(f"API key {item} 已进入冷却状态，冷却时间 {cooling_time} 秒") async def is_rate_limited(self, item) -> bool: @@ -95,9 +108,19 @@ async def is_rate_limited(self, item) -> bool: if now < self.cooling_until[item]: return True - self.requests[item] = [req for req in self.requests[item] if req > now - self.period] - if len(self.requests[item]) >= self.rate_limit: - return True + # 检查所有速率限制条件 + for limit_count, limit_period in self.rate_limits: + # 计算在当前时间窗口内的请求数量，而不是直接修改请求列表 + recent_requests = sum(1 for req in self.requests[item] if req > now - limit_period) + if recent_requests >= limit_count: + logger.warning(f"API key {item} 已达到速率限制 ({limit_count}/{limit_period}秒)") + return True + + # 清理太旧的请求记录（比最长时间窗口还要老的记录） + max_period = max(period for _, period in self.rate_limits) + self.requests[item] = [req for req in self.requests[item] if req > now - max_period] + + # 所有限制条件都通过，记录新的请求 self.requests[item].append(now) return False @@ -111,12 +134,10 @@ async def next(self): if not await self.is_rate_limited(item): return item - logger.warning(f"API key {item} 已达到速率限制 ({self.rate_limit}/{self.period}秒)") - # 如果已经检查了所有的 API key 都被限制 if self.index == start_index: - logger.warning(f"所有 API key 都已达到速率限制 ({self.rate_limit}/{self.period}秒)") - return None + logger.warning(f"All API keys are rate limited!") + raise HTTPException(status_code=429, detail="Too many requests") async def after_next_current(self): # 返回当前取出的 API，因为已经调用了 next，所以当前API应该是上一个 From 466f9a4d856e3dddc6769e89fbe8fd46450d318a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 Nov 2024 21:41:12 +0000 Subject: [PATCH 218/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.69?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fcae301f..9a52cbd4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.68 +0.0.69 From cb8aca60e76fdc502e331e5914f96abe9133a781 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 16:46:51 +0800 Subject: [PATCH 219/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20return=20data=20type=20of=20frequency=20limit?= =?UTF-8?q?=20is=20incorrect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 14c0e090..c9b03cf9 100644 --- a/utils.py +++ b/utils.py @@ -72,7 +72,7 @@ async def get_user_rate_limit(app, api_index: str = None): # print("not api_index or not raw_rate_limit", api_index == None, not raw_rate_limit, api_index == None or not raw_rate_limit, api_index, raw_rate_limit) if api_index == None or not raw_rate_limit: - return (30, 60) + return [(30, 60)] rate_limit = parse_rate_limit(raw_rate_limit) return rate_limit From 782dde130e397ba3e12caaceebc642a465f97a05 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 08:47:27 +0000 Subject: [PATCH 220/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.70?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9a52cbd4..c8aa910f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.69 +0.0.70 From d30f0dd9fdaf39a9b332d99a065ec2bba524d6d1 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 18:41:41 +0800 Subject: [PATCH 221/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Modify=20the=20d?= =?UTF-8?q?efault=20behavior=20of=20API=20key=20cooldown,=20change=20it=20?= =?UTF-8?q?to=20default=20off=20for=20API=20key=20cooldown.=20When=20there?= =?UTF-8?q?=20is=20only=20one=20API=20key=20in=20the=20channel,=20it=20wil?= =?UTF-8?q?l=20not=20cooldown=20under=20any=20circumstances.=20When=20API?= =?UTF-8?q?=5FKEY=5FCOOLDOWN=5FPERIOD=20is=200,=20it=20will=20not=20cooldo?= =?UTF-8?q?wn.=20After=20enabling=20cooldown,=20any=20error=20with=20the?= =?UTF-8?q?=20API=20key=20will=20trigger=20a=20cooldown.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- main.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 265c9df9..fca511b7 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ providers: preferences: API_KEY_RATE_LIMIT: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. # API_KEY_RATE_LIMIT: 15/min,10/day # Supports multiple frequency constraints - API_KEY_COOLDOWN_PERIOD: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 60 seconds. + API_KEY_COOLDOWN_PERIOD: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. diff --git a/README_CN.md b/README_CN.md index 53b8cfac..9610b094 100644 --- a/README_CN.md +++ b/README_CN.md @@ -92,7 +92,7 @@ providers: preferences: API_KEY_RATE_LIMIT: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min # API_KEY_RATE_LIMIT: 15/min,10/day # 支持多个频率约束条件 - API_KEY_COOLDOWN_PERIOD: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 60 秒 + API_KEY_COOLDOWN_PERIOD: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 diff --git a/main.py b/main.py index a60e8fc3..261a3081 100644 --- a/main.py +++ b/main.py @@ -1013,9 +1013,11 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques num_matching_providers = len(matching_providers) index = 0 - if status_code == 429: + cooling_time = safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=0) + api_key_count = provider_api_circular_list[channel_id].get_items_count() + if cooling_time > 0 and api_key_count > 1: current_api = await provider_api_circular_list[channel_id].after_next_current() - await provider_api_circular_list[channel_id].set_cooling(current_api, cooling_time=safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=60)) + await provider_api_circular_list[channel_id].set_cooling(current_api, cooling_time=cooling_time) logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") if is_debug: From cd2710f49cee4da932801c6c4c4627b0a6b27eba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 10:42:06 +0000 Subject: [PATCH 222/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.71?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c8aa910f..c09f75a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.70 +0.0.71 From a5e6e3459712a7cae063f4ac8f5b4379c7cb0d12 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:16:40 +0800 Subject: [PATCH 223/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Optimize=20to=20?= =?UTF-8?q?reduce=20unnecessary=20log=20output.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 261a3081..486c0553 100644 --- a/main.py +++ b/main.py @@ -620,7 +620,7 @@ async def ensure_config(request: Request, call_next): if "default" not in app.state.config['preferences'].get('model_timeout', {}): app.state.timeouts["default"] = DEFAULT_TIMEOUT - print("app.state.timeouts", app.state.timeouts) + # print("app.state.timeouts", app.state.timeouts) if app and not hasattr(app.state, "channel_manager"): if app.state.config and 'preferences' in app.state.config: diff --git a/utils.py b/utils.py index c9b03cf9..1e2c1469 100644 --- a/utils.py +++ b/utils.py @@ -289,7 +289,8 @@ async def load_config(app=None): logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") config, api_keys_db, api_list = {}, {}, [] except FileNotFoundError: - logger.error("'api.yaml' not found. Please check the file path.") + if not os.environ.get('CONFIG_URL'): + logger.error("'api.yaml' not found. Please check the file path.") config, api_keys_db, api_list = {}, {}, [] except YAMLError as e: logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。%s", e) From e762b8580bbab7054eadd750c8928b288c270702 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 13:16:58 +0000 Subject: [PATCH 224/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.72?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c09f75a3..36e6a204 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.71 +0.0.72 From 09d868f0f1d380f5aee3af6a5399f853685d095b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:26:24 +0800 Subject: [PATCH 225/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Increase=20the?= =?UTF-8?q?=20maximum=20runtime=20to=2060=20seconds=20when=20deploying=20w?= =?UTF-8?q?ith=20Vercel.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vercel.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vercel.json b/vercel.json index bc1bf397..455e1c3c 100644 --- a/vercel.json +++ b/vercel.json @@ -10,5 +10,10 @@ "src": "/(.*)", "dest": "main.py" } - ] + ], + "functions": { + "main.py": { + "maxDuration": 60 + } + } } \ No newline at end of file From 7904b10f324c0b5e2e7bfc20bc450c2d2636a0a4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:29:08 +0800 Subject: [PATCH 226/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20functions=20and=20builds=20properties=20cannot=20be?= =?UTF-8?q?=20used=20simultaneously=20in=20Vercel=20configuration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vercel.json | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/vercel.json b/vercel.json index 455e1c3c..0034d88a 100644 --- a/vercel.json +++ b/vercel.json @@ -1,19 +1,17 @@ { - "builds": [ - { - "src": "main.py", - "use": "@vercel/python" - } - ], - "routes": [ - { - "src": "/(.*)", - "dest": "main.py" - } - ], - "functions": { - "main.py": { + "builds": [ + { + "src": "main.py", + "use": "@vercel/python", + "config": { "maxDuration": 60 } } - } \ No newline at end of file + ], + "routes": [ + { + "src": "/(.*)", + "dest": "main.py" + } + ] +} \ No newline at end of file From 92795caa767e6dfdeab6b95d32cb56d47b569dd2 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:38:24 +0800 Subject: [PATCH 227/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20os=20module=20was=20not=20imported.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 1e2c1469..cda0fba5 100644 --- a/utils.py +++ b/utils.py @@ -279,6 +279,7 @@ def update_config(config_data, use_config_url=False): # 读取YAML配置文件 async def load_config(app=None): + import os try: with open(API_YAML_PATH, 'r', encoding='utf-8') as file: conf = yaml.load(file) @@ -302,7 +303,6 @@ async def load_config(app=None): if config != {}: return config, api_keys_db, api_list - import os # 新增： 从环境变量获取配置URL并拉取配置 config_url = os.environ.get('CONFIG_URL') if config_url: From 9790e2d145b28485b0355912b209160889cade8d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 13:38:49 +0000 Subject: [PATCH 228/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.73?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 36e6a204..2225cdf1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.72 +0.0.73 From a80a8594fd2dd4513e07055999033655916c05fc Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:51:32 +0800 Subject: [PATCH 229/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Add=20log=20disp?= =?UTF-8?q?lay=20for=20model=20not=20found.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 486c0553..2b7a325c 100644 --- a/main.py +++ b/main.py @@ -879,7 +879,7 @@ async def get_right_order_providers(request_model, config, api_index, scheduling matching_providers = get_matching_providers(request_model, config, api_index) if not matching_providers: - raise HTTPException(status_code=404, detail="No matching model found") + raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") num_matching_providers = len(matching_providers) if app.state.channel_manager.cooldown_period > 0 and num_matching_providers > 1: @@ -947,10 +947,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques api_list = app.state.api_list api_index = api_list.index(token) + request_model = request.model if not safe_get(config, 'api_keys', api_index, 'model'): - raise HTTPException(status_code=404, detail="No matching model found") + raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") - request_model = request.model scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) From d1ebf230bd5761c23359bb5feda1d9fbd67be957 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 13:51:58 +0000 Subject: [PATCH 230/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.74?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2225cdf1..30eb585d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.73 +0.0.74 From 3e31cf53f91acf04f44c2465c8e753ec59278ca3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:54:41 +0800 Subject: [PATCH 231/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20maxDuration=20in=20the=20Vercel=20configuratio?= =?UTF-8?q?n=20file=20does=20not=20take=20effect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vercel.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vercel.json b/vercel.json index 0034d88a..032a3be6 100644 --- a/vercel.json +++ b/vercel.json @@ -3,8 +3,8 @@ { "src": "main.py", "use": "@vercel/python", - "config": { - "maxDuration": 60 + "functions": { + "main.py": { "maxDuration": 60 } } } ], From a3a763544c0b0d7804dc0971c4b4654101afe91c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 21:59:30 +0800 Subject: [PATCH 232/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Remove=20the=20m?= =?UTF-8?q?axDuration=20field=20from=20the=20vercel=20configuration=20file?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vercel.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vercel.json b/vercel.json index 032a3be6..6e08ab58 100644 --- a/vercel.json +++ b/vercel.json @@ -2,10 +2,7 @@ "builds": [ { "src": "main.py", - "use": "@vercel/python", - "functions": { - "main.py": { "maxDuration": 60 } - } + "use": "@vercel/python" } ], "routes": [ From 1778d52c924b5ba5f61d6282631553825e7e237c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 5 Nov 2024 22:11:58 +0800 Subject: [PATCH 233/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fca511b7..cdbc6df7 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ yym68686/uni-api:latest [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) -After clicking the one-click deployment button, set the environment variable `CONFIG_URL` to the direct link of the configuration file, and set `DISABLE_DATABASE` to true, then click Create to create the project. +After clicking the one-click deploy button above, set the environment variable `CONFIG_URL` to the direct link of the configuration file, `DISABLE_DATABASE` to true, and then click Create to create the project. After deployment, you need to manually set the Function Max Duration to 60 seconds in the Vercel project panel under Settings -> Functions, and then click the Deployments menu and click Redeploy to redeploy, which will set the timeout to 60 seconds. If you do not redeploy, the default timeout will remain at the original 10 seconds. Note that you should not delete the Vercel project and recreate it; instead, click redeploy in the Deployments menu within the currently deployed Vercel project to make the Function Max Duration modification take effect. ## Serv00 remote deployment diff --git a/README_CN.md b/README_CN.md index 9610b094..bb048d3a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -195,7 +195,7 @@ yym68686/uni-api:latest [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) -点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。 +点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。部署完之后需要手动在 vercel 项目面板的 Settings -> Funcitons -> Function Max Duration 设置为 60 秒，然后点击 Deployments 菜单点击 Redeploy 重新部署，即可将超时时间设置为 60 秒，如果不重新部署，默认超时时间将是原来的 10 秒。注意不是删掉 vercel 项目重建，而是在当前部署好的 vercel 项目里面的 Deployments 菜单里面点 redeploy，这样才能让 Function Max Duration 的修改生效。 ## serv00 远程部署 From cdf3ed903e7f51db37686f2c61f387e6da09150a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 01:05:33 +0800 Subject: [PATCH 234/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20support=20setting=20rate=20limit=20for=20each=20model=20indi?= =?UTF-8?q?vidually?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 +++++--- README_CN.md | 9 +++++--- main.py | 27 +++++++++++----------- request.py | 45 ++++++++++++++++++------------------ utils.py | 64 ++++++++++++++++++++++++++++++++++++++-------------- 5 files changed, 95 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index cdbc6df7..8ed80f78 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,12 @@ providers: - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true preferences: - API_KEY_RATE_LIMIT: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. - # API_KEY_RATE_LIMIT: 15/min,10/day # Supports multiple frequency constraints - API_KEY_COOLDOWN_PERIOD: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. + api_key_rate_limit: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day + # api_key_rate_limit: # You can set different frequency limits for each model + # gpt-4o: 3/min + # chatgpt-4o-latest: 2/min + # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default + api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. diff --git a/README_CN.md b/README_CN.md index bb048d3a..4afb0bf8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -90,9 +90,12 @@ providers: - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 tools: true preferences: - API_KEY_RATE_LIMIT: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min - # API_KEY_RATE_LIMIT: 15/min,10/day # 支持多个频率约束条件 - API_KEY_COOLDOWN_PERIOD: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。 + api_key_rate_limit: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min。支持多个频率约束条件：15/min,10/day + # api_key_rate_limit: # 可以为每个模型设置不同的频率限制 + # gpt-4o: 3/min + # chatgpt-4o-latest: 2/min + # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 + api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 diff --git a/main.py b/main.py index 2b7a325c..b703dbd5 100644 --- a/main.py +++ b/main.py @@ -655,20 +655,22 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "gpt" model_dict = get_model_dict(provider) - if "claude" not in model_dict[request.model] \ - and "gpt" not in model_dict[request.model] \ - and "gemini" not in model_dict[request.model] \ + original_model = model_dict[request.model] + + if "claude" not in original_model \ + and "gpt" not in original_model \ + and "gemini" not in original_model \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" - if "claude" in model_dict[request.model] and engine == "vertex": + if "claude" in original_model and engine == "vertex": engine = "vertex-claude" - if "gemini" in model_dict[request.model] and engine == "vertex": + if "gemini" in original_model and engine == "vertex": engine = "vertex-gemini" - if "o1-preview" in model_dict[request.model] or "o1-mini" in model_dict[request.model]: + if "o1-preview" in original_model or "o1-mini" in original_model: engine = "o1" request.stream = False @@ -702,17 +704,16 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) current_info = request_info.get() - model = model_dict[request.model] timeout_value = None # 先尝试精确匹配 - if model in app.state.timeouts: - timeout_value = app.state.timeouts[model] + if original_model in app.state.timeouts: + timeout_value = app.state.timeouts[original_model] else: # 如果没有精确匹配，尝试模糊匹配 for timeout_model in app.state.timeouts: - if timeout_model in model: + if timeout_model in original_model: timeout_value = app.state.timeouts[timeout_model] break @@ -723,11 +724,11 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A try: async with app.state.client_manager.get_client(timeout_value) as client: if request.stream: - generator = fetch_response_stream(client, url, headers, payload, engine, model) + generator = fetch_response_stream(client, url, headers, payload, engine, original_model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: - generator = fetch_response(client, url, headers, payload, engine, model) + generator = fetch_response(client, url, headers, payload, engine, original_model) wrapped_generator, first_response_time = await error_handling_wrapper(generator) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") @@ -1013,7 +1014,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques num_matching_providers = len(matching_providers) index = 0 - cooling_time = safe_get(provider, "preferences", "API_KEY_COOLDOWN_PERIOD", default=0) + cooling_time = safe_get(provider, "preferences", "api_key_cooldown_period", default=0) api_key_count = provider_api_circular_list[channel_id].get_items_count() if cooling_time > 0 and api_key_count > 1: current_api = await provider_api_circular_list[channel_id].after_next_current() diff --git a/request.py b/request.py index 241c2344..623a3571 100644 --- a/request.py +++ b/request.py @@ -125,9 +125,9 @@ async def get_gemini_payload(request, engine, provider): gemini_stream = "streamGenerateContent" url = provider['base_url'] if url.endswith("v1beta"): - url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next()) + url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next(model)) if url.endswith("v1"): - url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next()) + url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next(model)) messages = [] systemInstruction = None @@ -596,8 +596,10 @@ async def get_gpt_payload(request, engine, provider): headers = { 'Content-Type': 'application/json', } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] messages = [] @@ -637,8 +639,6 @@ async def get_gpt_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) - model_dict = get_model_dict(provider) - model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -663,8 +663,10 @@ async def get_openrouter_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] @@ -696,8 +698,6 @@ async def get_openrouter_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) - model_dict = get_model_dict(provider) - model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -730,8 +730,10 @@ async def get_cohere_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] @@ -759,8 +761,6 @@ async def get_cohere_payload(request, engine, provider): else: messages.append({"role": role_map[msg.role], "message": content}) - model_dict = get_model_dict(provider) - model = model_dict[request.model] chat_history = messages[:-1] query = messages[-1].get("message") payload = { @@ -798,11 +798,11 @@ async def get_cloudflare_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } - if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" - model_dict = get_model_dict(provider) model = model_dict[request.model] + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + url = "https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model_id}".format(cf_account_id=provider['cf_account_id'], cf_model_id=model) msg = request.messages[-1] @@ -816,7 +816,6 @@ async def get_cloudflare_payload(request, engine, provider): content = msg.content name = msg.name - model = model_dict[request.model] payload = { "prompt": content, } @@ -848,8 +847,10 @@ async def get_o1_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] @@ -871,8 +872,6 @@ async def get_o1_payload(request, engine, provider): elif msg.role != "system": messages.append({"role": msg.role, "content": content}) - model_dict = get_model_dict(provider) - model = model_dict[request.model] payload = { "model": model, "messages": messages, @@ -925,7 +924,7 @@ async def get_claude_payload(request, engine, provider): model = model_dict[request.model] headers = { "content-type": "application/json", - "x-api-key": f"{await provider_api_circular_list[provider['provider']].next()}", + "x-api-key": f"{await provider_api_circular_list[provider['provider']].next(model)}", "anthropic-version": "2023-06-01", "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" if "claude-3-5-sonnet" in model else "tools-2024-05-16", } @@ -1068,7 +1067,7 @@ async def get_dalle_payload(request, engine, provider): "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] url = BaseAPI(url).image_url @@ -1088,7 +1087,7 @@ async def get_whisper_payload(request, engine, provider): # "Content-Type": "multipart/form-data", } if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] url = BaseAPI(url).audio_transcriptions @@ -1115,7 +1114,7 @@ async def get_moderation_payload(request, engine, provider): "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] url = BaseAPI(url).moderations @@ -1132,7 +1131,7 @@ async def get_embedding_payload(request, engine, provider): "Content-Type": "application/json", } if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next()}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] url = BaseAPI(url).embeddings diff --git a/utils.py b/utils.py index cda0fba5..156dcc26 100644 --- a/utils.py +++ b/utils.py @@ -80,13 +80,21 @@ async def get_user_rate_limit(app, api_index: str = None): import asyncio class ThreadSafeCircularList: - def __init__(self, items, rate_limit="99999/min"): + def __init__(self, items, rate_limit={"default": "999999/min"}): self.items = items self.index = 0 self.lock = asyncio.Lock() - self.requests = defaultdict(list) # 用于追踪每个 API key 的请求时间 + # 修改为二级字典，第一级是item，第二级是model + self.requests = defaultdict(lambda: defaultdict(list)) self.cooling_until = defaultdict(float) - self.rate_limits = parse_rate_limit(rate_limit) # 现在返回一个限制条件列表 + self.rate_limits = {} + if isinstance(rate_limit, dict): + for rate_limit_model, rate_limit_value in rate_limit.items(): + self.rate_limits[rate_limit_model] = parse_rate_limit(rate_limit_value) + elif isinstance(rate_limit, str): + self.rate_limits["default"] = parse_rate_limit(rate_limit) + else: + logger.error(f"Error ThreadSafeCircularList: Unknown rate_limit type: {type(rate_limit)}, rate_limit: {rate_limit}") async def set_cooling(self, item: str, cooling_time: int = 60): """设置某个 item 进入冷却状态 @@ -102,36 +110,58 @@ async def set_cooling(self, item: str, cooling_time: int = 60): # self.requests[item] = [] logger.warning(f"API key {item} 已进入冷却状态，冷却时间 {cooling_time} 秒") - async def is_rate_limited(self, item) -> bool: + async def is_rate_limited(self, item, model: str = None) -> bool: now = time() # 检查是否在冷却中 if now < self.cooling_until[item]: return True + # 获取适用的速率限制 + + if model: + model_key = model + else: + model_key = "default" + + rate_limit = None + # 先尝试精确匹配 + if model and model in self.rate_limits: + rate_limit = self.rate_limits[model] + else: + # 如果没有精确匹配，尝试模糊匹配 + for limit_model in self.rate_limits: + if limit_model != "default" and model and limit_model in model: + rate_limit = self.rate_limits[limit_model] + break + + # 如果都没匹配到，使用默认值 + if rate_limit is None: + rate_limit = self.rate_limits.get("default", [(999999, 60)]) # 默认限制 + # 检查所有速率限制条件 - for limit_count, limit_period in self.rate_limits: - # 计算在当前时间窗口内的请求数量，而不是直接修改请求列表 - recent_requests = sum(1 for req in self.requests[item] if req > now - limit_period) + for limit_count, limit_period in rate_limit: + # 使用特定模型的请求记录进行计算 + recent_requests = sum(1 for req in self.requests[item][model_key] if req > now - limit_period) if recent_requests >= limit_count: - logger.warning(f"API key {item} 已达到速率限制 ({limit_count}/{limit_period}秒)") + logger.warning(f"API key {item} 对模型 {model_key} 已达到速率限制 ({limit_count}/{limit_period}秒)") return True - # 清理太旧的请求记录（比最长时间窗口还要老的记录） - max_period = max(period for _, period in self.rate_limits) - self.requests[item] = [req for req in self.requests[item] if req > now - max_period] + # 清理太旧的请求记录 + max_period = max(period for _, period in rate_limit) + self.requests[item][model_key] = [req for req in self.requests[item][model_key] if req > now - max_period] - # 所有限制条件都通过，记录新的请求 - self.requests[item].append(now) + # 记录新的请求 + self.requests[item][model_key].append(now) return False - async def next(self): + async def next(self, model: str = None): async with self.lock: start_index = self.index while True: item = self.items[self.index] self.index = (self.index + 1) % len(self.items) - if not await self.is_rate_limited(item): + if not await self.is_rate_limited(item, model): return item # 如果已经检查了所有的 API key 都被限制 @@ -220,12 +250,12 @@ def update_config(config_data, use_config_url=False): if isinstance(provider_api, str): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( [provider_api], - safe_get(provider, "preferences", "API_KEY_RATE_LIMIT", default="999999/min") + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}) ) if isinstance(provider_api, list): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( provider_api, - safe_get(provider, "preferences", "API_KEY_RATE_LIMIT", default="999999/min") + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}) ) if not provider.get("model"): From 8067d72b117f478b3a5e9b560ea6ec12137045d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 17:05:53 +0000 Subject: [PATCH 235/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.75?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 30eb585d..fb1e9b1d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.74 +0.0.75 From c8dbefdb9df9cee3e29e522d5903842cf34fe9ee Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 01:08:01 +0800 Subject: [PATCH 236/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- README_CN.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8ed80f78..20e3c63c 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ providers: preferences: api_key_rate_limit: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day # api_key_rate_limit: # You can set different frequency limits for each model - # gpt-4o: 3/min - # chatgpt-4o-latest: 2/min + # gemini-1.5-pro: 3/min + # gemini-1.5-flash: 2/min # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. diff --git a/README_CN.md b/README_CN.md index 4afb0bf8..57842019 100644 --- a/README_CN.md +++ b/README_CN.md @@ -92,8 +92,8 @@ providers: preferences: api_key_rate_limit: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min。支持多个频率约束条件：15/min,10/day # api_key_rate_limit: # 可以为每个模型设置不同的频率限制 - # gpt-4o: 3/min - # chatgpt-4o-latest: 2/min + # gemini-1.5-pro: 3/min + # gemini-1.5-flash: 2/min # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 From 4e9c49c3279bb8cc1bab15192c424012a0a2be2e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 03:35:50 +0800 Subject: [PATCH 237/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20API=20initialization=20throws=20an=20error=20when=20?= =?UTF-8?q?no=20API=20key=20is=20present.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 156dcc26..6c16ea8a 100644 --- a/utils.py +++ b/utils.py @@ -80,7 +80,7 @@ async def get_user_rate_limit(app, api_index: str = None): import asyncio class ThreadSafeCircularList: - def __init__(self, items, rate_limit={"default": "999999/min"}): + def __init__(self, items = [], rate_limit={"default": "999999/min"}): self.items = items self.index = 0 self.lock = asyncio.Lock() @@ -247,6 +247,8 @@ def update_config(config_data, use_config_url=False): provider_api = provider.get('api', None) if provider_api: + if isinstance(provider_api, int): + provider_api = str(provider_api) if isinstance(provider_api, str): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( [provider_api], From 26c50ad4d0cbe184486513055be46743148f9992 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 19:36:10 +0000 Subject: [PATCH 238/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.76?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fb1e9b1d..7818a4fc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.75 +0.0.76 From 11f264ceef81df68190e43946cf7cc199fabc6cb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 04:08:44 +0800 Subject: [PATCH 239/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20issue?= =?UTF-8?q?=20where=20an=20error=20is=20returned=20directly=20when=20Autho?= =?UTF-8?q?rization=20is=20empty.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💻 Code: Optimize log display --- main.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b703dbd5..0200a52d 100644 --- a/main.py +++ b/main.py @@ -408,7 +408,14 @@ async def dispatch(self, request: Request, call_next): if request.headers.get("x-api-key"): token = request.headers.get("x-api-key") elif request.headers.get("Authorization"): - token = request.headers.get("Authorization").split(" ")[1] + api_split_list = request.headers.get("Authorization").split(" ") + if len(api_split_list) > 1: + token = api_split_list[1] + else: + return JSONResponse( + status_code=403, + content={"error": "Invalid or missing API Key"} + ) else: token = None if token: @@ -1016,11 +1023,11 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques cooling_time = safe_get(provider, "preferences", "api_key_cooldown_period", default=0) api_key_count = provider_api_circular_list[channel_id].get_items_count() + current_api = await provider_api_circular_list[channel_id].after_next_current() if cooling_time > 0 and api_key_count > 1: - current_api = await provider_api_circular_list[channel_id].after_next_current() await provider_api_circular_list[channel_id].set_cooling(current_api, cooling_time=cooling_time) - logger.error(f"Error {status_code} with provider {channel_id}: {error_message}") + logger.error(f"Error {status_code} with provider {channel_id} API key: {current_api}: {error_message}") if is_debug: import traceback traceback.print_exc() From ec11ef5e800f012415eee8172d5f5103c31cd396 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 20:09:18 +0000 Subject: [PATCH 240/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.77?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7818a4fc..b76f49a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.76 +0.0.77 From cd90ffda118af0622dddab37f1ea0e8b8d17d412 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 04:34:02 +0800 Subject: [PATCH 241/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20GitHub=20models=20cannot=20retrieve=20the=20model=20?= =?UTF-8?q?when=20the=20model=20field=20is=20not=20set.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 6c16ea8a..21994cb9 100644 --- a/utils.py +++ b/utils.py @@ -205,9 +205,10 @@ def update_initial_model(api_url, api): endpoint_models_url = endpoint.v1_models if isinstance(api, list): api = api[0] + headers = {"Authorization": f"Bearer {api}"} response = httpx.get( endpoint_models_url, - headers={"Authorization": f"Bearer {api}"}, + headers=headers, ) models = response.json() if models.get("error"): @@ -260,6 +261,16 @@ def update_config(config_data, use_config_url=False): safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}) ) + if "models.inference.ai.azure.com" in provider['base_url'] and not provider.get("model"): + provider['model'] = [ + "gpt-4o", + "gpt-4o-mini", + "o1-mini", + "o1-preview", + "text-embedding-3-small", + "text-embedding-3-large", + ] + if not provider.get("model"): model_list = update_initial_model(provider['base_url'], provider['api']) if model_list: From 12a925dca81e7749cdec8230161a100bcddbca53 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 Nov 2024 20:34:24 +0000 Subject: [PATCH 242/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.78?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b76f49a4..4ed248ba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.77 +0.0.78 From e0315959b4bd3cfaefb9e2d724276dbed682bcd7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 05:52:41 +0800 Subject: [PATCH 243/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++++ README_CN.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 20e3c63c..96dda828 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,10 @@ All scheduling algorithms need to be enabled by setting api_keys.(api).preferenc 4. round_robin: Round-robin load balancing, requests the channel that owns the model requested by the user according to the configuration order in the configuration file api_keys.(api).model. You can check the previous question on how to set the priority of channels. +- How should the base_url be filled in correctly? + +Except for some special channels shown in the advanced configuration, all OpenAI format providers need to fill in the base_url completely, which means the base_url must end with /v1/chat/completions. If you are using GitHub models, the base_url should be filled in as https://models.inference.ai.azure.com/chat/completion, not Azure's URL. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index 57842019..826ba749 100644 --- a/README_CN.md +++ b/README_CN.md @@ -384,6 +384,10 @@ api_keys: 4. round_robin：轮训负载均衡，按照配置文件 api_keys.(api).model 的配置顺序请求拥有用户请求的模型的渠道。可以查看上一个问题，如何设置渠道的优先级。 +- 应该怎么正确填写 base_url？ + +除了高级配置里面所展示的一些特殊的渠道，所有 OpenAI 格式的提供商需要把 base_url 填完整，也就是说 base_url 必须以 /v1/chat/completions 结尾。如果你使用的 GitHub models，base_url 应该填写为 https://models.inference.ai.azure.com/chat/completion，而不是 Azure 的 URL。 + ## ⭐ Star 历史 From af636f77db043b36a290f3905077d9c08f09f1d6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 20:02:28 +0800 Subject: [PATCH 244/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20ZeroDivisionError=20when=20the=20API=20key=20does=20not?= =?UTF-8?q?=20exist.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils.py b/utils.py index 21994cb9..8907f9b7 100644 --- a/utils.py +++ b/utils.py @@ -103,6 +103,8 @@ async def set_cooling(self, item: str, cooling_time: int = 60): item: 需要冷却的 item cooling_time: 冷却时间(秒)，默认60秒 """ + if item == None: + return now = time() async with self.lock: self.cooling_until[item] = now + cooling_time @@ -171,6 +173,8 @@ async def next(self, model: str = None): async def after_next_current(self): # 返回当前取出的 API，因为已经调用了 next，所以当前API应该是上一个 + if len(self.items) == 0: + return None async with self.lock: item = self.items[(self.index - 1) % len(self.items)] return item From 0b31e2efd48aee25dd0352804225d4a2ac417900 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Nov 2024 12:02:58 +0000 Subject: [PATCH 245/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.79?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4ed248ba..2786fad6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.78 +0.0.79 From 9bde0e5dda49c5ec90fea51876811982bbb05ced Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 6 Nov 2024 21:39:46 +0800 Subject: [PATCH 246/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20response=20body=20error=20returned=20by=20the?= =?UTF-8?q?=20error=20report=20incorrectly=20uses=20raise.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 0200a52d..3995e83b 100644 --- a/main.py +++ b/main.py @@ -1034,13 +1034,19 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if auto_retry: continue else: - raise HTTPException(status_code=status_code, detail=f"Error: Current provider response failed: {error_message}") + return JSONResponse( + status_code=status_code, + content={"error": f"Error: Current provider response failed: {error_message}"} + ) current_info = request_info.get() current_info["first_response_time"] = -1 current_info["success"] = False current_info["provider"] = None - raise HTTPException(status_code=status_code, detail=f"All {request.model} error: {error_message}") + return JSONResponse( + status_code=status_code, + content={"error": f"All {request.model} error: {error_message}"} + ) model_handler = ModelRequestHandler() From 1b4cad3feb59a1c008d2f0a55c8e408502440d05 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Nov 2024 13:40:08 +0000 Subject: [PATCH 247/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2786fad6..ee92deb8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.79 +0.0.80 From 478e740dea35cef6d944c0bbcaf8347a26bd2822 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 7 Nov 2024 18:30:02 +0800 Subject: [PATCH 248/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20The=20bug=20where?= =?UTF-8?q?=20the=20provider's=20name,=20which=20is=20purely=20numeric,=20?= =?UTF-8?q?was=20not=20converted=20to=20a=20string=20has=20been=20fixed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- utils.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 3995e83b..018ecf69 100644 --- a/main.py +++ b/main.py @@ -799,7 +799,7 @@ def lottery_scheduling(weights): def get_provider_rules(model_rule, config, request_model): provider_rules = [] if model_rule == "all": - # 如���模型名为 all，则返回所有模型 + # 如模型名为 all，则返回所有模型 for provider in config["providers"]: model_dict = get_model_dict(provider) for model in model_dict.keys(): diff --git a/utils.py b/utils.py index 8907f9b7..972693fb 100644 --- a/utils.py +++ b/utils.py @@ -250,6 +250,9 @@ def update_config(config_data, use_config_url=False): if provider.get('cf_account_id'): provider['base_url'] = 'https://api.cloudflare.com/' + if isinstance(provider['provider'], int): + provider['provider'] = str(provider['provider']) + provider_api = provider.get('api', None) if provider_api: if isinstance(provider_api, int): @@ -440,7 +443,12 @@ async def new_generator(): try: async for item in generator: yield ensure_string(item) - except (httpx.ReadError, asyncio.CancelledError, httpx.RemoteProtocolError) as e: + except asyncio.CancelledError: + # 客户端断开连接是正常行为，不需要记录错误日志 + logger.debug("Stream cancelled by client") + return + except (httpx.ReadError, httpx.RemoteProtocolError) as e: + # 只记录真正的网络错误 logger.error(f"Network error in new_generator: {e}") raise From b11023ac6810011775ec266ae95839cc36515eab Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 10:30:29 +0000 Subject: [PATCH 249/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee92deb8..1a793f04 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.80 +0.0.81 From a7303e84146788dea484d357140a313c43684f60 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 7 Nov 2024 18:58:57 +0800 Subject: [PATCH 250/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20embedding=20URL=20for=20GLM=20and=20GitHub=20m?= =?UTF-8?q?odels=20is=20incorrect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_baseurl.py | 22 ++++++++++++++++++++++ utils.py | 22 ++++++++++------------ 2 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 test/test_baseurl.py diff --git a/test/test_baseurl.py b/test/test_baseurl.py new file mode 100644 index 00000000..90d23aca --- /dev/null +++ b/test/test_baseurl.py @@ -0,0 +1,22 @@ +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from utils import BaseAPI + +def print_base_api(url): + base_api = BaseAPI(url) + print("base_url ", base_api.base_url) + print("v1_url ", base_api.v1_url) + print("chat_url ", base_api.chat_url) + print("image_url ", base_api.image_url) + print("audio_transcriptions", base_api.audio_transcriptions) + print("moderations ", base_api.moderations) + print("embeddings ", base_api.embeddings) + print("-"*50) + + +print_base_api("https://api.openai.com/v1/chat/completions") +print_base_api("https://api.deepseek.com/chat/completions") +print_base_api("https://models.inference.ai.azure.com/chat/completions") +print_base_api("https://open.bigmodel.cn/api/paas/v4/chat/completions") diff --git a/utils.py b/utils.py index 972693fb..a9875219 100644 --- a/utils.py +++ b/utils.py @@ -573,23 +573,21 @@ def __init__( self.source_api_url: str = api_url from urllib.parse import urlparse, urlunparse parsed_url = urlparse(self.source_api_url) + # print("parsed_url", parsed_url) if parsed_url.scheme == "": raise Exception("Error: API_URL is not set") if parsed_url.path != '/': - before_v1 = parsed_url.path.split("/v1")[0] + before_v1 = parsed_url.path.split("chat/completions")[0] else: before_v1 = "" - self.base_url: str = urlunparse(parsed_url[:2] + (before_v1,) + ("",) * 3) - self.v1_url: str = urlunparse(parsed_url[:2]+ (before_v1 + "/v1",) + ("",) * 3) - self.v1_models: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/models",) + ("",) * 3) - if parsed_url.netloc == "api.deepseek.com": - self.chat_url: str = urlunparse(parsed_url[:2] + ("/chat/completions",) + ("",) * 3) - else: - self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/chat/completions",) + ("",) * 3) - self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/images/generations",) + ("",) * 3) - self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/audio/transcriptions",) + ("",) * 3) - self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/moderations",) + ("",) * 3) - self.embeddings: str = urlunparse(parsed_url[:2] + (before_v1 + "/v1/embeddings",) + ("",) * 3) + self.base_url: str = urlunparse(parsed_url[:2] + ("",) + ("",) * 3) + self.v1_url: str = urlunparse(parsed_url[:2]+ (before_v1,) + ("",) * 3) + self.v1_models: str = urlunparse(parsed_url[:2] + (before_v1 + "models",) + ("",) * 3) + self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "chat/completions",) + ("",) * 3) + self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "images/generations",) + ("",) * 3) + self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "audio/transcriptions",) + ("",) * 3) + self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "moderations",) + ("",) * 3) + self.embeddings: str = urlunparse(parsed_url[:2] + (before_v1 + "embeddings",) + ("",) * 3) def safe_get(data, *keys, default=None): for key in keys: From 1c49e1ae2e21699871b15e2d6cce1ae78326527b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 10:59:19 +0000 Subject: [PATCH 251/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1a793f04..2c2b6571 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.81 +0.0.82 From ecf565074c0b2ec1d77a7babdebbe24eb55fed01 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 7 Nov 2024 22:49:29 +0800 Subject: [PATCH 252/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20that=20as=20long=20as=20the=20prefix=20of=20the=20?= =?UTF-8?q?API=20key=20exists=20in=20the=20configuration=20file,=20the=20A?= =?UTF-8?q?PI=20key=20is=20valid.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 89 +++++++++++++++++++++++++++++++++----------------------- utils.py | 7 ++--- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/main.py b/main.py index 018ecf69..91f0c6c3 100644 --- a/main.py +++ b/main.py @@ -418,14 +418,20 @@ async def dispatch(self, request: Request, call_next): ) else: token = None + + api_index = None if token: try: api_list = app.state.api_list api_index = api_list.index(token) - enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) # token不在api_list中，使用默认值（不开启） pass + + if api_index is not None: + enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) else: # 如果token为None，检查全局设置 enable_moderation = config.get('ENABLE_MODERATION', False) @@ -473,7 +479,7 @@ async def dispatch(self, request: Request, call_next): if enable_moderation and moderated_content: - moderation_response = await self.moderate_content(moderated_content, token) + moderation_response = await self.moderate_content(moderated_content, api_index) is_flagged = moderation_response.get('results', [{}])[0].get('flagged', False) if is_flagged: @@ -518,11 +524,11 @@ async def dispatch(self, request: Request, call_next): # print("current_request_info", current_request_info) request_info.reset(current_request_info) - async def moderate_content(self, content, token): + async def moderate_content(self, content, api_index): moderation_request = ModerationRequest(input=content) # 直接调用 moderations 函数 - response = await moderations(moderation_request, token) + response = await moderations(moderation_request, api_index) # 读取流式响应的内容 moderation_result = b"" @@ -640,7 +646,7 @@ async def ensure_config(request: Request, call_next): return await call_next(request) # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, token=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -745,17 +751,14 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A # response = JSONResponse(first_element) # 更新成功计数和首次响应时间 - await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) - # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=True) + await update_channel_stats(current_info["request_id"], provider['provider'], request.model, current_info["api_key"], success=True) current_info["first_response_time"] = first_response_time current_info["success"] = True current_info["provider"] = provider['provider'] return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: - await update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) - # await app.middleware_stack.app.update_channel_stats(current_info["request_id"], provider['provider'], request.model, token, success=False) - + await update_channel_stats(current_info["request_id"], provider['provider'], request.model, current_info["api_key"], success=False) raise e def weighted_round_robin(weights): @@ -950,11 +953,8 @@ def __init__(self): self.last_provider_indices = defaultdict(lambda: -1) self.locks = defaultdict(asyncio.Lock) - async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], token: str, endpoint=None): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], api_index: int = None, endpoint=None): config = app.state.config - api_list = app.state.api_list - api_index = api_list.index(token) - request_model = request.model if not safe_get(config, 'api_keys', api_index, 'model'): raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") @@ -988,7 +988,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques index += 1 provider = matching_providers[current_index] try: - response = await process_request(request, provider, endpoint, token) + response = await process_request(request, provider, endpoint) return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: @@ -1058,9 +1058,12 @@ async def rate_limit_dependency(request: Request, credentials: HTTPAuthorization try: api_index = api_list.index(token) except ValueError: - print("error: Invalid or missing API Key:", token) - api_index = None - token = None + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + if api_index is None: + print("error: Invalid or missing API Key:", token) + api_index = None + token = None # 使用 IP 地址和 token（如果有）作为限制键 client_ip = request.client.host @@ -1073,32 +1076,44 @@ async def rate_limit_dependency(request: Request, credentials: HTTPAuthorization def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): api_list = app.state.api_list token = credentials.credentials - if token not in api_list: + api_index = None + try: + api_index = api_list.index(token) + except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + if api_index is None: raise HTTPException(status_code=403, detail="Invalid or missing API Key") - return token + return api_index def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): api_list = app.state.api_list token = credentials.credentials - if token not in api_list: + api_index = None + try: + api_index = api_list.index(token) + except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + if api_index is None: raise HTTPException(status_code=403, detail="Invalid or missing API Key") - for api_key in app.state.api_keys_db: - if api_key['api'] == token: - if api_key.get('role') != "admin": - raise HTTPException(status_code=403, detail="Permission denied") + # for api_key in app.state.api_keys_db: + # if token.startswith(api_key['api']): + if app.state.api_keys_db[api_index].get('role') != "admin": + raise HTTPException(status_code=403, detail="Permission denied") return token @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) -async def request_model(request: RequestModel, token: str = Depends(verify_api_key)): - return await model_handler.request_model(request, token) +async def request_model(request: RequestModel, api_index: int = Depends(verify_api_key)): + return await model_handler.request_model(request, api_index) @app.options("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) async def options_handler(): return JSONResponse(status_code=200, content={"detail": "OPTIONS allowed"}) @app.get("/v1/models", dependencies=[Depends(rate_limit_dependency)]) -async def list_models(token: str = Depends(verify_api_key)): - models = post_all_models(token, app.state.config, app.state.api_list) +async def list_models(api_index: int = Depends(verify_api_key)): + models = post_all_models(api_index, app.state.config) return JSONResponse(content={ "object": "list", "data": models @@ -1107,23 +1122,23 @@ async def list_models(token: str = Depends(verify_api_key)): @app.post("/v1/images/generations", dependencies=[Depends(rate_limit_dependency)]) async def images_generations( request: ImageGenerationRequest, - token: str = Depends(verify_api_key) + api_index: int = Depends(verify_api_key) ): - return await model_handler.request_model(request, token, endpoint="/v1/images/generations") + return await model_handler.request_model(request, api_index, endpoint="/v1/images/generations") @app.post("/v1/embeddings", dependencies=[Depends(rate_limit_dependency)]) async def embeddings( request: EmbeddingRequest, - token: str = Depends(verify_api_key) + api_index: int = Depends(verify_api_key) ): - return await model_handler.request_model(request, token, endpoint="/v1/embeddings") + return await model_handler.request_model(request, api_index, endpoint="/v1/embeddings") @app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) async def moderations( request: ModerationRequest, - token: str = Depends(verify_api_key) + api_index: int = Depends(verify_api_key) ): - return await model_handler.request_model(request, token, endpoint="/v1/moderations") + return await model_handler.request_model(request, api_index, endpoint="/v1/moderations") from fastapi import UploadFile, File, Form, HTTPException import io @@ -1131,7 +1146,7 @@ async def moderations( async def audio_transcriptions( file: UploadFile = File(...), model: str = Form(...), - token: str = Depends(verify_api_key) + api_index: int = Depends(verify_api_key) ): try: # 读取上传的文件内容 @@ -1144,7 +1159,7 @@ async def audio_transcriptions( model=model ) - return await model_handler.request_model(request, token, endpoint="/v1/audio/transcriptions") + return await model_handler.request_model(request, api_index, endpoint="/v1/audio/transcriptions") except UnicodeDecodeError: raise HTTPException(status_code=400, detail="Invalid audio file encoding") except Exception as e: diff --git a/utils.py b/utils.py index a9875219..855a1dab 100644 --- a/utils.py +++ b/utils.py @@ -63,7 +63,7 @@ async def is_rate_limited(self, key: str, limits) -> bool: rate_limiter = InMemoryRateLimiter() -async def get_user_rate_limit(app, api_index: str = None): +async def get_user_rate_limit(app, api_index: int = None): # 这里应该实现根据 token 获取用户速率限制的逻辑 # 示例： 返回 (次数， 秒数) config = app.state.config @@ -457,13 +457,10 @@ async def new_generator(): except StopAsyncIteration: raise HTTPException(status_code=400, detail="data: {'error': 'No data returned'}") -def post_all_models(token, config, api_list): +def post_all_models(api_index, config): all_models = [] unique_models = set() - if token not in api_list: - raise HTTPException(status_code=403, detail="Invalid or missing API Key") - api_index = api_list.index(token) if config['api_keys'][api_index]['model']: for model in config['api_keys'][api_index]['model']: if model == "all": From 4ed5703d846adf2d4ef8ee4ae018046895754951 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 14:49:50 +0000 Subject: [PATCH 253/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2c2b6571..c31d8e91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.82 +0.0.83 From ebc2add9924a594626c3cb68a843b3beed4e86c2 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 7 Nov 2024 23:17:06 +0800 Subject: [PATCH 254/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20embedding=20model=20cannot=20receive=20arrays.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index c8d17777..cd1672aa 100644 --- a/models.py +++ b/models.py @@ -112,7 +112,7 @@ class ImageGenerationRequest(BaseRequest): stream: bool = False class EmbeddingRequest(BaseRequest): - input: str + input: Union[str, List[Union[str, int, List[int]]]] # 支持字符串或数组 model: str encoding_format: Optional[str] = "float" stream: bool = False From fc9790ba6754902abd93e58a3ab1eee14b7bcec1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 15:17:26 +0000 Subject: [PATCH 255/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c31d8e91..16113fa0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.83 +0.0.84 From 08f4342b9c9bc3192b0c3f9a171369f187e52e09 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 02:41:43 +0800 Subject: [PATCH 256/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Optimize=20log?= =?UTF-8?q?=20display:=20add=20error=20provider=20name=20display.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 +++++++------ utils.py | 14 +++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index 91f0c6c3..45651f33 100644 --- a/main.py +++ b/main.py @@ -706,7 +706,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if provider.get("engine"): engine = provider["engine"] - logger.info(f"provider: {provider['provider']:<11} model: {request.model:<22} engine: {engine}") + channel_id = f"{provider['provider']}" + logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine}") url, headers, payload = await get_payload(request, engine, provider) if is_debug: @@ -738,11 +739,11 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A async with app.state.client_manager.get_client(timeout_value) as client: if request.stream: generator = fetch_response_stream(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: generator = fetch_response(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id) first_element = await anext(wrapped_generator) first_element = first_element.lstrip("data: ") # print("first_element", first_element) @@ -751,14 +752,14 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A # response = JSONResponse(first_element) # 更新成功计数和首次响应时间 - await update_channel_stats(current_info["request_id"], provider['provider'], request.model, current_info["api_key"], success=True) + await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=True) current_info["first_response_time"] = first_response_time current_info["success"] = True - current_info["provider"] = provider['provider'] + current_info["provider"] = channel_id return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: - await update_channel_stats(current_info["request_id"], provider['provider'], request.model, current_info["api_key"], success=False) + await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=False) raise e def weighted_round_robin(weights): diff --git a/utils.py b/utils.py index 855a1dab..ddbe887d 100644 --- a/utils.py +++ b/utils.py @@ -405,7 +405,7 @@ def ensure_string(item): import asyncio import time as time_module -async def error_handling_wrapper(generator): +async def error_handling_wrapper(generator, channel_id): start_time = time_module.time() try: first_item = await generator.__anext__() @@ -418,18 +418,18 @@ async def error_handling_wrapper(generator): if first_item_str.startswith("data:"): first_item_str = first_item_str.lstrip("data: ") if first_item_str.startswith("[DONE]"): - logger.error("error_handling_wrapper [DONE]!") + logger.error(f"provider: {channel_id:<11} error_handling_wrapper [DONE]!") raise StopAsyncIteration if "The bot's usage is covered by the developer" in first_item_str: - logger.error("error const string: %s", first_item_str) + logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) raise StopAsyncIteration if "process this request due to overload or policy" in first_item_str: - logger.error("error const string: %s", first_item_str) + logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) raise StopAsyncIteration try: first_item_str = json.loads(first_item_str) except json.JSONDecodeError: - logger.error("error_handling_wrapper JSONDecodeError!" + repr(first_item_str)) + logger.error(f"provider: {channel_id:<11} error_handling_wrapper JSONDecodeError! {repr(first_item_str)}") raise StopAsyncIteration if isinstance(first_item_str, dict) and 'error' in first_item_str: # 如果第一个 yield 的项是错误信息，抛出 HTTPException @@ -445,11 +445,11 @@ async def new_generator(): yield ensure_string(item) except asyncio.CancelledError: # 客户端断开连接是正常行为，不需要记录错误日志 - logger.debug("Stream cancelled by client") + logger.debug(f"provider: {channel_id:<11} Stream cancelled by client") return except (httpx.ReadError, httpx.RemoteProtocolError) as e: # 只记录真正的网络错误 - logger.error(f"Network error in new_generator: {e}") + logger.error(f"provider: {channel_id:<11} Network error in new_generator: {e}") raise return new_generator(), first_response_time From a0d13b0f5723bfdaaddbc258c2c9ada786bf0d39 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 18:42:18 +0000 Subject: [PATCH 257/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 16113fa0..d920f649 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.84 +0.0.85 From d2272e86d3e3f900edb3b56b70d7da6f4754d2d1 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 03:39:52 +0800 Subject: [PATCH 258/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20undefined=20review=20content.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 12 +++++++++++- models.py | 10 ++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 45651f33..2cd32aad 100644 --- a/main.py +++ b/main.py @@ -470,14 +470,24 @@ async def dispatch(self, request: Request, call_next): model = request_model.model current_info["model"] = model + moderated_content = None if request_model.request_type == "chat": moderated_content = request_model.get_last_text_message() elif request_model.request_type == "image": moderated_content = request_model.prompt + elif request_model.request_type == "moderation": + pass + elif request_model.request_type == "embedding": + if isinstance(request_model.input, list) and len(request_model.input) > 0 and isinstance(request_model.input[0], str): + moderated_content = "\n".join(request_model.input) + else: + moderated_content = request_model.input + else: + logger.error(f"Unknown request type: {request_model.request_type}") + if moderated_content: current_info["text"] = moderated_content - if enable_moderation and moderated_content: moderation_response = await self.moderate_content(moderated_content, api_index) is_flagged = moderation_response.get('results', [{}])[0].get('flagged', False) diff --git a/models.py b/models.py index cd1672aa..e4b00f75 100644 --- a/models.py +++ b/models.py @@ -115,6 +115,8 @@ class EmbeddingRequest(BaseRequest): input: Union[str, List[Union[str, int, List[int]]]] # 支持字符串或数组 model: str encoding_format: Optional[str] = "float" + dimensions: Optional[int] = None + user: Optional[str] = None stream: bool = False class AudioTranscriptionRequest(BaseRequest): @@ -130,7 +132,7 @@ class Config: arbitrary_types_allowed = True class ModerationRequest(BaseRequest): - input: str + input: Union[str, List[str]] model: Optional[str] = "text-moderation-latest" stream: bool = False @@ -150,12 +152,12 @@ def set_request_type(cls, values): elif "file" in values: values["data"] = AudioTranscriptionRequest(**values) values["data"].request_type = "audio" + elif "text-embedding" in values.get("model", ""): + values["data"] = EmbeddingRequest(**values) + values["data"].request_type = "embedding" elif "input" in values: values["data"] = ModerationRequest(**values) values["data"].request_type = "moderation" - elif "input" in values: - values["data"] = EmbeddingRequest(**values) - values["data"].request_type = "embedding" else: raise ValueError("无法确定请求类型") return values \ No newline at end of file From eeaa4ee045311f7ad10a3a00a9ddd845c3a241c3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 19:40:28 +0000 Subject: [PATCH 259/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d920f649..0222dbeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.85 +0.0.86 From 926469de4a410baca6551d254061dc6a9790551d Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 04:08:55 +0800 Subject: [PATCH 260/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20support=20setting=20API=20key=20random=20load=20balancing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 17 +++++++++-------- README_CN.md | 9 +++++---- utils.py | 18 ++++++++++++++---- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 96dda828..d2517157 100644 --- a/README.md +++ b/README.md @@ -13,20 +13,20 @@ ## Introduction -For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, DeepBricks, OpenRouter, and more. +For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, OpenRouter, and more. ## ✨ Features - No front-end, pure configuration file to configure API channels. You can run your own API station just by writing a file, and the documentation has a detailed configuration guide, beginner-friendly. -- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. +- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. - Simultaneously supports Anthropic, Gemini, Vertex AI, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. - Support OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Support OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Support four types of load balancing. -1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. -2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. -3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. -4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. + 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. + 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. + 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. - Support automatic retry, when an API channel response fails, automatically retry the next API channel. - Support channel cooling: When an API channel response fails, the channel will automatically be excluded and cooled for a period of time, and requests to the channel will be stopped. After the cooling period ends, the model will automatically be restored until it fails again, at which point it will be cooled again. - Support fine-grained model timeout settings, allowing different timeout durations for each model. @@ -48,7 +48,7 @@ You must fill in the configuration file in advance to start `uni-api`, and you m ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required, automatically uses base_url and api to get all available models through the /v1/models endpoint. # Multiple providers can be configured here, each provider can configure multiple API Keys, and each API Key can configure multiple models. @@ -61,7 +61,7 @@ Detailed advanced configuration of `api.yaml`: ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, can be any name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required model: # Optional, if model is not configured, all available models will be automatically obtained through base_url and api via the /v1/models endpoint. @@ -96,6 +96,7 @@ providers: # gemini-1.5-flash: 2/min # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. + api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. diff --git a/README_CN.md b/README_CN.md index 826ba749..d21e67fa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,12 +13,12 @@ ## 介绍 -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Groq、Cloudflare、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Groq、Cloudflare、OpenRouter 等。 ## ✨ 特性 - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 -- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 - 同时支持 Anthropic、Gemini、Vertex AI、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 @@ -48,7 +48,7 @@ ```yaml providers: - - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter，随便取名字，必填 base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填，自动使用 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个 API Key 可以配置多个模型。 @@ -61,7 +61,7 @@ api_keys: ```yaml providers: - - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter，随便取名字，必填 base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 model: # 选填，如果不配置 model，会自动通过 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 @@ -96,6 +96,7 @@ providers: # gemini-1.5-flash: 2/min # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 + api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡。 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 diff --git a/utils.py b/utils.py index ddbe887d..c3048793 100644 --- a/utils.py +++ b/utils.py @@ -80,8 +80,16 @@ async def get_user_rate_limit(app, api_index: int = None): import asyncio class ThreadSafeCircularList: - def __init__(self, items = [], rate_limit={"default": "999999/min"}): - self.items = items + def __init__(self, items = [], rate_limit={"default": "999999/min"}, schedule_algorithm="round_robin"): + if schedule_algorithm == "random": + import random + self.items = random.sample(items, len(items)) + elif schedule_algorithm == "round_robin": + self.items = items + else: + self.items = items + logger.warning(f"Unknown schedule algorithm: {schedule_algorithm}, use (round_robin, random) instead") + self.index = 0 self.lock = asyncio.Lock() # 修改为二级字典，第一级是item，第二级是model @@ -260,12 +268,14 @@ def update_config(config_data, use_config_url=False): if isinstance(provider_api, str): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( [provider_api], - safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}) + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}), + safe_get(provider, "preferences", "api_key_schedule_algorithm", default="round_robin") ) if isinstance(provider_api, list): provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( provider_api, - safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}) + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}), + safe_get(provider, "preferences", "api_key_schedule_algorithm", default="round_robin") ) if "models.inference.ai.azure.com" in provider['base_url'] and not provider.get("model"): From 7775638ce1d8b2c0b77b870e9901de212ff56111 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 20:09:15 +0000 Subject: [PATCH 261/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0222dbeb..58a1699d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.86 +0.0.87 From 0a213f9397153fadc6399d3315d819301d8f5417 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 05:21:06 +0800 Subject: [PATCH 262/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20setting=20model=20timeout=20at=20the=20channel=20l?= =?UTF-8?q?evel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++++++ README_CN.md | 8 ++++++++ main.py | 52 +++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d2517157..abf56752 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,10 @@ providers: # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. + model_timeout: # Model timeout, in seconds, default 100 seconds, optional + gemini-1.5-pro: 10 # Model gemini-1.5-pro timeout is 10 seconds + gemini-1.5-flash: 10 # Model gemini-1.5-flash timeout is 10 seconds + default: 10 # Model does not have a timeout set, use the default timeout of 10 seconds, when requesting a model not in model_timeout, the timeout is also 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, the default timeout is 100 seconds - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. @@ -389,6 +393,10 @@ All scheduling algorithms need to be enabled by setting api_keys.(api).preferenc Except for some special channels shown in the advanced configuration, all OpenAI format providers need to fill in the base_url completely, which means the base_url must end with /v1/chat/completions. If you are using GitHub models, the base_url should be filled in as https://models.inference.ai.azure.com/chat/completion, not Azure's URL. +- How does the model timeout time work? What is the priority of the channel-level timeout setting and the global model timeout setting? + +The channel-level timeout setting has higher priority than the global model timeout setting. The priority order is: channel-level model timeout setting > channel-level default timeout setting > global model timeout setting > global default timeout setting > environment variable TIMEOUT. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index d21e67fa..1f36b5d8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -97,6 +97,10 @@ providers: # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡。 + model_timeout: # 模型超时时间，单位为秒，默认 100 秒，选填 + gemini-1.5-pro: 10 # 模型 gemini-1.5-pro 的超时时间为 10 秒 + gemini-1.5-flash: 10 # 模型 gemini-1.5-flash 的超时时间为 10 秒 + default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用全局配置的模型超时时间。 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 @@ -389,6 +393,10 @@ api_keys: 除了高级配置里面所展示的一些特殊的渠道，所有 OpenAI 格式的提供商需要把 base_url 填完整，也就是说 base_url 必须以 /v1/chat/completions 结尾。如果你使用的 GitHub models，base_url 应该填写为 https://models.inference.ai.azure.com/chat/completion，而不是 Azure 的 URL。 +- 模型超时时间是如何确认的？渠道级别的超时设置和全局模型超时设置的优先级是什么？ + +渠道级别的超时设置优先级高于全局模型超时设置。优先级顺序：渠道级别模型超时设置 > 渠道级别默认超时设置 > 全局模型超时设置 > 全局默认超时设置 > 环境变量 TIMEOUT。 + ## ⭐ Star 历史 diff --git a/main.py b/main.py index 2cd32aad..77a0cd06 100644 --- a/main.py +++ b/main.py @@ -39,7 +39,7 @@ import string import json -DEFAULT_TIMEOUT = float(os.getenv("TIMEOUT", 100)) +DEFAULT_TIMEOUT = int(os.getenv("TIMEOUT", 100)) is_debug = bool(os.getenv("DEBUG", False)) # is_debug = False @@ -643,7 +643,22 @@ async def ensure_config(request: Request, call_next): if "default" not in app.state.config['preferences'].get('model_timeout', {}): app.state.timeouts["default"] = DEFAULT_TIMEOUT - # print("app.state.timeouts", app.state.timeouts) + app.state.provider_timeouts = defaultdict(lambda: defaultdict(lambda: DEFAULT_TIMEOUT)) + for provider in app.state.config["providers"]: + # print("provider", provider) + provider_timeout_settings = safe_get(provider, "preferences", "model_timeout", default={}) + # print("provider_timeout_settings", provider_timeout_settings) + if provider_timeout_settings: + for model_name, timeout_value in provider_timeout_settings.items(): + app.state.provider_timeouts[provider['provider']][model_name] = timeout_value + + # app.state.provider_timeouts["global_time_out"] = app.state.timeouts + # provider_timeouts_dict = { + # provider: dict(timeouts) + # for provider, timeouts in app.state.provider_timeouts.items() + # } + # print("app.state.provider_timeouts", provider_timeouts_dict) + # print("ai" in app.state.provider_timeouts) if app and not hasattr(app.state, "channel_manager"): if app.state.config and 'preferences' in app.state.config: @@ -655,6 +670,21 @@ async def ensure_config(request: Request, call_next): return await call_next(request) +def get_timeout_value(provider_timeouts, original_model): + timeout_value = None + if original_model in provider_timeouts: + timeout_value = provider_timeouts[original_model] + else: + # 尝试模糊匹配模型 + for timeout_model in provider_timeouts: + if timeout_model != "default" and timeout_model in original_model: + timeout_value = provider_timeouts[timeout_model] + break + else: + # 如果模糊匹配失败，使用渠道的默认值 + timeout_value = provider_timeouts.get("default") + return timeout_value + # 在 process_request 函数中更新成功和失败计数 async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None): url = provider['base_url'] @@ -729,21 +759,13 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A current_info = request_info.get() - timeout_value = None - # 先尝试精确匹配 - - if original_model in app.state.timeouts: - timeout_value = app.state.timeouts[original_model] - else: - # 如果没有精确匹配，尝试模糊匹配 - for timeout_model in app.state.timeouts: - if timeout_model in original_model: - timeout_value = app.state.timeouts[timeout_model] - break - - # 如果都没匹配到，使用默认值 + provider_timeouts = safe_get(app.state.provider_timeouts, channel_id, default=app.state.provider_timeouts["global_time_out"]) + timeout_value = get_timeout_value(provider_timeouts, original_model) + if timeout_value is None: + timeout_value = get_timeout_value(app.state.provider_timeouts["global_time_out"], original_model) if timeout_value is None: timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) + print("timeout_value", timeout_value) try: async with app.state.client_manager.get_client(timeout_value) as client: From 825d319fde42b4ab6dbbd9a474a26dcd957344e1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 21:21:28 +0000 Subject: [PATCH 263/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58a1699d..fe520fe5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.87 +0.0.88 From 97c356d931676378c2b6cb61dcbd11d3748403b8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 05:29:40 +0800 Subject: [PATCH 264/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ README_CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index abf56752..246e7e11 100644 --- a/README.md +++ b/README.md @@ -397,6 +397,8 @@ Except for some special channels shown in the advanced configuration, all OpenAI The channel-level timeout setting has higher priority than the global model timeout setting. The priority order is: channel-level model timeout setting > channel-level default timeout setting > global model timeout setting > global default timeout setting > environment variable TIMEOUT. +By adjusting the model timeout time, you can avoid the error of some channels timing out. If you encounter the error `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}`, please try to increase the model timeout time. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index 1f36b5d8..238836da 100644 --- a/README_CN.md +++ b/README_CN.md @@ -397,6 +397,8 @@ api_keys: 渠道级别的超时设置优先级高于全局模型超时设置。优先级顺序：渠道级别模型超时设置 > 渠道级别默认超时设置 > 全局模型超时设置 > 全局默认超时设置 > 环境变量 TIMEOUT。 +通过调整模型超时时间，可以避免出现某些渠道请求超时报错的情况。如果你遇到 `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}` 错误，请尝试增加模型超时时间。 + ## ⭐ Star 历史 From 3d5dc7e6d50bea293eed20fee1d432308fc7fd7c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 8 Nov 2024 21:40:42 +0800 Subject: [PATCH 265/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20201=20status=20code=20is=20mistakenly=20consid?= =?UTF-8?q?ered=20an=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- response.py | 2 +- test/provider_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 77a0cd06..24aaee40 100644 --- a/main.py +++ b/main.py @@ -765,7 +765,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A timeout_value = get_timeout_value(app.state.provider_timeouts["global_time_out"], original_model) if timeout_value is None: timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) - print("timeout_value", timeout_value) + # print("timeout_value", timeout_value) try: async with app.state.client_manager.get_client(timeout_value) as client: diff --git a/response.py b/response.py index 41c2b1a1..a10ffcc4 100644 --- a/response.py +++ b/response.py @@ -77,7 +77,7 @@ async def generate_no_stream_response(timestamp, model, content=None, tools_id=N return json_data async def check_response(response, error_log): - if response and response.status_code != 200: + if response and not (200 <= response.status_code < 300): error_message = await response.aread() error_str = error_message.decode('utf-8', errors='replace') try: diff --git a/test/provider_test.py b/test/provider_test.py index fc00d88b..b1014e72 100644 --- a/test/provider_test.py +++ b/test/provider_test.py @@ -82,7 +82,7 @@ def test_request_model(test_client, api_key, get_model): response = test_client.post("/v1/chat/completions", json=request_data, headers=headers) for line in response.iter_lines(): print(line.lstrip("data: ")) - assert response.status_code == 200 + assert 200 <= response.status_code < 300 if __name__ == "__main__": pytest.main(["-s", "test/provider_test.py"]) \ No newline at end of file From d6768b9cf219386d1c8b929421a9251d40f65b17 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 8 Nov 2024 13:41:09 +0000 Subject: [PATCH 266/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fe520fe5..a82a0477 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.88 +0.0.89 From f67c99bea57d362f39f826596f0c125a71cd1c21 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 9 Nov 2024 21:37:57 +0800 Subject: [PATCH 267/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 21 +++++++++++++++++++++ README_CN.md | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/README.md b/README.md index 246e7e11..5fcc961b 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,27 @@ The channel-level timeout setting has higher priority than the global model time By adjusting the model timeout time, you can avoid the error of some channels timing out. If you encounter the error `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}`, please try to increase the model timeout time. +- How does api_key_rate_limit work? How do I set the same rate limit for multiple models? + +If you want to set the same frequency limit for the four models gemini-1.5-pro-latest, gemini-1.5-pro, gemini-1.5-pro-001, gemini-1.5-pro-002 simultaneously, you can set it like this: + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min +``` + +This will match all models containing the gemini-1.5-pro string. The frequency limit for these four models, gemini-1.5-pro-latest, gemini-1.5-pro, gemini-1.5-pro-001, gemini-1.5-pro-002, will all be set to 1000/min. The logic for configuring the api_key_rate_limit field is as follows, here is a sample configuration file: + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min + gemini-1.5-pro-002: 500/min +``` + +At this time, if there is a request using the model gemini-1.5-pro-002. + +First, the uni-api will attempt to precisely match the model in the api_key_rate_limit. If the rate limit for gemini-1.5-pro-002 is set, then the rate limit for gemini-1.5-pro-002 is 500/min. If the requested model at this time is not gemini-1.5-pro-002, but gemini-1.5-pro-latest, since the api_key_rate_limit does not have a rate limit set for gemini-1.5-pro-latest, it will look for any model with the same prefix as gemini-1.5-pro-latest that has been set, thus the rate limit for gemini-1.5-pro-latest will be set to 1000/min. + ## ⭐ Star History diff --git a/README_CN.md b/README_CN.md index 238836da..8dbaed05 100644 --- a/README_CN.md +++ b/README_CN.md @@ -399,6 +399,27 @@ api_keys: 通过调整模型超时时间，可以避免出现某些渠道请求超时报错的情况。如果你遇到 `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}` 错误，请尝试增加模型超时时间。 +- api_key_rate_limit 是怎么工作的？我如何给多个模型设置相同的频率限制？ + +如果你想同时给 gemini-1.5-pro-latest，gemini-1.5-pro，gemini-1.5-pro-001，gemini-1.5-pro-002 这四个模型设置相同的频率限制，可以这样设置： + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min +``` + +这会匹配所有含有 gemini-1.5-pro 字符串的模型。gemini-1.5-pro-latest，gemini-1.5-pro，gemini-1.5-pro-001，gemini-1.5-pro-002 这四个模型频率限制都会设置为 1000/min。api_key_rate_limit 字段配置的逻辑如下，这是一个示例配置文件： + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min + gemini-1.5-pro-002: 500/min +``` + +此时如果有一个使用模型 gemini-1.5-pro-002 的请求。 + +首先，uni-api 会尝试精确匹配 api_key_rate_limit 的模型。如果刚好设置了 gemini-1.5-pro-002 的频率限制，则 gemini-1.5-pro-002 的频率限制则为 500/min，如果此时请求的模型不是 gemini-1.5-pro-002，而是 gemini-1.5-pro-latest，由于 api_key_rate_limit 没有设置 gemini-1.5-pro-latest 的频率限制，因此会寻找有没有前缀和 gemini-1.5-pro-latest 相同的模型被设置了，因此 gemini-1.5-pro-latest 的频率限制会被设置为 1000/min。 + ## ⭐ Star 历史 From 4fe51b7d39622405296a2ddb125061397d64ab67 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 10 Nov 2024 06:30:18 +0800 Subject: [PATCH 268/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20global=20timeout=20configuration=20does=20not?= =?UTF-8?q?=20take=20effect.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 24aaee40..47cdc0b2 100644 --- a/main.py +++ b/main.py @@ -652,7 +652,8 @@ async def ensure_config(request: Request, call_next): for model_name, timeout_value in provider_timeout_settings.items(): app.state.provider_timeouts[provider['provider']][model_name] = timeout_value - # app.state.provider_timeouts["global_time_out"] = app.state.timeouts + app.state.provider_timeouts["global_time_out"] = app.state.timeouts + # provider_timeouts_dict = { # provider: dict(timeouts) # for provider, timeouts in app.state.provider_timeouts.items() From 974f096fd299811da2588c5cdd0b1e05674c6a66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 Nov 2024 22:30:38 +0000 Subject: [PATCH 269/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a82a0477..4eaed40c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.89 +0.0.90 From 28f36b62edfb8cec089e80e9b78a12c6dbc4aaa3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 12 Nov 2024 18:26:02 +0800 Subject: [PATCH 270/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20vertex=20API=20key=20concatenation=20syntax=20err?= =?UTF-8?q?or.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index 623a3571..c90d7568 100644 --- a/request.py +++ b/request.py @@ -316,7 +316,7 @@ async def get_vertex_gemini_payload(request, engine, provider): model_dict = get_model_dict(provider) model = model_dict[request.model] location = gem - url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) + url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) messages = [] systemInstruction = None From 74089d78fd1bcf68e69bf12d6a0b70828c745eca Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 Nov 2024 10:26:38 +0000 Subject: [PATCH 271/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4eaed40c..32e0d574 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.90 +0.0.91 From 4e57426cea2c73368d9f0de8e3e2fd360593c1a9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 12 Nov 2024 22:03:17 +0800 Subject: [PATCH 272/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20response=20ID=20is=20the=20same=20every=20time?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/response.py b/response.py index a10ffcc4..1ee3ad66 100644 --- a/response.py +++ b/response.py @@ -1,5 +1,7 @@ import json import httpx +import random +import string from datetime import datetime from log_config import logger @@ -14,8 +16,10 @@ # end_of_line = "\n" async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) sample_data = { - "id": "chatcmpl-9ijPeRHa0wtyA2G8wq5z8FC3wGMzc", + "id": f"chatcmpl-{random_str}", "object": "chat.completion.chunk", "created": timestamp, "model": model, @@ -49,8 +53,10 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f return sse_response async def generate_no_stream_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) sample_data = { - "id": "chatcmpl-ALGS9hpJBb8xVAe62DRriY2SpoT4L", + "id": f"chatcmpl-{random_str}", "object": "chat.completion", "created": timestamp, "model": model, From 0f49d25327cc73c5e28785b3003d29794ea4fa70 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 Nov 2024 14:03:48 +0000 Subject: [PATCH 273/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 32e0d574..95391832 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.91 +0.0.92 From 625766611a612562134ddff23dffa86c1b2f12de Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 12 Nov 2024 22:13:29 +0800 Subject: [PATCH 274/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20response=20ID=20is=20the=20same=20every=20time?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/response.py b/response.py index 1ee3ad66..282ec2c2 100644 --- a/response.py +++ b/response.py @@ -186,6 +186,9 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod yield "data: [DONE]" + end_of_line async def fetch_gpt_response_stream(client, url, headers, payload): + timestamp = int(datetime.timestamp(datetime.now())) + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_gpt_response_stream") if error_message: @@ -199,7 +202,9 @@ async def fetch_gpt_response_stream(client, url, headers, payload): line, buffer = buffer.split("\n", 1) # logger.info("line: %s", repr(line)) if line and line != "data: " and line != "data:" and not line.startswith(": "): - yield line.strip() + end_of_line + line = json.loads(line.lstrip("data: ")) + line['id'] = f"chatcmpl-{random_str}" + yield "data: " + json.dumps(line).strip() + end_of_line async def fetch_cloudflare_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) From 2bfe3df64bc6de4f8875e7c4314136a8d026ad02 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 Nov 2024 14:13:50 +0000 Subject: [PATCH 275/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 95391832..ef25911d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.92 +0.0.93 From 2f00898716bd30cc1642a4b4ccba4d16de4e26bf Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 13 Nov 2024 07:22:42 +0800 Subject: [PATCH 276/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20bugs=20caus?= =?UTF-8?q?ed=20by=20concurrent=20errors=20in=20multiple=20database=20writ?= =?UTF-8?q?e=20operations.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 81 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/main.py b/main.py index 47cdc0b2..0e757df6 100644 --- a/main.py +++ b/main.py @@ -282,40 +282,67 @@ def calculate_cost(model: str, input_tokens: int, output_tokens: int) -> Decimal # 返回精确到15位小数的结果 return total_cost.quantize(Decimal('0.000000000000001')) +from asyncio import Semaphore + +# 创建一个信号量来控制数据库访问 +db_semaphore = Semaphore(1) # 限制同时只有1个写入操作 + async def update_stats(current_info): if DISABLE_DATABASE: return - # 这里添加更新数据库的逻辑 - async with async_session() as session: - async with session.begin(): - try: - columns = [column.key for column in RequestStat.__table__.columns] - filtered_info = {k: v for k, v in current_info.items() if k in columns} - new_request_stat = RequestStat(**filtered_info) - session.add(new_request_stat) - await session.commit() - except Exception as e: - await session.rollback() - logger.error(f"Error updating stats: {str(e)}") + + try: + # 等待获取数据库访问权限 + async with db_semaphore: + async with async_session() as session: + async with session.begin(): + try: + columns = [column.key for column in RequestStat.__table__.columns] + filtered_info = {k: v for k, v in current_info.items() if k in columns} + new_request_stat = RequestStat(**filtered_info) + session.add(new_request_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating stats: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + except Exception as e: + logger.error(f"Error acquiring database lock: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() async def update_channel_stats(request_id, provider, model, api_key, success): if DISABLE_DATABASE: return - async with async_session() as session: - async with session.begin(): - try: - channel_stat = ChannelStat( - request_id=request_id, - provider=provider, - model=model, - api_key=api_key, - success=success, - ) - session.add(channel_stat) - await session.commit() - except Exception as e: - await session.rollback() - logger.error(f"Error updating channel stats: {str(e)}") + + try: + async with db_semaphore: + async with async_session() as session: + async with session.begin(): + try: + channel_stat = ChannelStat( + request_id=request_id, + provider=provider, + model=model, + api_key=api_key, + success=success, + ) + session.add(channel_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating channel stats: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + except Exception as e: + logger.error(f"Error acquiring database lock: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() class LoggingStreamingResponse(Response): def __init__(self, content, status_code=200, headers=None, media_type=None, current_info=None): From d0860ecde53ea8734988f97b38ce9fe455b1fb67 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 Nov 2024 23:23:04 +0000 Subject: [PATCH 277/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ef25911d..4cdce1d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.93 +0.0.94 From 1130ba950c7e7f74658bfcb598356f218479cb09 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 13 Nov 2024 22:02:07 +0800 Subject: [PATCH 278/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Remove=20redundan?= =?UTF-8?q?t=20error-catching=20code=20snippets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💰 Sponsors: Thanks to @PowerHunter for the ¥1400 sponsorship, sponsorship information has been added to the README. --- README.md | 2 +- README_CN.md | 2 +- main.py | 7 +++++-- response.py | 50 +++++++++++++++++++++++++------------------------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5fcc961b..d254c598 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ We thank the following sponsors for their support: -- @PowerHunter: ¥1000 +- @PowerHunter: ¥1400 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index 8dbaed05..6be6d984 100644 --- a/README_CN.md +++ b/README_CN.md @@ -332,7 +332,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ 我们感谢以下赞助商的支持： -- @PowerHunter：¥1000 +- @PowerHunter：¥1400 ## 如何赞助我们 diff --git a/main.py b/main.py index 0e757df6..2043802e 100644 --- a/main.py +++ b/main.py @@ -818,7 +818,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A current_info["provider"] = channel_id return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=False) raise e @@ -1051,12 +1051,15 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques try: response = await process_request(request, provider, endpoint) return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout) as e: + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: # 根据异常类型设置状态码和错误消息 if isinstance(e, httpx.ReadTimeout): status_code = 504 # Gateway Timeout error_message = "Request timed out" + elif isinstance(e, httpx.ConnectError): + status_code = 503 # Service Unavailable + error_message = "Unable to connect to service" elif isinstance(e, httpx.ReadError): status_code = 502 # Bad Gateway error_message = "Network read error" diff --git a/response.py b/response.py index 282ec2c2..f760f493 100644 --- a/response.py +++ b/response.py @@ -364,28 +364,28 @@ async def fetch_response(client, url, headers, payload, engine, model): yield response_json async def fetch_response_stream(client, url, headers, payload, engine, model): - try: - if engine == "gemini" or engine == "vertex-gemini": - async for chunk in fetch_gemini_response_stream(client, url, headers, payload, model): - yield chunk - elif engine == "claude" or engine == "vertex-claude": - async for chunk in fetch_claude_response_stream(client, url, headers, payload, model): - yield chunk - elif engine == "gpt": - async for chunk in fetch_gpt_response_stream(client, url, headers, payload): - yield chunk - elif engine == "openrouter": - async for chunk in fetch_gpt_response_stream(client, url, headers, payload): - yield chunk - elif engine == "cloudflare": - async for chunk in fetch_cloudflare_response_stream(client, url, headers, payload, model): - yield chunk - elif engine == "cohere": - async for chunk in fetch_cohere_response_stream(client, url, headers, payload, model): - yield chunk - else: - raise ValueError("Unknown response") - except httpx.ConnectError as e: - yield {"error": f"500", "details": "fetch_response_stream Connect Error"} - except httpx.ReadTimeout as e: - yield {"error": f"500", "details": "fetch_response_stream Read Response Timeout"} \ No newline at end of file + # try: + if engine == "gemini" or engine == "vertex-gemini": + async for chunk in fetch_gemini_response_stream(client, url, headers, payload, model): + yield chunk + elif engine == "claude" or engine == "vertex-claude": + async for chunk in fetch_claude_response_stream(client, url, headers, payload, model): + yield chunk + elif engine == "gpt": + async for chunk in fetch_gpt_response_stream(client, url, headers, payload): + yield chunk + elif engine == "openrouter": + async for chunk in fetch_gpt_response_stream(client, url, headers, payload): + yield chunk + elif engine == "cloudflare": + async for chunk in fetch_cloudflare_response_stream(client, url, headers, payload, model): + yield chunk + elif engine == "cohere": + async for chunk in fetch_cohere_response_stream(client, url, headers, payload, model): + yield chunk + else: + raise ValueError("Unknown response") + # except httpx.ConnectError as e: + # yield {"error": f"500", "details": "fetch_response_stream Connect Error"} + # except httpx.ReadTimeout as e: + # yield {"error": f"500", "details": "fetch_response_stream Read Response Timeout"} \ No newline at end of file From 09cfbba4a998715e6aca2cd2bc571f35c46ecb69 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 Nov 2024 14:02:37 +0000 Subject: [PATCH 279/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4cdce1d7..28087766 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.94 +0.0.95 From 98717385b1fa3c6475131dd9f61d80ac0cee91b3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 13 Nov 2024 22:42:06 +0800 Subject: [PATCH 280/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20socks5=20proxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_CN.md | 1 + main.py | 61 ++++++++++++++++++++++++++++++++++++++++++------ requirements.txt | 1 + 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d254c598..86c16cb2 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ providers: gemini-1.5-pro: 10 # Model gemini-1.5-pro timeout is 10 seconds gemini-1.5-flash: 10 # Model gemini-1.5-flash timeout is 10 seconds default: 10 # Model does not have a timeout set, use the default timeout of 10 seconds, when requesting a model not in model_timeout, the timeout is also 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, the default timeout is 100 seconds + proxy: socks5://[username]:[password]@[ip]:[port] # Proxy address, optional. Supports socks5 and http proxies, default is not used. - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. diff --git a/README_CN.md b/README_CN.md index 6be6d984..3373accb 100644 --- a/README_CN.md +++ b/README_CN.md @@ -101,6 +101,7 @@ providers: gemini-1.5-pro: 10 # 模型 gemini-1.5-pro 的超时时间为 10 秒 gemini-1.5-flash: 10 # 模型 gemini-1.5-flash 的超时时间为 10 秒 default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用全局配置的模型超时时间。 + proxy: socks5://[用户名]:[密码]@[IP地址]:[端口] # 代理地址，选填。支持 socks5 和 http 代理，默认不使用代理。 - provider: vertex project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 diff --git a/main.py b/main.py index 2043802e..17ad6dd6 100644 --- a/main.py +++ b/main.py @@ -600,7 +600,7 @@ async def init(self, default_config): self.default_config = default_config @asynccontextmanager - async def get_client(self, timeout_value): + async def get_client(self, timeout_value, proxy=None): # 直接获取或创建客户端,不使用锁 timeout_value = int(timeout_value) if timeout_value not in self.clients: @@ -610,11 +610,42 @@ async def get_client(self, timeout_value): write=30.0, pool=self.pool_size ) - self.clients[timeout_value] = httpx.AsyncClient( - timeout=timeout, - limits=httpx.Limits(max_connections=self.pool_size), - **self.default_config - ) + limits = httpx.Limits(max_connections=self.pool_size) + + client_config = { + **self.default_config, + "timeout": timeout, + "limits": limits + } + + if proxy: + # 解析代理URL + from urllib.parse import urlparse + parsed = urlparse(proxy) + + # 修改这里: 将 socks5h 转换为 socks5 + scheme = parsed.scheme.rstrip('h') + # print("scheme", scheme) + + if scheme == 'socks5': + try: + from httpx_socks import AsyncProxyTransport + # 使用修改后的scheme创建代理URL + proxy = proxy.replace('socks5h://', 'socks5://') + # 创建SOCKS5代理传输 + transport = AsyncProxyTransport.from_url(proxy) + client_config["transport"] = transport + except ImportError: + logger.error("httpx-socks package is required for SOCKS proxy support") + raise ImportError("Please install httpx-socks package for SOCKS proxy support: pip install httpx-socks") + else: + # 对于HTTP/HTTPS代理使用原有方式 + client_config["proxies"] = { + "http://": proxy, + "https://": proxy + } + + self.clients[timeout_value] = httpx.AsyncClient(**client_config) try: yield self.clients[timeout_value] @@ -795,8 +826,24 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) # print("timeout_value", timeout_value) + proxy = safe_get(provider, "preferences", "proxy", default=None) + # print("proxy", proxy) + try: - async with app.state.client_manager.get_client(timeout_value) as client: + async with app.state.client_manager.get_client(timeout_value, proxy) as client: + # 打印client配置信息 + # logger.info(f"Client config - Timeout: {client.timeout}") + # logger.info(f"Client config - Headers: {client.headers}") + # if hasattr(client, '_transport'): + # if hasattr(client._transport, 'proxy_url'): + # logger.info(f"Client config - Proxy: {client._transport.proxy_url}") + # elif hasattr(client._transport, 'proxies'): + # logger.info(f"Client config - Proxies: {client._transport.proxies}") + # else: + # logger.info("Client config - No proxy configured") + # else: + # logger.info("Client config - No transport configured") + # logger.info(f"Client config - Follow Redirects: {client.follow_redirects}") if request.stream: generator = fetch_response_stream(client, url, headers, payload, engine, original_model) wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id) diff --git a/requirements.txt b/requirements.txt index 8fbaf43e..6611b75b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,6 @@ sqlalchemy watchfiles ruamel.yaml httpx[http2] +httpx-socks cryptography python-multipart \ No newline at end of file From 58f793c6e8685a34ba033d85e21d4e391d2ef663 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 Nov 2024 14:42:30 +0000 Subject: [PATCH 281/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 28087766..a2dc0c7e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.95 +0.0.96 From ac2ca1748c24075efbf688d9dcdb761a0b15a4d3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 14 Nov 2024 22:44:25 +0800 Subject: [PATCH 282/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20different=20base=20URLs=20use=20the=20same=20clie?= =?UTF-8?q?nt.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where Vercel uses a different event loop, causing httpx's connection pool to attempt to reuse the same connection in a different event loop, which is not allowed in asyncio. 💰 Sponsors: Thanks to @PowerHunter for the ¥1800 sponsorship, sponsorship information has been added to the README. --- README.md | 2 +- README_CN.md | 2 +- main.py | 39 +++++++++++++++++++++++---------------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 86c16cb2..76406c9b 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ We thank the following sponsors for their support: -- @PowerHunter: ¥1400 +- @PowerHunter: ¥1800 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index 3373accb..986650af 100644 --- a/README_CN.md +++ b/README_CN.md @@ -333,7 +333,7 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ 我们感谢以下赞助商的支持： -- @PowerHunter：¥1400 +- @PowerHunter：¥1800 ## 如何赞助我们 diff --git a/main.py b/main.py index 17ad6dd6..28695914 100644 --- a/main.py +++ b/main.py @@ -48,6 +48,8 @@ # 添加新的环境变量检查 DISABLE_DATABASE = os.getenv("DISABLE_DATABASE", "false").lower() == "true" +IS_VERCEL = os.path.dirname(os.path.abspath(__file__)).startswith('/var/task') +logger.info("IS_VERCEL: %s", IS_VERCEL) async def create_tables(): if DISABLE_DATABASE: @@ -594,16 +596,28 @@ async def moderate_content(self, content, api_index): class ClientManager: def __init__(self, pool_size=100): self.pool_size = pool_size - self.clients = {} # {timeout_value: AsyncClient} + self.clients = {} # {host_timeout_proxy: AsyncClient} async def init(self, default_config): self.default_config = default_config @asynccontextmanager - async def get_client(self, timeout_value, proxy=None): + async def get_client(self, timeout_value, base_url, proxy=None): # 直接获取或创建客户端,不使用锁 timeout_value = int(timeout_value) - if timeout_value not in self.clients: + + # 从base_url中提取主机名 + parsed_url = urlparse(base_url) + host = parsed_url.netloc + + # 创建唯一的客户端键 + client_key = f"{host}_{timeout_value}" + if proxy: + # 对代理URL进行规范化处理 + proxy_normalized = proxy.replace('socks5h://', 'socks5://') + client_key += f"_{proxy_normalized}" + + if client_key not in self.clients or IS_VERCEL: timeout = httpx.Timeout( connect=15.0, read=timeout_value, @@ -620,39 +634,32 @@ async def get_client(self, timeout_value, proxy=None): if proxy: # 解析代理URL - from urllib.parse import urlparse parsed = urlparse(proxy) - - # 修改这里: 将 socks5h 转换为 socks5 scheme = parsed.scheme.rstrip('h') - # print("scheme", scheme) if scheme == 'socks5': try: from httpx_socks import AsyncProxyTransport - # 使用修改后的scheme创建代理URL proxy = proxy.replace('socks5h://', 'socks5://') - # 创建SOCKS5代理传输 transport = AsyncProxyTransport.from_url(proxy) client_config["transport"] = transport except ImportError: logger.error("httpx-socks package is required for SOCKS proxy support") raise ImportError("Please install httpx-socks package for SOCKS proxy support: pip install httpx-socks") else: - # 对于HTTP/HTTPS代理使用原有方式 client_config["proxies"] = { "http://": proxy, "https://": proxy } - self.clients[timeout_value] = httpx.AsyncClient(**client_config) + self.clients[client_key] = httpx.AsyncClient(**client_config) try: - yield self.clients[timeout_value] + yield self.clients[client_key] except Exception as e: - if timeout_value in self.clients: - tmp_client = self.clients[timeout_value] - del self.clients[timeout_value] # 先删除引用 + if client_key in self.clients: + tmp_client = self.clients[client_key] + del self.clients[client_key] # 先删除引用 await tmp_client.aclose() # 然后关闭客户端 raise e @@ -830,7 +837,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A # print("proxy", proxy) try: - async with app.state.client_manager.get_client(timeout_value, proxy) as client: + async with app.state.client_manager.get_client(timeout_value, url, proxy) as client: # 打印client配置信息 # logger.info(f"Client config - Timeout: {client.timeout}") # logger.info(f"Client config - Headers: {client.headers}") From e13d491df417407a3dfc0236af6521e18c8a1b9d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Nov 2024 14:44:53 +0000 Subject: [PATCH 283/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a2dc0c7e..dcb37be6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.96 +0.0.97 From 83b5b1b58b48077a72a31d7f660d41194cabf597 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 15 Nov 2024 00:06:38 +0800 Subject: [PATCH 284/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20infinite=20loop=20error=20when=20there=20is=20only=20on?= =?UTF-8?q?e=20weighted=20channel.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 28695914..d76355cf 100644 --- a/main.py +++ b/main.py @@ -1036,6 +1036,8 @@ async def get_right_order_providers(request_model, config, api_index, scheduling # 步骤 3: 计算交集 intersection = all_providers.intersection(weight_keys) # print("intersection", intersection) + if len(intersection) == 1: + intersection = None if intersection: filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k in intersection} @@ -1097,6 +1099,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques retry_count = 0 while True: + # print("start_index", start_index) + # print("index", index) + # print("num_matching_providers", num_matching_providers) + # print("retry_count", retry_count) if index >= num_matching_providers + retry_count: break current_index = (start_index + index) % num_matching_providers @@ -1136,8 +1142,10 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # source_model = list(provider['model'][0].keys())[0] await app.state.channel_manager.exclude_model(channel_id, request_model) matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) + last_num_matching_providers = num_matching_providers num_matching_providers = len(matching_providers) - index = 0 + if num_matching_providers != last_num_matching_providers: + index = 0 cooling_time = safe_get(provider, "preferences", "api_key_cooldown_period", default=0) api_key_count = provider_api_circular_list[channel_id].get_items_count() From ae1a71ea1f3db08d3ffc015b65f2eb9fda295ccd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Nov 2024 16:07:08 +0000 Subject: [PATCH 285/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dcb37be6..51ce6b0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.97 +0.0.98 From 765297f6101d058e2e18544c335505cdfbfad28b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 17 Nov 2024 03:13:56 +0800 Subject: [PATCH 286/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20using=20wildcard=20configuration=20for=20weights=20f?= =?UTF-8?q?ails=20to=20find=20available=20channels.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index d76355cf..bf90bdfd 100644 --- a/main.py +++ b/main.py @@ -1040,7 +1040,7 @@ async def get_right_order_providers(request_model, config, api_index, scheduling intersection = None if intersection: - filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k in intersection} + filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k.split("/")[0] + "/" + request_model in intersection} # print("filtered_weights", filtered_weights) if scheduling_algorithm == "weighted_round_robin": From d69517f4aa89b132615a6ece20029e289d449965 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 16 Nov 2024 19:14:17 +0000 Subject: [PATCH 287/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 51ce6b0e..9a0bf0d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.98 +0.0.99 From cd5ba5bf6523c8712a2b4d3b947a4c32d340f8d9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 17 Nov 2024 03:45:44 +0800 Subject: [PATCH 288/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 35 ++++++++++++++++++++++++++++++++++- README_CN.md | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76406c9b..9ce8679f 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,17 @@ yym68686/uni-api:latest After clicking the one-click deploy button above, set the environment variable `CONFIG_URL` to the direct link of the configuration file, `DISABLE_DATABASE` to true, and then click Create to create the project. After deployment, you need to manually set the Function Max Duration to 60 seconds in the Vercel project panel under Settings -> Functions, and then click the Deployments menu and click Redeploy to redeploy, which will set the timeout to 60 seconds. If you do not redeploy, the default timeout will remain at the original 10 seconds. Note that you should not delete the Vercel project and recreate it; instead, click redeploy in the Deployments menu within the currently deployed Vercel project to make the Function Max Duration modification take effect. -## Serv00 remote deployment +## Ubuntu deployment + +In the warehouse Releases, find the latest version of the corresponding binary file, for example, a file named uni-api-linux-x86_64-0.0.99.pex. Download the binary file on the server and run it: + +```bash +wget https://github.com/yym68686/uni-api/releases/download/v0.0.99/uni-api-linux-x86_64-0.0.99.pex +chmod +x uni-api-linux-x86_64-0.0.99.pex +./uni-api-linux-x86_64-0.0.99.pex +``` + +## Serv00 Remote Deployment (FreeBSD 14.0) First, log in to the panel, in Additional services click on the tab Run your own applications to enable the option to run your own programs, then go to the panel Port reservation to randomly open a port. @@ -329,6 +339,29 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` +pex linux packaging: + +```bash +VERSION=$(cat VERSION) +pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex +``` + +macOS packaging: + +```bash +VERSION=$(cat VERSION) +pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex +``` + ## Sponsors We thank the following sponsors for their support: diff --git a/README_CN.md b/README_CN.md index 986650af..30152000 100644 --- a/README_CN.md +++ b/README_CN.md @@ -206,7 +206,17 @@ yym68686/uni-api:latest 点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。部署完之后需要手动在 vercel 项目面板的 Settings -> Funcitons -> Function Max Duration 设置为 60 秒，然后点击 Deployments 菜单点击 Redeploy 重新部署，即可将超时时间设置为 60 秒，如果不重新部署，默认超时时间将是原来的 10 秒。注意不是删掉 vercel 项目重建，而是在当前部署好的 vercel 项目里面的 Deployments 菜单里面点 redeploy，这样才能让 Function Max Duration 的修改生效。 -## serv00 远程部署 +## Ubuntu 部署 + +在仓库 Releases 找到对应的二进制文件最新版本，例如名为 uni-api-linux-x86_64-0.0.99.pex 的文件。在服务器下载二进制文件并运行： + +```bash +wget https://github.com/yym68686/uni-api/releases/download/v0.0.99/uni-api-linux-x86_64-0.0.99.pex +chmod +x uni-api-linux-x86_64-0.0.99.pex +./uni-api-linux-x86_64-0.0.99.pex +``` + +## serv00 远程部署（FreeBSD 14.0） 首先登录面板，Additional services 里面点击选项卡 Run your own applications 开启允许运行自己的程序，然后到面板 Port reservation 去随便开一个端口。 @@ -329,6 +339,29 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` +pex linux 打包： + +```bash +VERSION=$(cat VERSION) +pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex +``` + +macos 打包： + +```bash +VERSION=$(cat VERSION) +pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex +``` + ## 赞助商 我们感谢以下赞助商的支持： From 6f7d288b17f8a73dbe8327b55172a3319be39cdb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 17 Nov 2024 22:07:04 +0800 Subject: [PATCH 289/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20errors=20are=20not=20correctly=20returned=20when=20t?= =?UTF-8?q?here=20is=20a=20syntax=20error=20in=20the=20configuration=20fil?= =?UTF-8?q?e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 12 +++++++++++- utils.py | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index bf90bdfd..dadacbd0 100644 --- a/main.py +++ b/main.py @@ -682,7 +682,17 @@ async def ensure_config(request: Request, call_next): if len(app.state.api_keys_db) >= 1: app.state.admin_api_key = app.state.api_keys_db[0].get("api") else: - raise Exception("No admin API key found") + from utils import yaml_error_message + if yaml_error_message: + return JSONResponse( + status_code=500, + content={"error": yaml_error_message} + ) + else: + return JSONResponse( + status_code=500, + content={"error": "No admin API key found"} + ) if app and not hasattr(app.state, 'client_manager'): diff --git a/utils.py b/utils.py index c3048793..f9a68f94 100644 --- a/utils.py +++ b/utils.py @@ -246,6 +246,7 @@ def update_initial_model(api_url, api): yaml.indent(mapping=2, sequence=4, offset=2) API_YAML_PATH = "./api.yaml" +yaml_error_message = None def save_api_yaml(config_data): with open(API_YAML_PATH, "w", encoding="utf-8") as f: @@ -355,6 +356,8 @@ async def load_config(app=None): config, api_keys_db, api_list = {}, {}, [] except YAMLError as e: logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。%s", e) + global yaml_error_message + yaml_error_message = "配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。" config, api_keys_db, api_list = {}, {}, [] except OSError as e: logger.error(f"open 'api.yaml' failed: {e}") From 5962f8a5ddfcc88b52147d51297feaef77dae205 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Nov 2024 14:07:22 +0000 Subject: [PATCH 290/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9a0bf0d7..8e3cdca8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.99 +0.0.100 From 607b9a8b3a054cbbbc5922e50bf0c9a589081760 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Nov 2024 16:14:00 +0000 Subject: [PATCH 291/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.101?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8e3cdca8..39d8d9e1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.100 +0.0.101 From e9954877058de616c962a5f58b98656c945880fe Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 18 Nov 2024 01:53:46 +0800 Subject: [PATCH 292/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20setting=20AUTO=5FRETRY=20to=20a=20number=20to=20cu?= =?UTF-8?q?stomize=20the=20retry=20count.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- main.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9ce8679f..92844b95 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ api_keys: SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the channel of the first model with a request. Default is enabled, SCHEDULING_ALGORITHM default value is fixed_priority. SCHEDULING_ALGORITHM optional values are: fixed_priority, round_robin, weighted_round_robin, lottery, random. # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel of the model with a request. # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the model used by the user in order. - AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true + AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true. Also supports setting a number, indicating the number of retries. RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional # RATE_LIMIT: 2/min,10/day # Supports multiple frequency constraints ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. diff --git a/README_CN.md b/README_CN.md index 30152000..80ca6f80 100644 --- a/README_CN.md +++ b/README_CN.md @@ -150,7 +150,7 @@ api_keys: SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。默认开启，SCHEDULING_ALGORITHM 缺省值为 fixed_priority。SCHEDULING_ALGORITHM 可选值有：fixed_priority，round_robin，weighted_round_robin, lottery, random。 # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 # 当 SCHEDULING_ALGORITHM 为 round_robin 时，使用轮训负载均衡，按照顺序请求用户使用的模型的渠道。 - AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true + AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true。也可以设置为数字，表示重试次数。 RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 # RATE_LIMIT: 2/min,10/day 支持多个频率约束条件 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 diff --git a/main.py b/main.py index d6f14de4..11a5d7d8 100644 --- a/main.py +++ b/main.py @@ -1105,7 +1105,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if num_matching_providers == 1 and (count := provider_api_circular_list[matching_providers[0]['provider']].get_items_count()) > 1: retry_count = count else: - retry_count = 0 + retry_count = int(auto_retry) while True: # print("start_index", start_index) From 9892a5f2b6b4fb9e1b6728a5c3096da692713838 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Nov 2024 17:54:04 +0000 Subject: [PATCH 293/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.102?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 39d8d9e1..339aaaf4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.101 +0.0.102 From 25e0bb8a8493e8edef98e80ebdb2745f387b0b2a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 18 Nov 2024 05:53:46 +0800 Subject: [PATCH 294/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20request=20information=20cannot=20be=20stored=20in=20?= =?UTF-8?q?the=20database.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/main.py b/main.py index 11a5d7d8..a53accd7 100644 --- a/main.py +++ b/main.py @@ -261,29 +261,6 @@ class ChannelStat(Base): from starlette.types import Scope, Receive, Send from starlette.responses import Response -from decimal import Decimal, getcontext - -# 设置全局精度 -getcontext().prec = 17 # 设置为17是为了确保15位小数的精度 - -def calculate_cost(model: str, input_tokens: int, output_tokens: int) -> Decimal: - costs = { - "gpt-4": {"input": Decimal('5.0') / Decimal('1000000'), "output": Decimal('15.0') / Decimal('1000000')}, - "claude-3-sonnet": {"input": Decimal('3.0') / Decimal('1000000'), "output": Decimal('15.0') / Decimal('1000000')} - } - - if model not in costs: - logger.error(f"Unknown model: {model}") - return 0 - - model_costs = costs[model] - input_cost = Decimal(input_tokens) * model_costs["input"] - output_cost = Decimal(output_tokens) * model_costs["output"] - total_cost = input_cost + output_cost - - # 返回精确到15位小数的结果 - return total_cost.quantize(Decimal('0.000000000000001')) - from asyncio import Semaphore # 创建一个信号量来控制数据库访问 @@ -383,9 +360,9 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.body_iterator.aclose() self._closed = True - process_time = time() - self.current_info["start_time"] - self.current_info["process_time"] = process_time - await update_stats(self.current_info) + process_time = time() - self.current_info["start_time"] + self.current_info["process_time"] = process_time + await update_stats(self.current_info) async def _logging_iterator(self): try: From 0e55435feb77d609628fcc67a0bf68427564aed4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Nov 2024 21:54:05 +0000 Subject: [PATCH 295/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.103?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 339aaaf4..6fa08006 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.102 +0.0.103 From c23690bb756ad1abd58a164ed257d61f3defa2f0 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 18 Nov 2024 06:31:46 +0800 Subject: [PATCH 296/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20tokens=20cannot=20be=20calculated.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index a53accd7..b385ca36 100644 --- a/main.py +++ b/main.py @@ -369,7 +369,7 @@ async def _logging_iterator(self): async for chunk in self.body_iterator: if isinstance(chunk, str): chunk = chunk.encode('utf-8') - if isinstance(chunk, bytes): + if self.current_info.get("endpoint") == "/v1/audio/speech": yield chunk continue line = chunk.decode('utf-8') From 88cac46b648d4f1abe093409bc404be6c9913c62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Nov 2024 22:32:13 +0000 Subject: [PATCH 297/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.104?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6fa08006..aa8574a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.103 +0.0.104 From da7c8720926b47d52b7dccc345278671eea6de8a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 19 Nov 2024 01:30:13 +0800 Subject: [PATCH 298/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20Gemini=20custom=20base=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💰 Sponsors: Thanks to @ioi for the ¥50 sponsorship, sponsorship information has been added to the README. --- README.md | 1 + README_CN.md | 1 + main.py | 2 +- request.py | 11 +++++++---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 92844b95..fb5f253f 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,7 @@ pex -r requirements.txt \ We thank the following sponsors for their support: - @PowerHunter: ¥1800 +- @ioi：¥50 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index 80ca6f80..5a2d178a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -367,6 +367,7 @@ pex -r requirements.txt \ 我们感谢以下赞助商的支持： - @PowerHunter：¥1800 +- @ioi：¥50 ## 如何赞助我们 diff --git a/main.py b/main.py index b385ca36..96ea071a 100644 --- a/main.py +++ b/main.py @@ -749,7 +749,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A parsed_url = urlparse(url) # print("parsed_url", parsed_url) engine = None - if parsed_url.netloc == 'generativelanguage.googleapis.com': + if parsed_url.path.startswith("/v1beta") or parsed_url.path.startswith("/v1"): engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" diff --git a/request.py b/request.py index a0aa963e..7a49c4be 100644 --- a/request.py +++ b/request.py @@ -124,10 +124,13 @@ async def get_gemini_payload(request, engine, provider): model = model_dict[request.model] gemini_stream = "streamGenerateContent" url = provider['base_url'] - if url.endswith("v1beta"): - url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next(model)) - if url.endswith("v1"): - url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=await provider_api_circular_list[provider['provider']].next(model)) + parsed_url = urllib.parse.urlparse(url) + if parsed_url.path.startswith("/v1beta") or parsed_url.path.startswith("/v1"): + api_version = parsed_url.path.split('/')[-1] # 获取 v1 或 v1beta + else: + api_version = "v1beta" + # https://generativelanguage.googleapis.com/v1beta/models/ + url = f"{parsed_url.scheme}://{parsed_url.netloc}/{api_version}/models/{model}:{gemini_stream}?key={await provider_api_circular_list[provider['provider']].next(model)}" messages = [] systemInstruction = None From 88a12f8cfcc0aab3bd890a0665ef82858bc95c8e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Nov 2024 17:30:43 +0000 Subject: [PATCH 299/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.105?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index aa8574a8..759a4a15 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.104 +0.0.105 From f79812fe30464ba0a9dadb2aab4e09351606b24c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 19 Nov 2024 05:03:34 +0800 Subject: [PATCH 300/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20caused=20by=20the=20direct=20return=20of=20[DONE]=20in=20?= =?UTF-8?q?the=20API=20leading=20to=20JSON=20parsing=20errors.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where the Gemini API's judgment range expands. --- .github/workflows/release.yml | 69 +++++++++++++++++++++++++++++++++++ main.py | 2 +- response.py | 6 ++- 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..8f005ca2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,69 @@ +name: Build and Release PEX + +on: + push: + tags: + - 'v*' + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + platform: linux + arch: x86_64 + - os: macos-latest + platform: macos + arch: arm64 + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10.12' + + - name: Install pex + run: pip install pex + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Get Version + id: get_version + run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV + + - name: Build Linux PEX + if: matrix.platform == 'linux' + run: | + pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex + + - name: Build MacOS PEX + if: matrix.platform == 'macos' + run: | + pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex + + - name: Create Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + name: Release ${{ env.VERSION }} + draft: false + prerelease: false + files: | + uni-api-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.VERSION }}.pex diff --git a/main.py b/main.py index 96ea071a..30fdc75c 100644 --- a/main.py +++ b/main.py @@ -749,7 +749,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A parsed_url = urlparse(url) # print("parsed_url", parsed_url) engine = None - if parsed_url.path.startswith("/v1beta") or parsed_url.path.startswith("/v1"): + if parsed_url.path.startswith("/v1beta") or parsed_url.path.endswith("/v1"): engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" diff --git a/response.py b/response.py index 9e46d219..eca415b6 100644 --- a/response.py +++ b/response.py @@ -202,7 +202,11 @@ async def fetch_gpt_response_stream(client, url, headers, payload): line, buffer = buffer.split("\n", 1) # logger.info("line: %s", repr(line)) if line and line != "data: " and line != "data:" and not line.startswith(": "): - line = json.loads(line.lstrip("data: ")) + result = line.lstrip("data: ") + if result.strip() == "[DONE]": + yield "data: [DONE]" + end_of_line + return + line = json.loads(result) line['id'] = f"chatcmpl-{random_str}" yield "data: " + json.dumps(line).strip() + end_of_line From 890c36ee287ed3ffa0a993bcc94832792f7d9513 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Nov 2024 21:03:55 +0000 Subject: [PATCH 301/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.106?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 759a4a15..4ff99ed9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.105 +0.0.106 From 7e538de19e37450549851c01fd3bbc25ccf18498 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 19 Nov 2024 23:29:11 +0800 Subject: [PATCH 302/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20non-streaming=20response=20content=20is=20empty=20wi?= =?UTF-8?q?thout=20throwing=20an=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 35483de9..56e19891 100644 --- a/utils.py +++ b/utils.py @@ -442,7 +442,7 @@ async def error_handling_wrapper(generator, channel_id): first_item = await generator.__anext__() first_response_time = time_module.time() - start_time first_item_str = first_item - # logger.info("first_item_str: %s", first_item_str) + # logger.info("first_item_str: %s :%s", type(first_item_str), first_item_str) if isinstance(first_item_str, (bytes, bytearray)): if identify_audio_format(first_item_str) in ["MP3", "MP3 with ID3", "OPUS", "AAC (ADIF)", "AAC (ADTS)", "FLAC", "WAV"]: return first_item, first_response_time @@ -470,6 +470,10 @@ async def error_handling_wrapper(generator, channel_id): status_code = first_item_str.get('status_code', 500) detail = first_item_str.get('details', f"{first_item_str}") raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) + if isinstance(first_item_str, dict): + content = safe_get(first_item_str, "choices", 0, "message", "content", default=None) + if content == "" or content is None: + raise StopAsyncIteration # 如果不是错误，创建一个新的生成器，首先yield第一个项，然后yield剩余的项 async def new_generator(): From c38f266bf3dc7d7042e74a5ee13ff9e41b76f846 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 19 Nov 2024 15:29:32 +0000 Subject: [PATCH 303/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.107?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4ff99ed9..fbad1dd7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.106 +0.0.107 From 6a4ecbbfe7d19bfd980b79a027339444666158b0 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 19 Nov 2024 23:35:36 +0800 Subject: [PATCH 304/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fixing=20the=20bu?= =?UTF-8?q?g=20where=20non-streaming=20response=20content=20is=20empty=20w?= =?UTF-8?q?ithout=20restricting=20non-streaming=20responses.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 4 ++-- utils.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 30fdc75c..309e838a 100644 --- a/main.py +++ b/main.py @@ -835,11 +835,11 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A async with app.state.client_manager.get_client(timeout_value, url, proxy) as client: if request.stream: generator = fetch_response_stream(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: generator = fetch_response(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream) # 处理音频和其他二进制响应 if endpoint == "/v1/audio/speech": diff --git a/utils.py b/utils.py index 56e19891..2538c66b 100644 --- a/utils.py +++ b/utils.py @@ -436,7 +436,7 @@ def identify_audio_format(file_bytes): import asyncio import time as time_module -async def error_handling_wrapper(generator, channel_id): +async def error_handling_wrapper(generator, channel_id, engine, stream): start_time = time_module.time() try: first_item = await generator.__anext__() @@ -470,7 +470,8 @@ async def error_handling_wrapper(generator, channel_id): status_code = first_item_str.get('status_code', 500) detail = first_item_str.get('details', f"{first_item_str}") raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) - if isinstance(first_item_str, dict): + + if isinstance(first_item_str, dict) and engine not in ["tts", "embedding", "dalle", "moderation", "whisper"] and stream == False: content = safe_get(first_item_str, "choices", 0, "message", "content", default=None) if content == "" or content is None: raise StopAsyncIteration From 570d6caf90716ede1882297afe2aca095f98406e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 19 Nov 2024 15:36:25 +0000 Subject: [PATCH 305/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.108?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fbad1dd7..48c15efb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.107 +0.0.108 From e3ef29e6dad8857e2e1a4ada02b20a87b482feef Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 21 Nov 2024 23:15:10 +0800 Subject: [PATCH 306/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20Gemini=20v1=20API=20that=20does=20not=20support=20syste?= =?UTF-8?q?m=20prompts.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/request.py b/request.py index 7a49c4be..b4ae8701 100644 --- a/request.py +++ b/request.py @@ -6,7 +6,7 @@ import urllib.parse from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, BaseAPI, get_model_dict, provider_api_circular_list +from utils import c35s, c3s, c3o, c3h, gem, BaseAPI, get_model_dict, provider_api_circular_list, safe_get import imghdr @@ -213,7 +213,13 @@ async def get_gemini_payload(request, engine, provider): ] } if systemInstruction: - payload["systemInstruction"] = systemInstruction + if api_version == "v1beta": + payload["systemInstruction"] = systemInstruction + if api_version == "v1": + first_message = safe_get(payload, "contents", 0, "parts", 0, "text", default=None) + system_instruction = safe_get(systemInstruction, "parts", 0, "text", default=None) + if first_message and system_instruction: + payload["contents"][0]["parts"][0]["text"] = system_instruction + "\n" + first_message miss_fields = [ 'model', From ea7c5c155f33dfd149a83ef16a9b9997ce5eee43 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 Nov 2024 15:15:29 +0000 Subject: [PATCH 307/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.109?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 48c15efb..4fa9ef0f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.108 +0.0.109 From 6d00cc2a73235c472d8101cc5f7cf9aaab02a986 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 24 Nov 2024 12:05:32 +0800 Subject: [PATCH 308/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Optimize=20log?= =?UTF-8?q?=20display,=20add=20version=20number=20log=20display.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main.py b/main.py index 309e838a..ba58fc08 100644 --- a/main.py +++ b/main.py @@ -50,6 +50,15 @@ DISABLE_DATABASE = os.getenv("DISABLE_DATABASE", "false").lower() == "true" IS_VERCEL = os.path.dirname(os.path.abspath(__file__)).startswith('/var/task') logger.info("IS_VERCEL: %s", IS_VERCEL) +logger.info("DISABLE_DATABASE: %s", DISABLE_DATABASE) + +# 读取VERSION文件内容 +try: + with open('VERSION', 'r') as f: + VERSION = f.read().strip() +except: + VERSION = 'unknown' +logger.info("VERSION: %s", VERSION) async def create_tables(): if DISABLE_DATABASE: From 4c36719bef11a7dfb29f08ab074f3283a96a0af1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 24 Nov 2024 04:05:57 +0000 Subject: [PATCH 309/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.110?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4fa9ef0f..8f2d5363 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.109 +0.0.110 From ee3ee1b38c66013f6c4a2f6e574a2f798e8ccd57 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 18:11:58 +0800 Subject: [PATCH 310/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Add=20support=20for=20error=20triggers.=20When=20the=20messa?= =?UTF-8?q?ge=20returned=20by=20the=20model=20contains=20any=20string=20fr?= =?UTF-8?q?om=20the=20error=20triggers,=20the=20channel=20will=20automatic?= =?UTF-8?q?ally=20return=20an=20error.=20Optional.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +++ README_CN.md | 3 +++ main.py | 11 +++++++++-- utils.py | 7 ++----- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fb5f253f..f9bf92b1 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,9 @@ preferences: # Global configuration o1-mini: 30 # Model o1-mini timeout is 30 seconds, when requesting models starting with o1-mini, the timeout is 30 seconds o1-preview: 100 # Model o1-preview timeout is 100 seconds, when requesting models starting with o1-preview, the timeout is 100 seconds cooldown_period: 300 # Channel cooldown time, in seconds, default 300 seconds, optional. When a model request fails, the channel will be automatically excluded and cooled down for a period of time, and will not request the channel again. After the cooldown time ends, the model will be automatically restored until the request fails again, and it will be cooled down again. When cooldown_period is set to 0, the cooling mechanism is not enabled. + error_triggers: # Error triggers, when the message returned by the model contains any of the strings in the error_triggers, the channel will return an error. Optional + - The bot's usage is covered by the developer + - process this request due to overload or policy ``` Mount the configuration file and start the uni-api docker container: diff --git a/README_CN.md b/README_CN.md index 5a2d178a..e06fab48 100644 --- a/README_CN.md +++ b/README_CN.md @@ -174,6 +174,9 @@ preferences: # 全局配置 o1-mini: 30 # 模型 o1-mini 的超时时间为 30 秒，当请求名字是 o1-mini 开头的模型时，超时时间是 30 秒 o1-preview: 100 # 模型 o1-preview 的超时时间为 100 秒，当请求名字是 o1-preview 开头的模型时，超时时间是 100 秒 cooldown_period: 300 # 渠道冷却时间，单位为秒，默认 300 秒，选填。当模型请求失败时，会自动将该渠道排除冷却一段时间，不再请求该渠道，冷却时间结束后，会自动将该模型恢复，直到再次请求失败，会重新冷却。当 cooldown_period 设置为 0 时，不启用冷却机制。 + error_triggers: # 错误触发器，当模型返回的消息包含错误触发器中的任意一个字符串时，该渠道会自动返回报错。选填 + - The bot's usage is covered by the developer + - process this request due to overload or policy ``` 挂载配置文件并启动 uni-api docker 容器： diff --git a/main.py b/main.py index ba58fc08..e055ffbd 100644 --- a/main.py +++ b/main.py @@ -735,6 +735,13 @@ async def ensure_config(request: Request, call_next): app.state.channel_manager = ChannelManager(cooldown_period=COOLDOWN_PERIOD) + if app and not hasattr(app.state, "error_triggers"): + if app.state.config and 'preferences' in app.state.config: + ERROR_TRIGGERS = app.state.config['preferences'].get('error_triggers', []) + else: + ERROR_TRIGGERS = [] + app.state.error_triggers = ERROR_TRIGGERS + return await call_next(request) def get_timeout_value(provider_timeouts, original_model): @@ -844,11 +851,11 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A async with app.state.client_manager.get_client(timeout_value, url, proxy) as client: if request.stream: generator = fetch_response_stream(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream, app.state.error_triggers) response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") else: generator = fetch_response(client, url, headers, payload, engine, original_model) - wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream, app.state.error_triggers) # 处理音频和其他二进制响应 if endpoint == "/v1/audio/speech": diff --git a/utils.py b/utils.py index 2538c66b..6b6a1be8 100644 --- a/utils.py +++ b/utils.py @@ -436,7 +436,7 @@ def identify_audio_format(file_bytes): import asyncio import time as time_module -async def error_handling_wrapper(generator, channel_id, engine, stream): +async def error_handling_wrapper(generator, channel_id, engine, stream, error_triggers): start_time = time_module.time() try: first_item = await generator.__anext__() @@ -454,10 +454,7 @@ async def error_handling_wrapper(generator, channel_id, engine, stream): if first_item_str.startswith("[DONE]"): logger.error(f"provider: {channel_id:<11} error_handling_wrapper [DONE]!") raise StopAsyncIteration - if "The bot's usage is covered by the developer" in first_item_str: - logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) - raise StopAsyncIteration - if "process this request due to overload or policy" in first_item_str: + if all(x not in first_item_str for x in error_triggers): logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) raise StopAsyncIteration try: From 5f00224e18ab05c357accfb313988149f34f14c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Nov 2024 10:12:27 +0000 Subject: [PATCH 311/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.111?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8f2d5363..0abba19e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.110 +0.0.111 From cc42a70768ebef1d85df47392b3868c36661bc8a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 18:15:54 +0800 Subject: [PATCH 312/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20error=5Ftriggers=20condition=20judgment.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 6b6a1be8..d2259499 100644 --- a/utils.py +++ b/utils.py @@ -454,7 +454,7 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr if first_item_str.startswith("[DONE]"): logger.error(f"provider: {channel_id:<11} error_handling_wrapper [DONE]!") raise StopAsyncIteration - if all(x not in first_item_str for x in error_triggers): + if any(x in first_item_str for x in error_triggers): logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) raise StopAsyncIteration try: From 89affc8ae33e291dea0b2d70785a7f388b0e9e1c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Nov 2024 10:16:41 +0000 Subject: [PATCH 313/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.112?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0abba19e..16b5f264 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.111 +0.0.112 From 8e3a5c1f2caaad03070f418f22fb0cd4e3063fd2 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 19:16:46 +0800 Subject: [PATCH 314/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20features?= =?UTF-8?q?=20to=20support=20setting=20different=20rate=20limits=20for=20d?= =?UTF-8?q?ifferent=20models=20at=20the=20user=20level.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++++-- README_CN.md | 7 +++++-- main.py | 19 +++++++++++++++++++ utils.py | 2 +- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f9bf92b1..5720b611 100644 --- a/README.md +++ b/README.md @@ -151,8 +151,11 @@ api_keys: # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel of the model with a request. # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the model used by the user in order. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true. Also supports setting a number, indicating the number of retries. - RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional - # RATE_LIMIT: 2/min,10/day # Supports multiple frequency constraints + rate_limit: 15/min # Supports rate limiting, each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day + # rate_limit: # You can set different frequency limits for each model + # gemini-1.5-pro: 3/min + # gemini-1.5-flash: 2/min + # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. # Channel-level weighted load balancing configuration example diff --git a/README_CN.md b/README_CN.md index e06fab48..ac3eaffd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -151,8 +151,11 @@ api_keys: # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 # 当 SCHEDULING_ALGORITHM 为 round_robin 时，使用轮训负载均衡，按照顺序请求用户使用的模型的渠道。 AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true。也可以设置为数字，表示重试次数。 - RATE_LIMIT: 2/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min，选填 - # RATE_LIMIT: 2/min,10/day 支持多个频率约束条件 + rate_limit: 15/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认999999/min，选填。支持多个频率约束条件：15/min,10/day + # rate_limit: # 可以为每个模型设置不同的频率限制 + # gemini-1.5-pro: 3/min + # gemini-1.5-flash: 2/min + # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 # 渠道级加权负载均衡配置示例 diff --git a/main.py b/main.py index e055ffbd..2f663dea 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,7 @@ error_handling_wrapper, rate_limiter, provider_api_circular_list, + ThreadSafeCircularList, ) from collections import defaultdict @@ -488,6 +489,15 @@ async def dispatch(self, request: Request, call_next): model = request_model.model current_info["model"] = model + final_api_key = app.state.api_list[api_index] + try: + await app.state.user_api_keys_rate_limit[final_api_key].next(model) + except Exception as e: + return JSONResponse( + status_code=429, + content={"error": "Too many requests"} + ) + moderated_content = None if request_model.request_type == "chat": moderated_content = request_model.get_last_text_message() @@ -666,6 +676,15 @@ async def ensure_config(request: Request, call_next): # logger.warning("Config not found, attempting to reload") app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) + if app.state.api_list: + app.state.user_api_keys_rate_limit = defaultdict(ThreadSafeCircularList) + for api_index, api_key in enumerate(app.state.api_list): + app.state.user_api_keys_rate_limit[api_key] = ThreadSafeCircularList( + [api_key], + safe_get(app.state.config, 'api_keys', api_index, "preferences", "rate_limit", default={"default": "999999/min"}), + "round_robin" + ) + for item in app.state.api_keys_db: if item.get("role") == "admin": app.state.admin_api_key = item.get("api") diff --git a/utils.py b/utils.py index d2259499..700e39cd 100644 --- a/utils.py +++ b/utils.py @@ -67,7 +67,7 @@ async def get_user_rate_limit(app, api_index: int = None): # 这里应该实现根据 token 获取用户速率限制的逻辑 # 示例： 返回 (次数， 秒数) config = app.state.config - raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "RATE_LIMIT") + raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "rate_limit") # print("raw_rate_limit", raw_rate_limit) # print("not api_index or not raw_rate_limit", api_index == None, not raw_rate_limit, api_index == None or not raw_rate_limit, api_index, raw_rate_limit) From b2c7c58fdf38a5839cb9ae00fa2cbd947e1309a7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Nov 2024 11:17:06 +0000 Subject: [PATCH 315/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 16b5f264..9fdfff61 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.112 +0.0.113 From 751e7bea1e44aff49152b13ae654cb55512ea4c6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 19:20:39 +0800 Subject: [PATCH 316/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++---- README_CN.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5720b611..0889c234 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ providers: preferences: api_key_rate_limit: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day # api_key_rate_limit: # You can set different frequency limits for each model - # gemini-1.5-pro: 3/min - # gemini-1.5-flash: 2/min + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. @@ -153,8 +153,8 @@ api_keys: AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true. Also supports setting a number, indicating the number of retries. rate_limit: 15/min # Supports rate limiting, each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day # rate_limit: # You can set different frequency limits for each model - # gemini-1.5-pro: 3/min - # gemini-1.5-flash: 2/min + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. diff --git a/README_CN.md b/README_CN.md index ac3eaffd..6ad77732 100644 --- a/README_CN.md +++ b/README_CN.md @@ -92,8 +92,8 @@ providers: preferences: api_key_rate_limit: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min。支持多个频率约束条件：15/min,10/day # api_key_rate_limit: # 可以为每个模型设置不同的频率限制 - # gemini-1.5-pro: 3/min - # gemini-1.5-flash: 2/min + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡。 @@ -153,8 +153,8 @@ api_keys: AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true。也可以设置为数字，表示重试次数。 rate_limit: 15/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认999999/min，选填。支持多个频率约束条件：15/min,10/day # rate_limit: # 可以为每个模型设置不同的频率限制 - # gemini-1.5-pro: 3/min - # gemini-1.5-flash: 2/min + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 From db8226be22e9764cc680c20b09f701375a9af321 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 19:35:12 +0800 Subject: [PATCH 317/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20Vertex=20Gemini=20API=20to=20use=20the=20official?= =?UTF-8?q?=20Google=20Search=20Retrieval=20search=20interface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/request.py b/request.py index b4ae8701..2949e752 100644 --- a/request.py +++ b/request.py @@ -448,6 +448,16 @@ async def get_vertex_gemini_payload(request, engine, provider): else: payload[field] = value + if request.model.endswith("-search"): + if "tools" not in payload: + payload["tools"] = [{ + "googleSearchRetrieval": {} + }] + else: + payload["tools"].append({ + "googleSearchRetrieval": {} + }) + return url, headers, payload async def get_vertex_claude_payload(request, engine, provider): From a7fa1087f0b1f36f60a468bbdabdd2f14d8cfe95 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Nov 2024 11:35:34 +0000 Subject: [PATCH 318/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.114?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9fdfff61..58ca857c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.113 +0.0.114 From 6bc5b55666558214b46e818d0ba8c8f0c0515540 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 19:41:53 +0800 Subject: [PATCH 319/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_CN.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 0889c234..c544f48b 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ providers: model: - gemini-1.5-pro - gemini-1.5-flash + - gemini-1.5-pro: gemini-1.5-pro-search # Only supports using the gemini-1.5-pro-search model to request uni-api when using the Vertex Gemini API, to automatically use the Google official search tool. - claude-3-5-sonnet@20240620: claude-3-5-sonnet - claude-3-opus@20240229: claude-3-opus - claude-3-sonnet@20240229: claude-3-sonnet diff --git a/README_CN.md b/README_CN.md index 6ad77732..7c0e72f9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -110,6 +110,7 @@ providers: model: - gemini-1.5-pro - gemini-1.5-flash + - gemini-1.5-pro: gemini-1.5-pro-search # 仅支持在 vertex Gemini API 中，以 -search 后缀重命名模型后，使用 gemini-1.5-pro-search 模型请求 uni-api 时，表示 gemini-1.5-pro 模型自动使用 Google 官方搜索工具。 - claude-3-5-sonnet@20240620: claude-3-5-sonnet - claude-3-opus@20240229: claude-3-opus - claude-3-sonnet@20240229: claude-3-sonnet From 67e1a234b519c96df2d551e1abfcd54e0c073bb9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 25 Nov 2024 23:47:05 +0800 Subject: [PATCH 320/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20the=20old=20rate=20limit=20code=20did=20not=20remove.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 42 ++++++++++-------------------------------- utils.py | 14 -------------- 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/main.py b/main.py index 2f663dea..e240d5c7 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,6 @@ save_api_yaml, get_model_dict, post_all_models, - get_user_rate_limit, circular_list_encoder, error_handling_wrapper, rate_limiter, @@ -1199,27 +1198,6 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques security = HTTPBearer() -async def rate_limit_dependency(request: Request, credentials: HTTPAuthorizationCredentials = Depends(security)): - token = credentials.credentials if credentials else None - api_list = app.state.api_list - try: - api_index = api_list.index(token) - except ValueError: - # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 - api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) - if api_index is None: - print("error: Invalid or missing API Key:", token) - api_index = None - token = None - - # 使用 IP 地址和 token（如果有）作为限制键 - client_ip = request.client.host - rate_limit_key = f"{client_ip}:{token}" if token else client_ip - - limits = await get_user_rate_limit(app, api_index) - if await rate_limiter.is_rate_limited(rate_limit_key, limits): - raise HTTPException(status_code=429, detail="Too many requests") - def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): api_list = app.state.api_list token = credentials.credentials @@ -1250,15 +1228,15 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec raise HTTPException(status_code=403, detail="Permission denied") return token -@app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/chat/completions") async def request_model(request: RequestModel, api_index: int = Depends(verify_api_key)): return await model_handler.request_model(request, api_index) -@app.options("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)]) +@app.options("/v1/chat/completions") async def options_handler(): return JSONResponse(status_code=200, content={"detail": "OPTIONS allowed"}) -@app.get("/v1/models", dependencies=[Depends(rate_limit_dependency)]) +@app.get("/v1/models") async def list_models(api_index: int = Depends(verify_api_key)): models = post_all_models(api_index, app.state.config) return JSONResponse(content={ @@ -1266,28 +1244,28 @@ async def list_models(api_index: int = Depends(verify_api_key)): "data": models }) -@app.post("/v1/images/generations", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/images/generations") async def images_generations( request: ImageGenerationRequest, api_index: int = Depends(verify_api_key) ): return await model_handler.request_model(request, api_index, endpoint="/v1/images/generations") -@app.post("/v1/embeddings", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/embeddings") async def embeddings( request: EmbeddingRequest, api_index: int = Depends(verify_api_key) ): return await model_handler.request_model(request, api_index, endpoint="/v1/embeddings") -@app.post("/v1/audio/speech", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/audio/speech") async def audio_speech( request: TextToSpeechRequest, api_index: str = Depends(verify_api_key) ): return await model_handler.request_model(request, api_index, endpoint="/v1/audio/speech") -@app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/moderations") async def moderations( request: ModerationRequest, api_index: int = Depends(verify_api_key) @@ -1296,7 +1274,7 @@ async def moderations( from fastapi import UploadFile, File, Form, HTTPException import io -@app.post("/v1/audio/transcriptions", dependencies=[Depends(rate_limit_dependency)]) +@app.post("/v1/audio/transcriptions") async def audio_transcriptions( file: UploadFile = File(...), model: str = Form(...), @@ -1322,7 +1300,7 @@ async def audio_transcriptions( traceback.print_exc() raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}") -@app.get("/v1/generate-api-key", dependencies=[Depends(rate_limit_dependency)]) +@app.get("/v1/generate-api-key") def generate_api_key(): # Define the character set (only alphanumeric) chars = string.ascii_letters + string.digits @@ -1336,7 +1314,7 @@ def generate_api_key(): from sqlalchemy import func, desc, case from fastapi import Query -@app.get("/v1/stats", dependencies=[Depends(rate_limit_dependency)]) +@app.get("/v1/stats") async def get_stats( request: Request, token: str = Depends(verify_admin_api_key), diff --git a/utils.py b/utils.py index 700e39cd..f2173a23 100644 --- a/utils.py +++ b/utils.py @@ -63,20 +63,6 @@ async def is_rate_limited(self, key: str, limits) -> bool: rate_limiter = InMemoryRateLimiter() -async def get_user_rate_limit(app, api_index: int = None): - # 这里应该实现根据 token 获取用户速率限制的逻辑 - # 示例： 返回 (次数， 秒数) - config = app.state.config - raw_rate_limit = safe_get(config, 'api_keys', api_index, "preferences", "rate_limit") - # print("raw_rate_limit", raw_rate_limit) - # print("not api_index or not raw_rate_limit", api_index == None, not raw_rate_limit, api_index == None or not raw_rate_limit, api_index, raw_rate_limit) - - if api_index == None or not raw_rate_limit: - return [(30, 60)] - - rate_limit = parse_rate_limit(raw_rate_limit) - return rate_limit - import asyncio class ThreadSafeCircularList: From deb4c468001cc6fe3a9dbf3520e74cdf11da9bd9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 25 Nov 2024 15:47:27 +0000 Subject: [PATCH 321/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.115?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58ca857c..f64402b5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.114 +0.0.115 From 64c0556de04c2573d00080aaa719e40689f57e73 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 11:41:53 +0800 Subject: [PATCH 322/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20API=20key=20attribute=20inheritance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_CN.md | 1 + main.py | 92 ++++++++++++++++++++++++++++++++++++++-------------- request.py | 3 ++ 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c544f48b..c0a84de9 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ api_keys: - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. This syntax will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. - # By adding angle brackets on both sides of the model name, it will not search for the claude-3-5-sonnet model under the channel named anthropic, but will take the entire anthropic/claude-3-5-sonnet as the model name. This syntax can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moderation. + - sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo/* # Support using other API keys as channels preferences: SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the channel of the first model with a request. Default is enabled, SCHEDULING_ALGORITHM default value is fixed_priority. SCHEDULING_ALGORITHM optional values are: fixed_priority, round_robin, weighted_round_robin, lottery, random. # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel of the model with a request. diff --git a/README_CN.md b/README_CN.md index 7c0e72f9..2ccc135a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -147,6 +147,7 @@ api_keys: - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。这种写法不会匹配到other-provider提供的名为anthropic/claude-3-5-sonnet的模型。 - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 - openai-test/text-moderation-latest # 当开启消息道德审查后，可以使用名为 openai-test 渠道下的 text-moderation-latest 模型进行道德审查。 + - sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo/* # 支持将其他 api key 当作渠道 preferences: SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。默认开启，SCHEDULING_ALGORITHM 缺省值为 fixed_priority。SCHEDULING_ALGORITHM 可选值有：fixed_priority，round_robin，weighted_round_robin, lottery, random。 # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 diff --git a/main.py b/main.py index e240d5c7..64109671 100644 --- a/main.py +++ b/main.py @@ -760,6 +760,35 @@ async def ensure_config(request: Request, call_next): ERROR_TRIGGERS = [] app.state.error_triggers = ERROR_TRIGGERS + if app and app.state.api_keys_db and not hasattr(app.state, "models_list"): + app.state.models_list = {} + for item in app.state.api_keys_db: + api_key_model_list = item.get("model", []) + for provider_rule in api_key_model_list: + provider_name = provider_rule.split("/")[0] + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + models_list = [] + try: + # 构建请求头 + headers = { + "Authorization": f"Bearer {provider_name}" + } + # 发送GET请求获取模型列表 + base_url = "http://127.0.0.1:8000/v1/models" + async with app.state.client_manager.get_client(100, base_url) as client: + response = await client.get( + base_url, + headers=headers + ) + if response.status_code == 200: + models_data = response.json() + # 将获取到的模型添加到models_list + for model in models_data.get("data", []): + models_list.append(model["id"]) + except Exception as e: + logger.error(f"获取模型列表失败: {str(e)}") + app.state.models_list[provider_name] = models_list + return await call_next(request) def get_timeout_value(provider_timeouts, original_model): @@ -934,7 +963,7 @@ def lottery_scheduling(weights): break return selections -def get_provider_rules(model_rule, config, request_model): +async def get_provider_rules(model_rule, config, request_model): provider_rules = [] if model_rule == "all": # 如模型名为 all，则返回所有模型 @@ -955,10 +984,19 @@ def get_provider_rules(model_rule, config, request_model): provider_name = model_rule.split("/")[0] model_name_split = "/".join(model_rule.split("/")[1:]) models_list = [] - for provider in config['providers']: - model_dict = get_model_dict(provider) - if provider['provider'] == provider_name: - models_list.extend(list(model_dict.keys())) + + # api_keys 中 api 为 sk- 时，表示继承 api_keys，将 api_keys 中的 api key 当作 渠道 + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + if app.state.models_list.get(provider_name): + models_list = app.state.models_list[provider_name] + else: + models_list = [] + else: + for provider in config['providers']: + model_dict = get_model_dict(provider) + if provider['provider'] == provider_name: + models_list.extend(list(model_dict.keys())) + # print("models_list", models_list) # print("model_name", model_name) # print("model_name_split", model_name_split) @@ -992,29 +1030,33 @@ def get_provider_list(provider_rules, config, request_model): provider_list = [] # print("provider_rules", provider_rules) for item in provider_rules: - for provider in config['providers']: - model_dict = get_model_dict(provider) - model_name_split = "/".join(item.split("/")[1:]) - if "/" in item and provider['provider'] == item.split("/")[0] and model_name_split in model_dict.keys(): - new_provider = copy.deepcopy(provider) - # old: new - # print("item", item) - # print("model_dict", model_dict) - # print("model_name_split", model_name_split) - # print("request_model", request_model) - new_provider["model"] = [{model_dict[model_name_split]: request_model}] - if request_model in model_dict.keys() and model_name_split == request_model: - provider_list.append(new_provider) - - elif request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*")): - provider_list.append(new_provider) + provider_name = item.split("/")[0] + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + provider_list.append({"provider": provider_name, "base_url": "http://127.0.0.1:8000/v1/chat/completions", "model": [{request_model: request_model}], "engine": "gpt", "tools": True}) + else: + for provider in config['providers']: + model_dict = get_model_dict(provider) + model_name_split = "/".join(item.split("/")[1:]) + if "/" in item and provider['provider'] == provider_name and model_name_split in model_dict.keys(): + new_provider = copy.deepcopy(provider) + # old: new + # print("item", item) + # print("model_dict", model_dict) + # print("model_name_split", model_name_split) + # print("request_model", request_model) + new_provider["model"] = [{model_dict[model_name_split]: request_model}] + if request_model in model_dict.keys() and model_name_split == request_model: + provider_list.append(new_provider) + + elif request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*")): + provider_list.append(new_provider) return provider_list -def get_matching_providers(request_model, config, api_index): +async def get_matching_providers(request_model, config, api_index): provider_rules = [] for model_rule in config['api_keys'][api_index]['model']: - provider_rules.extend(get_provider_rules(model_rule, config, request_model)) + provider_rules.extend(await get_provider_rules(model_rule, config, request_model)) provider_list = get_provider_list(provider_rules, config, request_model) @@ -1022,7 +1064,7 @@ def get_matching_providers(request_model, config, api_index): return provider_list async def get_right_order_providers(request_model, config, api_index, scheduling_algorithm): - matching_providers = get_matching_providers(request_model, config, api_index) + matching_providers = await get_matching_providers(request_model, config, api_index) if not matching_providers: raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") @@ -1046,7 +1088,7 @@ async def get_right_order_providers(request_model, config, api_index, scheduling weight_keys = set(weights.keys()) provider_rules = [] for model_rule in weight_keys: - provider_rules.extend(get_provider_rules(model_rule, config, request_model)) + provider_rules.extend(await get_provider_rules(model_rule, config, request_model)) provider_list = get_provider_list(provider_rules, config, request_model) weight_keys = set([provider['provider'] + "/" + request_model for provider in provider_list]) # print("all_providers", all_providers) diff --git a/request.py b/request.py index 2949e752..d6e19585 100644 --- a/request.py +++ b/request.py @@ -619,6 +619,9 @@ async def get_gpt_payload(request, engine, provider): model = model_dict[request.model] if provider.get("api"): headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + elif provider['provider'].startswith("sk-"): + headers['Authorization'] = f"Bearer {provider['provider']}" + url = provider['base_url'] messages = [] From c5104a1d574390e88053e3378c87895e533160ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 03:43:05 +0000 Subject: [PATCH 323/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.116?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f64402b5..30f8347b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.115 +0.0.116 From 96efe10e4fe5ae23645685f057e5090261b8878b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 12:08:32 +0800 Subject: [PATCH 324/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20of=20httpx=20timeout=20being=20too=20long=20after=20hot?= =?UTF-8?q?=20reloading=20the=20configuration=20file.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where v1/models cannot obtain the API key channel model. --- main.py | 7 +++--- utils.py | 68 +++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/main.py b/main.py index 64109671..4db45d30 100644 --- a/main.py +++ b/main.py @@ -775,7 +775,7 @@ async def ensure_config(request: Request, call_next): } # 发送GET请求获取模型列表 base_url = "http://127.0.0.1:8000/v1/models" - async with app.state.client_manager.get_client(100, base_url) as client: + async with app.state.client_manager.get_client(1, base_url) as client: response = await client.get( base_url, headers=headers @@ -786,7 +786,8 @@ async def ensure_config(request: Request, call_next): for model in models_data.get("data", []): models_list.append(model["id"]) except Exception as e: - logger.error(f"获取模型列表失败: {str(e)}") + if str(e): + logger.error(f"获取模型列表失败: {str(e)}") app.state.models_list[provider_name] = models_list return await call_next(request) @@ -1280,7 +1281,7 @@ async def options_handler(): @app.get("/v1/models") async def list_models(api_index: int = Depends(verify_api_key)): - models = post_all_models(api_index, app.state.config) + models = post_all_models(api_index, app.state.config, app.state.api_list, app.state.models_list) return JSONResponse(content={ "object": "list", "data": models diff --git a/utils.py b/utils.py index f2173a23..2a5e6059 100644 --- a/utils.py +++ b/utils.py @@ -479,7 +479,7 @@ async def new_generator(): except StopAsyncIteration: raise HTTPException(status_code=400, detail="data: {'error': 'No data returned'}") -def post_all_models(api_index, config): +def post_all_models(api_index, config, api_list, models_list): all_models = [] unique_models = set() @@ -493,11 +493,8 @@ def post_all_models(api_index, config): provider = model.split("/")[0] model = model.split("/")[1] if model == "*": - for provider_item in config["providers"]: - if provider_item['provider'] != provider: - continue - model_dict = get_model_dict(provider_item) - for model_item in model_dict.keys(): + if provider.startswith("sk-") and provider in api_list: + for model_item in models_list[provider]: if model_item not in unique_models: unique_models.add(model_item) model_info = { @@ -505,24 +502,53 @@ def post_all_models(api_index, config): "object": "model", "created": 1720524448858, "owned_by": "uni-api" - # "owned_by": provider_item['provider'] } all_models.append(model_info) + else: + for provider_item in config["providers"]: + if provider_item['provider'] != provider: + continue + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys(): + if model_item not in unique_models: + unique_models.add(model_item) + model_info = { + "id": model_item, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + # "owned_by": provider_item['provider'] + } + all_models.append(model_info) else: - for provider_item in config["providers"]: - if provider_item['provider'] != provider: - continue - model_dict = get_model_dict(provider_item) - for model_item in model_dict.keys() : - if model_item not in unique_models and model_item == model: - unique_models.add(model_item) - model_info = { - "id": model_item, - "object": "model", - "created": 1720524448858, - "owned_by": "uni-api" - } - all_models.append(model_info) + if provider.startswith("sk-") and provider in api_list: + if model in models_list[provider] and model not in unique_models: + unique_models.add(model) + model_info = { + "id": model, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + } + all_models.append(model_info) + else: + for provider_item in config["providers"]: + if provider_item['provider'] != provider: + continue + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys(): + if model_item not in unique_models and model_item == model: + unique_models.add(model_item) + model_info = { + "id": model_item, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + } + all_models.append(model_info) + continue + + if model.startswith("sk-") and model in api_list: continue if model not in unique_models: From 5137bbe625691efdc2a3f6445eafd3fb211af657 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 04:09:04 +0000 Subject: [PATCH 325/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.117?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 30f8347b..cf648a75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.116 +0.0.117 From e5641e48a0dd75f3a40e80be36bac64bb619e55a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 12:13:21 +0800 Subject: [PATCH 326/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20channel=20is=20forcibly=20set=20to=20gpt=20whe?= =?UTF-8?q?n=20inheriting=20the=20API=20key.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 4db45d30..36e712da 100644 --- a/main.py +++ b/main.py @@ -1033,7 +1033,7 @@ def get_provider_list(provider_rules, config, request_model): for item in provider_rules: provider_name = item.split("/")[0] if provider_name.startswith("sk-") and provider_name in app.state.api_list: - provider_list.append({"provider": provider_name, "base_url": "http://127.0.0.1:8000/v1/chat/completions", "model": [{request_model: request_model}], "engine": "gpt", "tools": True}) + provider_list.append({"provider": provider_name, "base_url": "http://127.0.0.1:8000/v1/chat/completions", "model": [{request_model: request_model}], "tools": True}) else: for provider in config['providers']: model_dict = get_model_dict(provider) From 6114a0426bb16a0ffb6da66e8bccde0c6ba244ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 04:14:02 +0000 Subject: [PATCH 327/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.118?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cf648a75..424a70f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.117 +0.0.118 From d5eb66c008193a8d13959b60133305a3e4018521 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 23:04:42 +0800 Subject: [PATCH 328/476] =?UTF-8?q?=F0=9F=AA=9E=20Frontend:=20Add=20uni-ap?= =?UTF-8?q?i=20logo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add support for felo reverse API --- .gitignore | 2 +- json_str/Vertex/text.json | 30 -------- json_str/claude/request.json | 72 ------------------ json_str/claude/tool_use.json | 47 ------------ json_str/claude/tools.json | 95 ------------------------ json_str/gemini/request.json | 52 ------------- json_str/gpt/mess_sse.json | 12 --- json_str/gpt/tool_use.json | 8 -- json_str/gpt/tools.json | 91 ----------------------- main.py | 10 +++ request.py | 6 +- static/apple-touch-icon-precomposed.png | Bin 0 -> 43853 bytes static/apple-touch-icon.png | Bin 0 -> 43853 bytes static/favicon.ico | Bin 0 -> 43853 bytes 14 files changed, 16 insertions(+), 409 deletions(-) delete mode 100644 json_str/Vertex/text.json delete mode 100644 json_str/claude/request.json delete mode 100644 json_str/claude/tool_use.json delete mode 100644 json_str/claude/tools.json delete mode 100644 json_str/gemini/request.json delete mode 100644 json_str/gpt/mess_sse.json delete mode 100644 json_str/gpt/tool_use.json delete mode 100644 json_str/gpt/tools.json create mode 100644 static/apple-touch-icon-precomposed.png create mode 100644 static/apple-touch-icon.png create mode 100644 static/favicon.ico diff --git a/.gitignore b/.gitignore index 66e5bb64..5adf2c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ node_modules *.jpg *.json !vercel.json -*.png +# *.png *.db .aider* .idea diff --git a/json_str/Vertex/text.json b/json_str/Vertex/text.json deleted file mode 100644 index 33c4e402..00000000 --- a/json_str/Vertex/text.json +++ /dev/null @@ -1,30 +0,0 @@ -"contents": [ - { - "role": string, - "parts": [ - { - // Union field data can be only one of the following: - "text": string, - "inlineData": { - "mimeType": string, - "data": string - }, - "fileData": { - "mimeType": string, - "fileUri": string - }, - // End of list of possible types for union field data. - "videoMetadata": { - "startOffset": { - "seconds": integer, - "nanos": integer - }, - "endOffset": { - "seconds": integer, - "nanos": integer - } - } - } - ] - } - ], \ No newline at end of file diff --git a/json_str/claude/request.json b/json_str/claude/request.json deleted file mode 100644 index dd99c88e..00000000 --- a/json_str/claude/request.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "hi" - } - ] - } - ], - "temperature": 0.5, - "top_p": 0.7, - "max_tokens": 4096, - "stream": true, - "system": "You are Claude, a large language model trained by Anthropic. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Respond conversationally in English.", - "tools": [ - { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - }, - { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "input_schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - }, - { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - ], - "tool_choice": { - "type": "auto" - } -} \ No newline at end of file diff --git a/json_str/claude/tool_use.json b/json_str/claude/tool_use.json deleted file mode 100644 index aecfc326..00000000 --- a/json_str/claude/tool_use.json +++ /dev/null @@ -1,47 +0,0 @@ -data: {"type":"message_start","message":{"id":"msg_01Jp7JVrr2MFfTzUBL9hrgoH","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":558,"output_tokens":1}} } -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } -data: {"type": "ping"} -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" apolog"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ize, but I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" need to"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" respon"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d in"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" English as that"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" the language I've"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" been instruct"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ed to use."} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Let"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" me"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" r"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ephrase your"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" request"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" an"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d procee"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d with searching"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for today"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news."} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n\nTo"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" search for today"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news, I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" use the Google search"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" function"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":". Here"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" how"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I'll do that"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":"} } -data: {"type":"content_block_stop","index":0 } -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01M17un8HfqkS3uDKBPuBr35","name":"get_search_results","input":{}} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"promp"} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"t\""} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":": \"toda"} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"y's "} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"top news\"}"} } -data: {"type":"content_block_stop","index":1 } -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":124} } -data: {"type":"message_stop" } \ No newline at end of file diff --git a/json_str/claude/tools.json b/json_str/claude/tools.json deleted file mode 100644 index 1864f113..00000000 --- a/json_str/claude/tools.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "搜索今天的新闻" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "toolu_01RofFmKHUKsEaZvqESG5Hwz", - "name": "get_search_results", - "input": { - "prompt": "latest news today" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01RofFmKHUKsEaZvqESG5Hwz", - "content": "latest news today" - } - ] - } - ], - "temperature": 0.5, - "top_p": 0.7, - "max_tokens": 4096, - "stream": true, - "system": "You are Claude, a large language model trained by Anthropic. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Respond conversationally in English.", - "tools": [ - { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - }, - { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "input_schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - }, - { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - ], - "tool_choice": { - "type": "auto" - } -} \ No newline at end of file diff --git a/json_str/gemini/request.json b/json_str/gemini/request.json deleted file mode 100644 index 715a4e3c..00000000 --- a/json_str/gemini/request.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "hi" - } - ] - }, - { - "role": "model", - "parts": [ - { - "text": "Hi! \n\nHow are you today? What can I do for you? \n" - } - ] - }, - { - "role": "user", - "parts": [ - { - "text": "怎么解决" - }, - { - "inlineData": { - "mimeType": "image/jpeg", - "data": "/9j/***" - } - } - ] - } - ], - "safetySettings": [ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE" - } - ] -} \ No newline at end of file diff --git a/json_str/gpt/mess_sse.json b/json_str/gpt/mess_sse.json deleted file mode 100644 index a782b58e..00000000 --- a/json_str/gpt/mess_sse.json +++ /dev/null @@ -1,12 +0,0 @@ -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[],"usage":{"prompt_tokens":178,"completion_tokens":10,"total_tokens":188}} \ No newline at end of file diff --git a/json_str/gpt/tool_use.json b/json_str/gpt/tool_use.json deleted file mode 100644 index 25445f93..00000000 --- a/json_str/gpt/tool_use.json +++ /dev/null @@ -1,8 +0,0 @@ -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_hbFDbIHYbimw1J0v9d1qvpgl","type":"function","function":{"name":"get_search_results","arguments":""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"prompt"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"today"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"'s"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" news"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]} \ No newline at end of file diff --git a/json_str/gpt/tools.json b/json_str/gpt/tools.json deleted file mode 100644 index 1b5aa963..00000000 --- a/json_str/gpt/tools.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "model": "gpt-4o", - "messages": [ - { - "role": "system", - "content": "You are ChatGPT, a large language model trained by OpenAI. Respond conversationally in English. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Knowledge cutoff: 2023-12. Current date: [ 2024-07-09 ]" - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "搜索今天的新闻" - } - ] - }, - { - "role": "function", - "name": "get_search_results", - "content": "latest news today" - } - ], - "max_tokens": 4096, - "stream": true, - "temperature": 0.5, - "top_p": 1.0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "n": 1, - "user": "function", - "tools": [ - { - "type": "function", - "function": { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - } - }, - { - "type": "function", - "function": { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - } - }, - { - "type": "function", - "function": { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - } - ], - "tool_choice": "auto" -} \ No newline at end of file diff --git a/main.py b/main.py index 36e712da..c39a1939 100644 --- a/main.py +++ b/main.py @@ -2132,6 +2132,16 @@ async def delete_row(row_id: str): # import asgi # return await asgi.fetch(app, request, env) +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse + +# 添加静态文件挂载 +app.mount("/", StaticFiles(directory="./static", html=True), name="static") + +@app.get('/favicon.ico', include_in_schema=False) +async def favicon(): + return FileResponse('favicon.ico') + if __name__ == '__main__': import uvicorn uvicorn.run( diff --git a/request.py b/request.py index d6e19585..4ebe9724 100644 --- a/request.py +++ b/request.py @@ -618,7 +618,11 @@ async def get_gpt_payload(request, engine, provider): model_dict = get_model_dict(provider) model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + if provider['base_url'] == "https://api-ext.felo.ai/one-ai/completions": + headers['Authorization'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" + else: + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + elif provider['provider'].startswith("sk-"): headers['Authorization'] = f"Bearer {provider['provider']}" diff --git a/static/apple-touch-icon-precomposed.png b/static/apple-touch-icon-precomposed.png new file mode 100644 index 0000000000000000000000000000000000000000..54a5336357ae91659ba3f8ab5a0394889cf1670d GIT binary patch literal 43853 zcmeFa`9IX}_douaY1AMYOSCBSEM;lYPT8hXRJM>U(V}E2d$MF^+GwR!*%J|>>_j9* z8zuXmEFnwweZJ>s)6)_j4YP=M^mt6&|idTo{J&>`_(L z#xN56lZ26H!~fBjthxaIM{v+q*@49th_qrD72Bh{UH7cXuNuw=_nIDxk4aSs9=DS; zz7j7%Rwf^%5S|{rLOsGa_ce$1ipytis}+XH)%j|qZ@jq2xoBAN<+pF&-j^%K@N04I zH@mgVa9+O9GJY+Zme71Vg1**W^(P;5H%a9`jT#Jats5LJaq8d9-Q^NAI#J_YR?@-> z*ml66-V{{H!$|blQ(6X<4>(?7v}JqoE_@^-aBupyn*A-FOL%DE!kGw_~p^g z8}meDzjVVtB9_RXeg69Og*oj@wmC+2KHBNoCL({f`|eK9d4cw@GzT^l7TIba)}4QA zIftmn(2>_)AFCXzNogQcs5Az~tmYIH5)ulq{~*cX)K>g-Ecmfgd+D0UgDIU~UHjY4 z8Dx2k^R|~nJBOKMx_Y*k#nB@@#z*F~l|&DET$s%>P!_FkW18vOU;9))uGYW5y{t|r z<#dfc{4gsTR>-xLK6`m8=OSlZW31&W0jib`GskajeLsn;Yg#{0Ws}u&fmboL0siNt zLwz<=s4I&#aS8s%@#eC%CiLxO&EIj6fs-^*(c$vw0Q? zZn7w9RQ>VphLY@$>lYxqk!?pJVK+ILB3#!O%fxo>lg=!*Voi+eXAai7b=1M(_q$$=c5-xO^cEP}yc$07 zM!{{M*hwJx4Z*;VeZJQ{T9Y{M)vH(M1v`}*j{0x1=xJVP9`L)TH6r*GP@!2@{`|-r z@wMqcL&JaMc`J!ThtA(g-9;oa1bK8;C5f*)U7b22ozb1+G&!|x{ z4k~W*7#q6qzEwG#)F4U4yjqNP=O1In$HzC%;a&QRRqNJuD8ad_b^vbFEPSS`_FU;l z=`3UEcgI)V_;ZE&dUtb4RX0d*L9$KVD-)Zn@iP%JwP%0a);Do!`h?iFM?+X@1?USdcPi%+&bjR@Ly4m4%-TiLUlWaZ^R{=2VIl6(@YR#^>B zjCS#!YYB9A&G`LpaR+RCulG!6bp>@rhsA|Ilv^^JKl_Wyvc`H%6-Iw0$ZWPKG8?vt z2;A~GBg0)M%D+^EDMcn@QH-XqeVo@9ZfxGy1=4;Fv_5VLQgEw$)mOByC1YZw{q*2B z8OtSn$UF!{0#|@T!B!O{#t0 za1g1^!8(_gz}lD2U7g(x*QA9qf#llmIZj-g_OW?zQOb6p!d!GXNBGtA2*ZREi~TlP z#1DbS1UI;mb|~;18CZjn9Xhu=DOeUq-a;zq18g)-a%d@Nmd@V=u3n}2agZ>Jpwl6lHFkMRgkOyhRHbCX{Y-&<`js{{ z>HVK(&6!7vdUOi6C*%#RNnkNrU(%i0Qwo$=V>eph^2gNuw`8`&zU-{L_t{EeCsh(8 zAEvf9FnE=gk&$c3t8=5fL2qB0wG;$C3!+a9>)e`T9T;h3Z1xIYJLTWpgPsbLui`J-6z{^QJ|qN0YBIuEyEtGLWktI>fD;jDp5E2roTFJatI z;U+J8B|EJp$Sw8w$NSO&Zk=iS{I{IimKGX%lyehtk01pTv$G_WxsP_F=zAQbS9I3A zbURg*+|f|;(s`7a#VEW`$~n2$u#kk5)V3nHrm){;tK`JU$jC7;K$H7ji+-tuq{9W< zKDZpJCB3`8@Z0y#GIbfp20LrUlTX)F7OB%~f1Xli3z9@F=Rp3EB zy>%PxbXq%0k=IE4kAdzA*>h*5N>%cy(VR=dCPM~yR;o|vySNNLN$h*#ZN_!>@x4!J z;h}B*r_)=d2YyvJv=4Mt$${l(9+C1ZBK;-h2RYHY!k+%|{+cq-OLdoe=A4No{LD%U z1uMSnn(cP~-o4FrU?hX4uC3urkYxJVx>x544rjM)Z%6JOUFQpF(Pm4>#6vI7NEA9apI4#Nn{a~ z5KsiSgNY|!G=M@6DBUiZ(m$x zf_Y&bV#?qXb$`Fv1&_}Cv2+6Q7ALMMLTijxua%M-$%9BHzU*N6j_q7z?7WEECUx(n z3h8ErK?-$MU?)isc&ZnEjj^2ncL$Q<8r#-z-&g2)9{9%2fK3+KmnqCPA8NJ6zdA@4B+%-ro+kL$!I}i(5y*D$<}FEAkuvWa$Om832AB5h5k8${QE$r ztMmB1Ly`F(0n7h^oW2LBhJ~Pal1Sj|aIAg#EB+iX9$pGlVn+9SqdRzFq2bYWk%5Tr~?nB zfrM9K0`Y}6WNL>Bt5R%4)6R5M_6lgnZ}_X$l%cyc@B^k674c2Qb41;{XPV`4&KCvryJCZ(Xa4e^q__%&F?#unV|klRhjN5HO!n_z zAGkji_)jxueYWxFjrTS67-rQaTIgggkvkJ0Byl16FZHD5TEh{us*d$kp9in@T zx0@<0R{ODNJb{5xv)|bMxFIvq0RH}^3uGRjiPokc|FZ79(9iN0KTz7%H-T>ca{EqF zc?L!ec~ocraZL)kIK4ktKr>Ovww^`{w7NIt?#DP#j2j-u7E8YxYM{SDiQXfD%Wx=T ztM?MQ&l$!7n2$#ChNWjfX%ENlWk=^lNHt%H#CZ`JeQ zOgNXCYze1#SecGiK&GY-0VCZZ!KHTNl=T)O|DE2K^lNw#2OY%kur4n@(2`o%CIMNQz~Y?Y&A$s(E)(tW(4C z`Wr{wPw*M8fUG}r^r3msa3v&zi&T;$FR>jd?v=?qLO|Euq_f6)>IxpL|1m&aG%s=w zfp+z>f@|N;gU-Loe_yz_NSTIs2LgV%3GUJ4&yRHc-L9$zP z*S+&pTs0>dvp4roxtDiivGWJfiEht5i?>*(9)SSP)0_O_#+2h#lV}hN2AzL?`Tkw{ zTuSPX$;c%lO67xAQ&9l$f)0H~f2D=0z@D4;dQReXwOoR0#|GLh!>cTh6aJ zi1|GK%DZ^WFH@OQm7824r%~tEOSk{4(iGJzG@zr-qAa#1to2(2>-b(Hu23EVZO1$L z6Q3SPBei6uLs``c{Mndt@ss+)znW1b^(EEr$79op9$$T9_up^nTkRThmBNw=H}yB& zFe5&3>HRE2w}C_JARKmZa7yT3`Ma1*68nHz`(X6sVo-dlBSvgvUYsFL!c22GI!43TCOtj6RBbx z)PmAikGSkz_V-vuHYdlnGrv+>dOk|C>Oi%|IiH+G3pab8|KL3qcj)E7Ww>R@HA9OJ z|4dm%(<2I-oc;c&AW%-}mp(DR;X~C7Z?a=+(JG~}?l%u~!{<&> zBC@V#h(_SHds+P5jVRd)gn(Cfz-E5jRc*+R!cS~2OK-Y=@KvTuZ?mZ4aO0g&ipFBEa%+m-tC+D5?%Z#_DzO2(#E{`6*_v>?L>w=&zXFmf9^oA-e?bjrN1XJ{L~cxK&4nQxuG7`0{8Z$AYiM5V+^3@; zf;FC82Rb#5LO=abntfCF*2H}UM|61j;nD2Z7v6rPr+;KDC{S-$)F&~RmvRaPeCir61D=Gb_>R_k)KggR-RrbAs!mTi1qj|wP zT8S_P)2)j=oXsFO_^viAVvP^hwHVA&k^TcGpmo{rNg8q;iI3mDecRje+248meNcZ_ zo^bcOBtP-Tbk+p7u(U~aOa1E$CC0BmPE|Nsh!nBEtz^~DZxMx|!@Vto&px!vkE(!O z&ezNVU)&aJoM!($=MuNu5(U@9t8oLeQ{5lASUKeR^XDN@`s|BKYxaAEKpx{eNju!0 zVIf=sqH<@f-*i-^D2<4AO}OcPKfZJ-%^1^*V`QM?Wd>MfnL0A&+y8%Hy?85v8xoSoEd>8q;bz?0!8lf^Bmk6!-Y8?mrs>@ zzAp{%JH5n{Uw4pwI8w2CG1Yrny;(=!UcK%;Zf@r{wV=S5MwG}gCkAlecSwU#m?w+{ zG};cdm)X3By;oL&2B?@|eQ$+DPziG%-LVdZ= zq50Fo`yuhY+EFs8!AF-ObTm(tKK2CQhS9qjYS}BNa*RrON&}~*lmY02=mW5fXSdb{ zbiXBGd2_ojgSA)<_cWz+RU|}cWN=L3Q zSl`eBiO5J7c=?zvTU^)waFW);;U{C`@%&)QY1RfPBf7-1{QaD%n1qDHg01$AgJm&> zy^kSP4>EDkUNwECG07<9^tcIFr^g0J--GkQpHxOIAHwSIA3U9QswUmZ8P%hoDi@tU zHsxuMvk{%6Vx9%lMe&5%u{cXDSCl->Jn8?!6RB$Q;M^fm`iqut|;wA~ilKDTY z(|T9w8{54-1z=8Z^vm5vroz^kmY!beX~^w1>ctwnSQL3idQ24sMK+D?Cm|?&f!bYN zoWAXxCTSsd_x}B$k+I%FY14SK_xD;CKfO84IO~;(V9}Z;_U~6HNJ&X4lGP>iu@JCl z?}A_@)~ZK52eYOLPD|@c7M+#XTo_D0bW-Rw2e$L_+=rm~p$F^0$JV>@cG$N2PBo`I z0fZRe9BBP!{sHns6;Cn2V*h%+<-5lm-(HDN4U{1z3q-(C;42)eoLVo9$8NmQ$6#GTWk}CtN?=m2~#37gWY< zt4^i#p9NrgsR6*PyZ)JbzZ;S}C~}(*UTsvoH-aH_fCsADc>C6( z@FjQTR^I!IRDeUI#gZV$OnL>`ewiBbEwKY$Z_=OgVzcKiSm#`u;WDd%@oe_AYk5Uv z_m`O}*+76O%S%W<8MW65TW4MV-1NkThjO)WJDt$)@lz`TKyvrvImz>|u*O(Cskt$J zVBl^gu71M<&>*0{LZRr}Hu-Z?vkC^ykqG+i5oc8vTbBju=drLR>bIffJp#?bqO}CY zzMYzoe{uYNq{sQkN^r5|9;*MaD$afQ8q{9hD0zFESsGR9`vD6H3o}z+c_%GCYC+Jn zfI=XySg}GAuCUhPj_M(7-N`Q@bs%Z;Tu)Jt2g!!5o>SF<)I+uXAQ>Tuf>hnTwoY5z_WLE_rEL`nEt*d>0!Qv7Zu0fR4u5*E z{v6~wz1buE?bnJnh@6{ly&as6zsl`2LBOXGU>I5%y(~RYM*!<Y9q=jM`S|qI`(Kp!0^sLkzyd~7@Y;%h zRI|~)1)3ZGfOAcEM&;OTuh)TSa%LwOZsNcnw872$g!U+NT6AOIdv$G1Ygt%-#wLCq-8!~rW~ru z*cU(zZ-76y63!p~x^kMZ#Z*Jg3xoVYi%JzcSL7*T3!qV11HWJT;~?vY$K7I;DI_!3 zaY(VGvO~6CSc&5evfR1bFo`v5GCfM74)B8e3bza3E16PZ0r^L`0Y*Dv1BvzNsL03% zi}kM<%)y}epBQg=doAwjs-W~~L4uK;M}(&yM}IfGHN6?FYCVe)cdo-q$zO>+F*w*! zC0w=W$2MK3=?-Xu14or1lqj(}Z0MsG#=1OumK+;yb_KtBiq0CXlScJ&nz+qtfoV2H zqb?-|ZgX<&_-Hy||NA1Rz?Qtnp{$r?_X)^*vhu6b9Ok^^*{3jlQN0-vfzts{pL_J1 z6*i659TUdz5&;sqU-|$-9lhXwtlWE=@W{ow_1u7}F`+Z%(GEfSWyBM!UBe^g92lTA z<^5j~PZKC5 z@-_|nxLnXl?UUwJ9IAJY?#mHT6!FLE{p-e>k7u#QyL_LSX1y|;H=49*`qKPsw7i>3 zoaw_KO*@C%O<5xq8P;cSs~;7>kPmH!5^i=LVHeMIZcw?H%vo~^{2;&gktmI7UG##C z;aDRiRyt4$+Zm}Z?s1)}mlP^RmiIBj8v=Qc0H#|BVKIxO_AZuyrpixT2T8_^1zzf-WjBI5V7^j&QdKZd~dHxQ{s>!4Hp@IP0;)WP>>fWMo_0UE_KT$`mwWPtn>H<=d%92q`c z>M@;#A;}!Gp-q1d>i0Ljhg523F~C1tL0-5OHG*#R)WnQWw_n;RZsRi*$4AqlEu95@ z+T)5=@7*w{2N+wyp}$lNR7xdNS*RH#cp@7HmAB7Ob{%BkCIR^j3sr z&Nv}}WRzr?lm&_T1=R4}H5|1njz9yciUxw0(C;Y;I5%U3F~`VY7pr`1xHs;j{y7(J z3^g)PMA!&X%fslEUjpxROYn-OLF=xzt0cP22Qm|m;mGwi1V$Ch{rb11(-`C<#&q}K zqDm98e|~+u@4mFDVqORR_RU8|(?mrr!}}tL_ca+g4X~!0zOJtPt!M?Qvd=Eq zI@|he&t}MqR{N?OOpnrJF~h^C$ZCP7^1cdMyXet$aKJdGAT6iW9E%*>vX$8rBn%UUUZ_&okhyV@@^(O+6`!a0_J!!>1 zJwk8dsy_SU2CJ|B0p6=u|ITesp||*br7Ke`TCLKxskcS z@-98UB;0PuV+&B_3}rEoSG=m#v?zx$eMl(Zw}E`WOr^phj6j3-Z|{LGRLdiH~e9djw{1Ci8MFk@tf5@Mc5N;Hd&xD(fi6H^YqA*ynTpJYJDLYZ1u&Uv z+c#MXfTh&M7CbT8IVH11+|WIA80Oou+#zbd(&^V164aH$P=b7a{wh6Vy+HNMX;3Cy zEHWMM>7WLae^geyA0fLVrLNWiQlo38kn*ROR1{lHS8$9O#f|1nJm>H%Dn^jMiC&4m ze!BpI7ShbYX&ygN^JzvB!JvI6LB*i02(VLCRQlc|A_KaBrw%4r9{3n_x_K&SjSdSr zK?J0x>J@;gdIt`Le?V2R#cxO?g*#!UM&p1?ulx+rYi(qPhK4SO$(>&ZpA2qHd(Lb|N8MnU-a;^=%c#f z2ovgy0tfS~uY+~qpMllWiSg14>4_tNm#Ozd;{)-B()JRvA<{%&lGsnEn=b=ul2mxcu5h{wBM{U0AWr7JgOKB#f2KcX7DKKx^+yh1 zwKnG_hFdg0|L2m_#@qW8f7WHu+X0NU(%&LywgkHlz`6=*GzC|zZd*80YX^zTRc?-n zqUU?I$Q@sT^>tJ^S0LoqEFsc-rczXM_W>$8j8M@N2|A?&8YCWdF#qx6;eYRq&^9ZW zP&7ZDprc2~hAMV9f{q}KnrD{qp$R+;m z*z^J%CLv9kkoRc@0$iysUbQ(I3jsLsp#C$Xk~Xtz!_G5rn}w@fGN@dscvb&oVBp{u zvOWjX9BskY>sf#1lKwc?M*aHGYiqZa1KwK-EfP5;o0dDXG1PN-4dt4LUtLef^fHb_ zHd|%;OlCckl_s?B*zx1XJ5Ce^jTg7aWpxg+ zqN6%zxCkF}CYlDzhcY@ATG4fm8P7Bq;M98}>OJMltmI}=?}zJi72pyRiC~co!mQ>3 zm^Y$r9_-sN`xB#016qtu%@mWaP6%K?(lI8PBe%g8p`o@V z02cZqJW1=bLgw^t1(}kcfFnxCKg2+l@{841nm0ZV2S>J96otKaiT!qFhNq(OaO{Qx z*0LyUk4`)n2JNZKrU*_?DYMm_-Yo*P=vi(1hDWA)y!#C=eIsZ+K#^C}R(@MixVds# z+YCAXM>NRA{M1%D8~_uXuW$UK1r3}=Ty}HaI#a>SnW%T!9}(ml^>flXRF=A!tN^j=-X?;^yZI5%hVL#=>;W$=)(WA z=H;LODvWztW@8AiaDoui()@H~)%5g=K;+~8R~?-pR)Y`gr^6Ud)+qGl%dVZZ`8d-b zyYQ#Nw9dS9z5aFb#~}Tlqg2}?U`jMFGTIgibC`NTZ7+CcvU52T93BsNvYuZ**iNGr zISrr3nQJBoV*e$paSxu3z%XDXulHiim_FX0025B5CGsUl7td5&6&zMs4t4Mn$7(DD ziY0$2s61S54kXVAQ&b;u=28V$`2;{s&gAi6fLV3Fg8`p{4?1g-XQZhldof7jJ!NF9 z%WVhDi440R?@BVkdDYTk(4npQq|L|a)dga>>|nJ6Zyn6oM%)lpny_uUPoG$Yfw)>X zLOyfJCF;$VnIgT(f#xfm8*;B)Q{=;{9lxp5d$VWpj;+pfI~X**-lgQ<$yS%C>ANsz z>DhWwB|cP6Ircy#KmA#U=YOeRy$(ut$&DvIE$F|3g+KfWE=L9^DJm`ts1gV~TM9aSSO#01(3LjPn+o+G^SVBp1!V44$sd zNP90%WSscyx1=)DEk%j&AmX3$897;_Q3#+Bo{$zF>MF$ZF^?vIkjq3ATsM`u9h?E` z04HcijvRRn)na-z^mH_Z^g3Oj@p51N;fAIl`3{@^Q4vjI3N$ZH={!-BWJzGD5F%K= zU(XR8^9UM9{vSORXD+tM_}zl4$U1jHZ0@2>KV=ZIc`G9G)Xd})6cxs5z9Glm?@h#! zZqroT?<#cRFEgM4s?Sm;q_(jJ2ndItLVMN55XO#^YeuI}Sf=3}(LZR#o>71);x>51WOQDB@Fox#(PK7{&_XfVB zdHmN?#z=8p3)_beX;mY#(Zd8=G_kxAFgwY>HEq+zMBwhIF#4Gj2QG&W4rQ#_Ty|!>bmVde zf5h7U9VL}bk35%-*tA;j*PqEZ^i=;lM95BmxZvmi*ak^+B4=WJv=E?S50+zcxkvk26YQZ3ubJMah1<0XqB()692ig^XoUuH#?EaAi|5a9c4!*vw8SqcgOgFA zn>(}E48tR66*ol8u^!ZA)il4BK6Nk(u@Oxf>2t~baZ)+|u?c`CHbTjOly{*)zwm@z zhh}I8wHPl9KL@TuX%OP4Nj+{+s6s-f?lGj-?^k*ftS)7Z|Nz9WrFR3uRHLz1S$ zM@P8HSWZw@=QMnXN`avg>V?wj&6JC4V65M5B7IuVl1YQvVdh*T|7mQZB*S>>ui^IT zv4O}(!AKiUo|+bsXyq_$^SmX_ZQ2#d$}r^i{}cO{v(&xJhK2?uY5ie3z_$9Yq2;Rd z;w|7OTvDvONgqz(1+mFDhj8Jnq_{D?>-xOmb+P(#tlsRBH^lnKDjs7$M1z0r|5GwG zYSA#CoyPm{tc;XljUbCLAE(bs&34D!^40Nx*kf9$nMb?d_Fmh)E!JDVi8QX`yIMSYf|Z3 zrPLD~CmK9E-!~CQwyBPAji2H)7QS2k0;YS`Fzaml2TkJ8HNwqhxM#?J1`3w)N{_9J zSNI~I$*^`Xaz*$qA?2dwaKLc0Xh;vP)ign3(s;8enetZf8QV80n!?oiMLbe4vpITx zU!qs=j~>mo4wq+_5K(KXSY-M`efBSOIdwg%=*P(vUr|%81-^SCalcXoFS)2zDfhEO zZ4{N_>pekFH#6ljxb%s|}lQetw7egDkLTmIV$V=mF?zeZoU8eN1m|mWg!o?7h?K>pRqiBl7oE>GBd1Zf*a$vixI@#5hq+y?4-e7 zHh5k#U$w!X^}%F(1*1))x*gvk+Z@Q$;J}4Xrco_K$605^hzx9Agb)%kyqbXqYpd9! z4#hWV+BU3n#E>%u*AC&2GvRSxfu1>ArIs_pYTbeLPkov%28xL+9iNVDux-M$!Mk#c-3OAm+EX?? zg`G)U#J)LtH;b6QhJABgqQ$H6xJ~?+vXgcU+bX%ZKxYWJ;8Y`Nm33pFa<1#{;nTD) z=7Y%2Y~o^1mj_Gxlmgq6t%9*`Q}zlk4Y*%m_&qj>wWJ-XY$>+Y9pEQ70o%30k9|LaeonR3CJ?gsT#;udCJoCe zMgDXG&AlLSFFK661h#hkv|kwgl;TB`K)N3AinPw>0{gi9YoL@xr7?Xtc4UkTV;9{U`f=?`1BN#4 zDDh(}>JAif&a15y_|UKBhF5Ss(S_Hw0%qY{pd4dTz}J-94b5jP5Y-6B+1db#ab@wt z^SIRtGQS`*rry01tBIE6`@-I)MFZ@M9}pyuBeuSs^}~+MR(=A3v7B8k9F?daV0rKS zy$V||AAryDL+_f5@yl(ygqx8ON5(L#@I4n+izq0ucJ11HvkinZ6T@r3MRBsX41|ks z+qa5+mL+%UxRPQZ7E-D5fg4wTh`6NVFY-TPlivlv=CM6x!W*~Q@G>oz2opa`A~C9*xF-A3)uFNphLS)?GoCKFCk3fJ?aC%g8lLb!C zREN{JXVT#`TBJByFUM5#M4a#|{08&LJvJ{2Sv3+y@QOni{oOWXbxbliQE$q|YvCl!km|z5RHh}pA==r(tfoN`GS<`1 zZd$0O00}j{B}XJzGyN!4J5_y#Hug3Ts$t4v{hDUxiDs2;7Wn zm$eh#zhPf7fomTp`b@%pe7)_IAfU0V{xiK}2_J#hQBj3|mZC@;me;8{;dcQI`AN?W zIbfiHX>qLfW}6de_H5idX#La}a!8y?Q}Hxhz9dNB4| zSWiJm!Yl*)P>^FrBI{aVnBYXhHuv^gvTqmzw&}w5B*FaM%}R>!RTxUJVI(eq+tP5u ziMdX=Pnfg;tLFh&$n3KwxL%MHR{#xSY#P>se1ABJkg(r=o|foO)C#zQS)nF;@&pv} z1zf>vyHzSIzsxYf&5CprG6>mu=rOHc6D@ZZgysHMuHwtnV0mMKM$;q*WWYjpr{2Y= zJJj~!@x2?K*L!0ax=Ey9_gB{0pjl9~wuDNdzoryT)=9u#{-8lMBjN~VEsa{ zIG^iVpW+sE8!`yhcLBP1q6Mr^pz~d5J_+vQ=3#d0fc>ue%O|iEAS6|+#!rxp2N*uZ z%f$u|oEPgl`y~cG2=U$vHhF&J#O%GS@EAo#`Dk_iW^gpj=b)P6Rufb1YXgL3v^>TK z9K5u${Xz436M^rVUvSKwNrmva{kkC@4YefeAG6-efmJnrR(p%H3vynXyZPe{hzYQ~ zgMRUWi}7B}N5%ReF19vs%TGmL%tlOrr}l-wr?do}|4m?)K~^BT{^U|$0j!a)N)K)B z;|iO%sl;rD%`dKjm-~!;-kas@R<#`@GDy6dtvCU4@S9(|L#a3c+Hk~OzhP5&YnJXH zhdWDGAfPqySpn=v$$;G9y%JKiLajOYYEeY~iAVhSYJ=?H@oNhbh3*UdTOCl7;Mc`B z^-w8JJ_x&SYlaH!KjS(|9aQET?E}tRa@OL_EuX!xy;NS5Bz^%<3WZ)_EX@4=Hrzxj z!6Oz=%++NZ93GTT%mt^LdziczkCw%d8ftIcdjokiS~$adBBC7}V$o*J$yxzoj%8V0 zCw>vO&o~wjl3RH}3->u;J3z&*uD8T1sp^S{-np=Nt3HmP(mG&o%e_()apRaAhMpTfPveiSvdH=0;+!rQI9B{ z!EH!w5${_Sft!wn`p)Q7AEY`Sne2x8q|(2$2BrTrJ5&V8Lls4}uRnCyc4s>pJTc6T z6)7ECg3EOR^l~^MSpW|QtAV#izbsfQj(L68x{LehD-f~8&Ny<31MkYg+q`*@^*4m| zfH%$ePB?^{H#^*3ABJ%IfLGN8r%eKA3w;-Efat)zjcoxFqXrnZ;}&NR7MGv-V#(xk z6wOy2hGsiV z{gYOC8GuFPhHxI#fH%gOlKAZIQ2=|r`p*giAclJzPX*S8P7bU9rTTF8l-iieX1rQA zF#bQy88M#Tf-+HjEp;m1LL-4pm($eV^D7d3Zg025Oa7zaN4?%I5X6)2JWxpIrHT+C zWx0_|Y1C~XRLA>zs13m+qxi^p5V`_NtVK`s(DRT|P}d|yA_vl4$Z_GVERKh2v&l>) zC|gBlBl4-!avUD8_%h0ukDR*OLpB&c9WzXKY*k^IH?c;4u(G69|G@JKRUgoW?QgWD z;0{HrA+7L>h)xmXye_KrDx@b%MG5WdqX?9ZoQPTuXNaN9XNA4>RgD+4E8;yEfOA zXnvK*NL14n1#(qGz!izxp4_{#Ta>;{i(J0|G4~<-z^p(WK{%=z5TbIuwPq2xbtjw{ zA1K*4&MdYMh=_4{dLM{*$_MuBx8G7si(oo;!O!|6bp|aI*needa08-14N~Oy0sc{( z?nc>hM7)Mky$_5u?A1mig8b5x(SX0s6Ot~8p`X&kp#zvWs!84>pn{D!(4;`h5l654 zQM=6v!9?2yh*KiKDf<)Q+K5vo5bqG7B*o?RI_%o_Hn;}Wf*4(}1bAT@DgpfY%~j|* za;sBfH9Sup*Bm2nrqy30PAp@*0lj0If%)EEFMa@-?{Cmi0i_*06xl}%(^9K~HJ{uu@i20s&9!I?!7EkLT z0}YmjG)(K`tmkN8vuG{EWQ*YE6Xdb|P(g_ySxmqNLxrlB&;~I`7B@A%lcH8b4Zh(a z`pguy3x1GsrwXaVcjTbVOShri4&|>DIuJN!9<~D!h~nw8yeB7#C16In{sjm$@R5NZ z)JC2HFUz!@%tX~>d5~wUBj6Prl>pn5=I%k?ZBX@t^TX8DGD((e>kF~c>MvaI*Zd@q zWm!%sa?Xc|wPSO(*mm#_-GTIUKhni9(1hQ?x46)05(LvR-$*;j`!q2kym>4gIhQqt z43w6Ee#B1U;72K}D~o)Oaw-G3?sdIr2Rh>^oDq>Bgw8NP0l+WiG^hLO`X8(U1j$a7 zV>wEi#=Qhpsp9ZMha5{}xsA%8t^#`3kRVqhABxtCKo!9@xG8Fss|PjkF}X#T_~1+# z)+8pr47{kahUIqH>f#EBPcbg-_t_@LvG4wBCXS16B3(7p`hYyGF?f?arwwh(d=A2G z()yFO0hRXP(ef`vp($*N7jO~>4)gfBKeKKk?BF)4N@_ppqSVfSJQ<}8ibKf=R)I6K zYR>>nNz(;+M2bBFY?GM{7J+kHc6WA1)(_+g%^;7sy;bG_)V3jdP?*IX@bDLiJOD%_ zeXv0hqgWNxZMmZKO4#ZF06)^7(%SvtqPS1L4gOwrU`!V>Yi0)Uu~06JhckCK;>bp> zOCYHbBRJjig$p0hn=>DL57gO9Q4-7U-4($NCle}W2IyiTcY-=O7b)KAyif@^U3U+h zzG5vekViE}atO>U00-!dl+|Y*=;F5KNQ9p?8pD|m8rHLjF9adMo@|vwfnG-n`3plb z3Q|mA9r=7Pg>-YigruaTLA)8KAS|4Q)Z4ECwjs@=Xu_|52V9GSW3&#zF-}bxXxNI0 z?8WR$rI@VA@0`;>k!FGfo=BtUcVfr0vsaQG4~z%1Y< zIKodw$}F2uDha=f=LgR`|Bfd)49Q?Jc^w0XQ@;+G9RPH2G=;-uxadi%>#wnhOJBj4?xvPEnp|ufSh$)*CLyRV`;pZ{ zj2Qx)+wN`J3lw(01hobHx_ZJ>{#>R2l@bbGBKK)KO2_E@z}1^XrY(AkgFTd3c#7YW zK;(EeRQL~-@&rVKbDj*K9u7*8b%9YK7s|FU&z~oXjyumqUKG`%$V_0{)2u*H8^mnK z?n)3O?KozvrDb~;|AyRD*8o36Xv{_MF=Xd~ZQ}6N8qoJyOEQu<$MQaVSJCrus0p$W zcH^C_1*rmutm7Ji?&i9MARLU4MPz&6|4Y@$29Qoo@=X)SKD)7OJ)m+xHQx0Pfh?n5 z1gSIyv>)%z-&_uyW&_lJ?yEeh;-sQyTR16z6?SCxqQi-z+(RTr0{jyPIp?f1Tf;BQ zWK_{r@azme6^0a%$OqQxv)$?r4nxTafl#QnF$f(&od?x4hV;a~VMdZlJqZGX+#>Y2 z+X8`=D7p%wi2K_%X?MLCfM8u-NU0_PM;Z;;)S_g)zcz(2e{cdt4rt|F5P)Xu;iO$u zZn)Yc>O=t7$4;tMvp}w`!7t`<1^|4Wm4K&^;((~gE=E*6Le-aTq|q{;OYpltt3eWM zUaCfbnL8F7&?m+7O}dsnTfALy%$W#HI1&;P)#o&v+!_?uNuGMfU;LT69d017X>~Y1 zB-X;LU2N>Njdqn3e_y?WmqMX8-|$MqstZ<8ChgMKabIfXMH zP(tl3S{?BGKJxs{&kh1@U>3mJ$SdF$SGt<)144|zb7u+tKBXRVA8fFUjcmbps-Pm; zCVLzl1-N)vbp#d5&DBmQ8I}O=vtPmpX_yOSw!!AMReWBz(Col#7;-qOdrKP>i1zQt zJmJ8}$VWvv=A~cPthq-_YfEfHL@K-8$$n%%Ob%EkZ7ekrq zV$g^w8Cy0xNd)d>kMJMg^wO0S8`$eljzWSR_@9KKSCPj&L5Xe>rZ^fJ6+s1e{Yt^t z(dL0Tw&o^nVlLo@d;a(?fk16e&=WE1!yi_lC|EZ(8AZJ(JQe6Q$b(KqT=ql z=i^O4u$V5W&zIMd{!r5azwsbC8F^?_Vc5<%(&!El$v5eLACmPGp*v$I{KpuoKkDLN z?-Wymjqp;D*$o?g|7@Ih1oV}cJsm__3pY1|s-;8)`%DtU#|KMlwOG5OPU90@oDSvaSi`aYumEi#%#+ z;AA0lBHJ`?o2LV}oCM~2Bq7;~NX`T{XsHwEv!5gM6&j(nW8#xJ+TZaI^3IT+LQKzt zAJsOk2#>_gA1Yjb@cgF*_M?*L=JM_zqO4RjZ)}Y_kt&wpyaW4yJTH`-KDn?T;V3uf zcPn58vzUFzL$MKRW-1s=!xp}NP+jnY9z5A!QIQDVV=_T72R;NuJl+DbfebJ`W#;3XAwB5cy5oFW~kI(OX1d zEKJuZa@234sCptt#%>`)>W)z@1azbc#Mj(I*Ra>31Wb|w!oceh5C#p1AbLJA10taK zGH1~2!Dz*u&q#^G=0&jip5TxX2+-P4!jZO~QIAjtUhkd#6txS)kXOroB#GkgJ>m|m zg+c29wH3EHA+-`#`+;&;_&qU!!dC-2seoPv9EMmZATJ*-K<%biz3i2J_F<#u(1Cc) zcyj#vP;0L9T*Ms|3KzHG-CgTQ%$yx}&!{~yFY34+=x9#9)2T7~>P$q0xSh42WQCYA zUnFfiO@d@bQ&!$i+cJxP%XXptq{efi0^in$>h3SJzb_r^9lScgp{}ey^E68~tG8H( zv}g#_lc(NQq-^R-b)n}WmL!L6Ce>+0x#>SO!6$yD)L>J z5z?|>#cuX7Z3GRHtsmOHT{wK3rlFysXJ@yb{e@Es8Kg*k7Xl>%sO5Q}l1oR;gtvA= zys&65{Asda7J0S($G*~?M>=2&t(x~}h%T#uiOr=P{B_iM5yM6RrpVEpLA$&+d3~cB&$%6e7 z6bNr{VfTPiZEqQ>nsb0E@|(NG_!*DDS}q<{lU!9)#kxC*s@)a?viZYccni-ksISaH zmO}+5EkFnv3RSAwH|TVz9Kn8E1WdR+>F8Vw$($-y;$>37JPc$1Hv-mGvHL&)0`q@vl zZ|L&PG5df@WxyOLZ=bf|Ga4JfMDodQl`{QbS&3|Pfi#nkBZ!Yb3k^Jxw#5ToAHJ2~ zZ$5<`fcxroUbk-Upzoff!3N0JJl7tE1_d@aHsq!rKmXHy208fZul!R-fbdbywzip+weae8=~| zEU^nl$ZkmukkotC{>mKpuEm>(Oj!_6173oF1ho;ZO`nF2)=rqaleqp{g4+@XyF~*H z+i;z-jgp?CxjVt;>#&M|ju$@{6-5`q{0(LsGr)dCMfKK*bbO)HYy(t){HFxOjAGs%*7-Bzv^i{Q{)z zzY+O=Y<$D4k7q}N362MG=~V*B7UC)Ypt~r8YEWw5k2GX;Pc{g(?BUe+&y4rrKIs)w z!h%^19dy~2CwQw<0_F=t`c8DR_ZJ31UFNo(g>y*_xb#{y!;@9dM*FBf2;**>94bJ= zg(Q0Y)~#C&0n2_Zt&0Fb&~y$sKrqAzZ#|;_d;)o(I&++Nz+G1b-S{csR$;Z}c%zmg z0NXvE({YV;UO}`3w;8;lHyL7bmEr2u+ucS*t1BY$m*Vg?d2mu7eoh0NQ$Me~idA2* z`#7#FIL9qmsMzZpOldYCU z;jOq&aPkJFS@Ut>D#K2%7lx&(P8psI8DI;3`wEj3@-DZKv1ZR&Jz5z%fN+J z=HOd;t6ehw;UHxs)Xax%_;B3@lnf|<|NN6%DYN5A8(Zg5fK*8Re?47;p0W{LvN9Mj zWqe^fo*fmTotGqfvWh1=f(^?dP83c~;z$DO9yYIBg{8x;;wfIY#>Y(JvI1$J#~ktcP1yG@EqY)LK)te$MY9sEne39{=`LS zGe{Bs%wO#RQHX7yIP!3k-HmH4@ewO3<1 zgj`Gt>$prJ9tJ5x+tA3_7Pb-I1}PuE(ARCcMadZ{KgPTUO0afiV;6EN3>qis+=`1M zejnZYvaCHiSK`r07)-!eEx(SvfFEpNFWHItF=IT-QO+58VT^ej{ZXok{gDV;J^~>- zT$Im-C0$bxh^(6y5(XsS-kXMJ0w__ipPezT$r-&1KQ>z*$eNtHRJ{!sy}PvN4ZZbq z^>%gug6L=a)V`BFZ%l`LTVniH{`>durC{K5{KjOABkf%S4>mtsjNdf-65J$QZei_& zZ^1{@k^>rIfQERkboMI)V~{;*r9TE}Z*NNzdpbx<59DXaQKN~U3dL$>xlz<3wF2JaIj zNA@FHsL8O|HSgLtthYCe*|*eQ0fuld#tHZCXa2wDuKSGW~hwgXwjl5m8YnP6iUh{qe-M~g@{UJCnMvW@B6wB4xaviuOE8JeeQE# z_jP@)&%WMo`f~AduCnbHQ6f%q{=~tL+8T`{5dmJ707y_Ke5sN&$x&pLXze06PvR@R zjQjBNrRr*MSlsR#(HinC1Xz2^sbqS!D6BxLr!drinU?~czuFAz;L|ybqSyNcA`Xl7 zSb?}T=Se~ca#4s^vumcvy41lxfS5G&94`W464BN5 zjvw5$8{Pj5-q&7N+STMc9g}nW+m%sds)X-gcv?p)Ygj(waip5dl#9c3X7YMiNDfjU zz3!PtM@0LVLLO94Aat+(tSKf$5n@L`mOb^pT;ob>C@7J?2ndeEaJL>RxlRTuu@t*4 zfxA{c{{U^Xzhg45&R)q&C>+7W!-H)V{@CT@VXD#vcoU>6>ElL~ZRZB;^`E|ko-wjn zbx5X(jL}ojVJM~Ekx}=sitcK73~Mi3N>{ePO(f;xMEM5<1Q=OZjCu27Y8O(L4xZiB z67Db0P8^75Px(Qa(gvTW9XD^xrj49CqkI(AdXLqxpkg8I%-a*EOhYS|zUsv&m!aSt zh7O3N>p^m43wQT}pOKfypx_db-ql6TFqBgwGcBOu zl|^P+Ku{XXFnK z@9npikf-PbFh2~S#JM{c>d+lt$nHYo!$`i6tgjY3auC(!GB%99MR%lnF^Nw!Y-AX% zN97s?*AB(QUw8a55TFM>^(GwNTi&%(=y9?nd>koKG@!dpXr*+o3Ee zIbHvAJONuP2KiBY&zm1obKKF@PwBsi9VtYiE&S1wtu|x^eVDJ^496@pwvG?W$aj%w1G^vOGDhymtrp{tTaaysuu(R1q)fe~Hi zwfPC-?24#>*HP~9F@sm~?1UF2M{t&~HLX-yg{~r7$%{yjvShl^Hd7JG+E3-_oimKb zaX&x;EGuXGuZdzV7hE69kgx%5G+ufqBE7{uM z$NG0J=zQd(&t0}w?}+GX6K2W@WA5ZuEuw9Pga}(@Am&}x_gVx`QT30k2mt4;r+xy5 zzWN~*e<5%oZ`B>K>MtDPc`&@0qUKngv?aWEDxU;Ro&V68{@=6_^I07CW3D={eqrZl zyj9?>{VI$>W%bHv`c3w+RXQd^+SyvmNIw&{(KKwMutT|5>B}a#EYp<`bFf$*#)CSP zFvmRZULNZ}<_+-o{;a{!0EVkec&Ki2t$0Fr#3eC{EytBX(MqgvlYtUF(>hl%)4u1= z@wTl9JCq}XWq37(w+som*p-EXlUA0Y#c{}MZKzlmf8a;Z>W-JJ^6lKtE}KFp4eGku z(x;6Oe&;8MhLqvrMZb7&%D^}zczqUf5^LC@9aB-JI@;I~Ne8*Y^5}Z^-@Ueqr`WZI zW$qO&`&{b4n+8TIaq7A{qn-&~5EZ~W?z?@Cw};0Qzn`u)K%Ll&hPAI0(C9eY_;1%Y zhoNBLzhZ+8&nMFQM<+viK-*)j44#7{k%*qB?js93N9g262uP;=RBrgD)ewR#CA|s^ z4$_llt9Tl9u!y{(aD7?G6@ia*0zvo?Op-)gS4Y{^DgMHOlC0-`vR38aQulq=D=s_9 z@-4KWV+;nb->j>%chNsO9nZaFGSR(Jt8Gky2Q<KG+ zzO|3@UQcU?=e@stO(-BOObdaF%BimK(^}&$?M1iu@n^(~$q-U7iE-C_0I1H8OYw&Du8o0pORm6V6o4i~j|kAk?OX z)qo=)6peOE{{u%bhrkj1-su7s06pO4-E^GuUzi8N_2qY$Q2JE>>*VGU-oZ-#wD?>n zTt|9aOMyEnOUN$nUHuC%0)ZRRaRGQ(3jlUGyZ$ngj|q0%2h|`8sRKg%>bpDsf?5Ii z8@0P2-E2-EteB%6_=Dh0%-i9Kcz)@t)QVGTmL8xSd7%9lpogTSq?>m!67zWb&yQ38 zgP*ZOlmMb_Y6g(hb?Ff!PWf#6FKPu4gy&3j14%8n4UA$O>A$=VV2s+*;?f!~)pg`3 z12#Bh*IeE7CdT)m!|usy`XW$9nmryf{3dtZ+O zhe|HttxJ6$q4!Uh?dZ`76DI55FbKZ|vZS;{E>~j#4{=!o^b<)wO_98_;XUGsga72J z8m|6%J-`ciI{y{}(nacyi9pY1c)v`T{04{{et?>^Y-*%J0jV=+-tGc@)y*2a`eMx^ z5~QyJWk|mcp9HKH!E1B&sK)88UJ?x}<0As-9p@s5zM1DZoMHYJV-dGiH~+80EG{lK zLKEGOX5hMRO!u+^+{Mu2DWde@5dODH%w!7;82D0Q`t*)hl>+I&#BJ9jM6_FmRjpOy zYf&;&a?OGu#I36!mrLEpQ75oBUPAaUpZwdm0_RCX1;8UOLIzQQ(b;$u*yhn{@I{_} z{6NRQ{8#gdWxBYyzvT zk*IGBM}`7u=g$IwpF}OcX;jktlbEtrQF65vS9iGsU=XAggyUw8uk!I%v?tZ0>Y4G3 zg|4*^-9Ue|I&C=h8`!wHz!|qYZhSMjpS(0k5m>F8fYqvP3#T>$7k3?jAXZ47j)o-K z>U59yt)$F+`!Yh)^wnb;l?Vhd&Nsq$yEEP_w;w)_Pi#7Hj9Ly>f>SaibvQ-npHE@Z zX8d)iU=s{h18!OnhD!RRJHXY^)BqU44u$j`Ip*{ssh*)oEMWu8>5@n5=c9^x=T$ye z2MKyTr|}D0M+6J|EfY+UF>P0GRBP4507Wp^1e_jA!WQQ7 z!l|A+e6A1($xo(9j@DIT11Z2{D_h*&_^4#&-nV=vHQJc?1QM)eR?5al-U8BtDZp+u z`vEnkZPDMpG#w}~o(paAZpm7zP?_1;qX{wL{=a^yrRAq|0Njnh2W7N8-JeB-c(#4P zwp7rLvdQTu5HccwR4jW5P)b8*Pz;CQ9-WR5X!oKtpw1ka>G+neE%NWepn)$q6bu+w zI3e?z!b=7HTaN?TG#ZRpKcY|nw3yvbJeLWAq~^dLf_s`elaj&Xz5vAVTSn&6oBcj% zAPDv(rSrf)KW{gff|?t55k#I(b(~{(u}sgJNK7P}&H*5Q9&Mv(9)>we)2A0Z1NGE# zX54>9e29tnc|Al%CG^La7XdBqTPXk5Y(i}M2D}0`%da}E=D!|UMz)MdF(7-6LP@T# zd~MFj3Zf<}A%C`i52!B2ZuxSJ&YkuwxE#V307S(oDk;4TNY2_s@X9=8hkzY1S1abno@k z5;B_Zt)+lFZ_dwoUqau9Og5TeZ(-tUdIeK6b>HGvR>r841O4T(FoXwgvCv zJO{)_aRG3Zp8+mCYX}nbB{y@JEFOrgh zbrs*9bFu0DEFsEI{g+M!I8SN_DVfZNWx2crYtIstD-Z$^y+K3A6qLW15>mYGIwj}e z1Ui-bv6d;x`z>ZRp>M9&SQy?4Z^BmwjRoGZt;3x*rM9t@FG06oEcs$M)zu5o+5u9k zBUZa-vUus?o`(sCnGhQ>8#cnV^qI?ywUmEQkg7WfVP1zo%r{s4>5I<v(FRsS)Gk6RGWfk|WV;366%o&};3*q@MKlK3<;GC~Y0Q8+CBqa1K+%)=w0CRJL zmivcL)1C7i`#(b8g{`z9x%y$CY))DjMeqk-BQkJZRhWIBm-p*=Upf6z>_rC$uD-11 zYI}n-JqEa(uRc2nb`lkrv@|2i@1af%#yHzAA1)Aoxt220OAK*!gkRV zUh<}?&-3x)%P6^2QY|O}Ju|EkSKscb4<&)~%HQ z?*%_{1=>+93E-eoTHwhNUJ9H)vw;?6A8leg!|w7iGGGNl$1W*g)gV5e2;3OGe*5uzH~YPSQLlCDQtyGQQv>V*jkj5 z3i0iTG!Ut+20#Pki!!zkL8-4zuX(>V3iUgv^rLEi>+^o&NYSpgUMX7+F z_c=7ZryBzI+yJ9cUWu#69x#gui-_|>0N<;wf$7$ZJD|6V@}C+7dTcRZpSMlA$*kg! zgor#Egn$zs8;Kv*toAhrXbVsVIx0^G=trqDrjewvxR?E7j$weA+Ws5Z$j6xJ>+7F? z=^22q=vX$KG7mUqH35IS1uYeM%I5cPEs1kmtxKtZnziorh1r!M5R|iLngUL4)aWZ2 z5)2yNxC3aJoC83i)mrJIG0lxxhbZY4iIP^kJj&zMRD0%aElvDK$51PmvZjeqwTKy~ zfQa{S+M1Bms$T{|F>Q&%>f30tXVW(I(aLYnC&B zu2W|j4O!ohGUw0+!htI50#4%!V9dIPMFrt5->Y>%9+yT2OiTW6_mssvsJ*Dc;&fcO zQ9bc14rY6DqCq!i)&@2J{dTF=~CD5SS!K z;gzborI1T4P-`?7a=_lJ4H#S|5~IM>D{ z3UrDCLSR6H)+Ocyom*o=`4<%0F?8~G)buq7i==e&Rk_@q+@0bC`Q3@6MMl^b-H$fY zA9O{jub}%Yuz{)0(FWIWJGShfT*DfNE6(r9jv}-v#{u}c0E_zf1|5Hs!(X z+x9z>ZC>b*UF%wb$+IGyH#vJ%cWd=~%rwI7IA%|2YG)0J)znJnZk4BW<}9!;OZSH1 zC<&=OQbTu;_3S6ctVt>m?Mg661cdl@&K^OX0IFm&Ov$_$WCDM#nqy1DbMeOoxK?;C z+?XvHr&PhS-C)14rE_!a3Ao^bWG}La>)rzG9_;(6hOXZ6pY?8n{*}dtZ^0kvtj3mX zy&1oIAA9wzQIz~oM61~Zr`9TH8r9w}WHJ;Z&L2p zG$_bg^6@ABW5_JZNeP;4aL(QC(e``)ZY+;p*yheNF&7$O7k0qM&YQj=Iewk)ahI`u z&5!VLCbtjwPdaxe6oYy`X(C%y^ey|rVT-1%sz0dR}BqAl8FeL;CgwSlz!Iu)P zc%hb~r6C0iGyQkH;c{GwRhLO^)Pb=zi_h52ZrAbcP&xRh1>g(#2Po!NW zL>YgMBk!Rnb|C)>gwzMX+W!JQ{o(0FgN*vF%G2c}Xnap676*?#I+?bd*2w4& zyvfUGVxRGAo70(NM<=Hu5WRlBtFm|0MU4MK>sl`mJ$1Ijq3oE-lvxurs<%0iSj+yF z6CCmrDAiW0F3g%#4?M)pj4NI|)pF`PVv)y?i+4VKl5|w;p0x$TdWBUBRat zh~(BM*?NOCLrWm5X*Ut+3no&Uv&q|-?*H#ms@hA+nv3j8TfyBLKg3-1%3z8RJnSIJ zDrB0%mldSv@@m-K;__K)M{ZQ^9VGutw5+oNlQvaN2*iP!{ou&s-rxG-Ox;*5PxW|> zLp8S}`$H0vg)JibX|i*!GR@puvmmK{m3 z{KTkEoVl{q&x^bUG0eikE!RB5CUOCn*k}A*+=jy;N zU6D@}N6J^|3BnNRvm<}>Es?#OstO7Us9+@M*AQypn^LE|!VB1mf zrt4Yj5>{?AKDCwd4;=%Y>PnckTVYgVGY3c0UAsNAC$~y)hFN@TBwQB1)ghXl(wm|l zgxy;{jnXtRCSXJfo_ICDuirgtd zsPp{^(5?r;-STk3sm=_@#b7mg7HPSDL7ej#`M~fzZ&p5?p?>=}NzFr_#{cNmOgVi(;qO8hX(dNxvR}e&DRw&-4#{Gpy3m zk?SH}C87AP(TGHuhI_-!3Wvi~+3RoW@P&7x>m;oXdFG&J6~f=?Ll)ht_^l)ov(LA~ zpWEjNy25K$b4?4Ahfx;kb%?Sqr@%dUgoJ4My8De^`%prM8HKJmt5E6W=m@=|#nLfT z9r|e7IQz71IwZM&8@9p|R$b(g%voMfO!xDo6bv)7a&l&XR{p1610yz_cfGXKA(!dU;ESVXQDOi7l|H;8M7TqqgD4jh@mH^s_JTuXvJyrs`K{q_T>si zOb}p-Y?}#6Oa($1k%LTG^0_j{pq_hZJ$I1-`)`8Kz4{gCtuC*4f9^`jGD`pP+ewSP zP=W5;25#iVUXmA+`p`4o7+zm*wQz#~bJ{~ry{^|%&oQCHcPRDFuroy&lCte~kO5gc z&dO7Y)YPQhN|VW{|CA9nZSHKZNg5q4UIyh!|MmzEez6{|<6gwFwXOYCWcy%7{_BpQ zq3bC9g|S+eC6M4r$e*NCpqLVr=v~yk9Q3?}&JiTb0{1S+s>x(jyjGGiW$$b6TLkr( zuo1z42ND|3?hHO<+k1@Z&lpjLHyl&Z)g*QtSs1HoGZt%;fOw3x`bbw~R;$r;GISwm zg{`eEvTFJ)ki7sK#ccOH(C;Rp14dTxJ2{#gWZROVh7!U2JxKgM5l8A)NhI5pcOtBN z+BtIisC_z>w&f*kBeYt>1LM$`ob|0G<3SzRS0nJ1s$hoEkP3+JIw;vSK@v7>=m^$F z5NEwW9{JsqJ4bX38p*kV$>&CrZHaUdx0cu>o$|P@HaXsynbi!l_#F*vQSxHVM0qr0 zbH4krQ3%LKx*&(|N6erfG{uc=XM*mNqxL;hyLtQ@6m+ktmh63Bff&(UeZ-K68_4a7 zhAt|wooYByw9FIErkr)lR6~5nq29yB5w7z$e9AnV`YtvjqTXuO{K=M zmL409-M>wPJ9babgh`}m;tCD=CkR<>v5C3snHPrvB|!pS&(0K>foV8&HP#^UWOyGH zd4P?B6Z!BxW*s2u#3iMxn4{5~*ca*KD zbX7=%dZ%7MdLR-(XV`!|lTF~a=sqNBY^LyDP+Z$V2TWcC3cw4@;ll$aFsP9-zXyn` zzu5!#OUo+NeSgv5-FD;{Y4@ZGZ(T+Bt?Uprbhy3NY_HBbAvNs%Ng-pX~JfUG0yW>io<4k0|wc5cDdM^@uRxN9gK`=C%cke|II!Z@sN`TJt z&ofA79rBXxQX`d~HTQb7gpMyEH`Yvs+YmzNbzIlQv6rqylls(Q>vk!RBEA%yhF?!W za-OnimunosXX+XGJ4M2#UmuPp;H5O+d~T2akwY&3xd|6pZHKSD{qwnFF9&=8Mi zQ)ZMwpOmiB>aw_OrR-#MFj?7wFs?;ZRFqQ}9$g+ogEQT@aOfU5ZtJr_g;d%I+FcW! z$ByTEJGRV^JnsX_U0G_x2+t`s!A12psRozt`f7^PZK)G8f*ve*S-ncqL0QvvFj|G2gPTO_MH@# zIqCrU|4dnP-dyL~^;P`!St~`yOz_cTlbb)(BXQzGd_0*=Xi)L+oKZX))>*|W%@UOtg{bkDIq(fHyS3LO)Ro-Xviz(>!j-+f z?p-YR21VC(gz{MoE_#X<_@1MB<(t_lE0oa zUq(7=?$#T|yc5HcPS~p?88WZ{i4$>4kEYzb)X{Q_59adPw8^5f=ZMLB=5g;A*%lSU zc@8;H6|${kxGj(+$J<^%2R@_v*^t6l2{0;V8tdj3^m0?tV@&i+D_9QeZ=z7EF7s-r zVW%@**5~p`1V9|pBEjl_6YH$*P=08^^1ZJ<^4hD(T>DRFBW-Xg4b}l^v7EVDN=S_O z-*S%rXN-XOkw+Jnu2A^XoRHS}9wEK!OL(C_!?zAN-pk*WI6(Nk$C8xxWbRLsw6;ax zgWsm-<0#B#`}t59`MXf=(R+yRPa@9b@Pp^>_42fsA`BAqw!!T+dj}71`II&PCWbQ- z66K%b1ti2{dH>>N!4c7kn z?)g&au9@l+i0&#xT(ayT^po?Jf>as_lo!iAH6x7HgJ7aHw~#EH-(2m%!P3-I`|rNz zWWi}&)q;vJmoU|}KoS))0v8qWMRfB-y2}&VKP3^#Q*O%Ysf}hWATaI4jm*Pv% zfaI?;;dD4fiDv6+FES*#?mO7kd(|DkWLnRxe8ZPLcBaUOD5I*oc*kg)g4>@fNP&NB z(!jZfoAF(~H?Deo_8B&DTnUt6BE-nj6{FpuN}uQN+A%XFB9X>5hJh%Bo&Q0&e$|Iz zy7$pXa(+!>L(6iq@fJrAYI1hX;lGFQp>?59e`z+j?Io#s#Xc@%N39!=Lq>;pW#*4T z&aVdTTUvLpm)K||b;g;!q%HQFO`Jc4;SS2Vg z_p6Ru;JEeigSylrH(12n@*APN10}QtdL*5Q0A0@l{@8Jw+^D&pSWLS?X`7|C;e@!q zNRxlf0>oTJ%}7=z;O?p2CwNQC!AtJhj#)6DdtfZ@ge!TqJ+j=Px;6%{*pqwfm-R+d=7hVEw}N_-!V^y{Ap(#Np4@X*x(I2bGZ za_VENOgI7;WuZ*!T=_m=uSo9IAQlzm!|rWzXuS79>b)bW*|iu9yz%`MC@FFU3fGoMv)QneP5NkcWN z9~~u!ME=nfad{Rc7VI_X?(Nmnw>F)UnOWYR)?QM#Y!95p>kvUBVbedrY3k0i_&c)E zGyLyPtctmbl}A%NWV&Pe*U=y8V~_=>G*eDie*HRRKAKQipvp7J$MPHM)z!N2(;*V- zY(!II?I7AwWMM-vE04>JBCmG6l2;21;`!$pezL6H&{VEu>wFBDqPOs%K*)t(J=W;V z6n!)~G~(+1Som*^Bsy*-O?2{h&WgiYw+V<_t!% z0Xx!VW8|I*!p~C6g>`DeM=m#aG`{A#9ZR^Y&K85p9CPA?7-&1MH^Dw}vXM+E^c(17Xugr-? z;z;p*M<3}xaE`YhAq~I3_SV`d^L0S@utazVT7pj|1S6b?Za3u$uQa^Q-STaBP%9LdM%#UT=M{!Jz#2zFy>`oYe z?WSxpT`AEGva#lH`ayBcf=)P)ThPp3um#_iKW^dPNDGDLJCl3tI$KNxS&m8z_^6+5 zB7^k{<%w0E-!5!7_Madz0_t`BtbVy0X=#z%=S-}zz~>6-PdF1WMk=+Lpr&Im(KVtnQ^7bu$Z&Wz(5GnWZ@L!ILLKdh9)3@FXVzN_D^Us%YmW*12==kc=*i%L5zEL7` zTr@L&w79nHr2kspKFyFSs3Qj$}*xBf8JO|9UVF7m)a ztwwlm@;-twLF>SfE>B57TIXSj-J8%yZxJom#F*awc0|D_YI<7vF~x19SZkQR(qtU_ z1hnj|Wp)*}MTrtJ$zWoY300Pjse43#-|kodS;_aFqKn`(5s%MNTgG19rb3WnE@ zcn}E)@=DPVbI*ii)9j%VJUz8C53VyWgC${)_Vs=|RRv1?7}7=6PLLxdlBE2Bs$33g zE@pM$NzElqa+)VJZglTQe_OgyJbJ@}<_09M#}Umcj1%2pir@-&l3{m<@)MWr;`pJ-Jo-~`tF#fL?= zlV6ez!dJ+W%Z4a4>jmQD>%&)^Jw<-e$6s1d7vgk^N-0z;pvCJHvW6TbLy2RP5J+;x Yv%?nK?>QseLg7bu=^E{r#T&i<2Z(>~BLDyZ literal 0 HcmV?d00001 diff --git a/static/apple-touch-icon.png b/static/apple-touch-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..54a5336357ae91659ba3f8ab5a0394889cf1670d GIT binary patch literal 43853 zcmeFa`9IX}_douaY1AMYOSCBSEM;lYPT8hXRJM>U(V}E2d$MF^+GwR!*%J|>>_j9* z8zuXmEFnwweZJ>s)6)_j4YP=M^mt6&|idTo{J&>`_(L z#xN56lZ26H!~fBjthxaIM{v+q*@49th_qrD72Bh{UH7cXuNuw=_nIDxk4aSs9=DS; zz7j7%Rwf^%5S|{rLOsGa_ce$1ipytis}+XH)%j|qZ@jq2xoBAN<+pF&-j^%K@N04I zH@mgVa9+O9GJY+Zme71Vg1**W^(P;5H%a9`jT#Jats5LJaq8d9-Q^NAI#J_YR?@-> z*ml66-V{{H!$|blQ(6X<4>(?7v}JqoE_@^-aBupyn*A-FOL%DE!kGw_~p^g z8}meDzjVVtB9_RXeg69Og*oj@wmC+2KHBNoCL({f`|eK9d4cw@GzT^l7TIba)}4QA zIftmn(2>_)AFCXzNogQcs5Az~tmYIH5)ulq{~*cX)K>g-Ecmfgd+D0UgDIU~UHjY4 z8Dx2k^R|~nJBOKMx_Y*k#nB@@#z*F~l|&DET$s%>P!_FkW18vOU;9))uGYW5y{t|r z<#dfc{4gsTR>-xLK6`m8=OSlZW31&W0jib`GskajeLsn;Yg#{0Ws}u&fmboL0siNt zLwz<=s4I&#aS8s%@#eC%CiLxO&EIj6fs-^*(c$vw0Q? zZn7w9RQ>VphLY@$>lYxqk!?pJVK+ILB3#!O%fxo>lg=!*Voi+eXAai7b=1M(_q$$=c5-xO^cEP}yc$07 zM!{{M*hwJx4Z*;VeZJQ{T9Y{M)vH(M1v`}*j{0x1=xJVP9`L)TH6r*GP@!2@{`|-r z@wMqcL&JaMc`J!ThtA(g-9;oa1bK8;C5f*)U7b22ozb1+G&!|x{ z4k~W*7#q6qzEwG#)F4U4yjqNP=O1In$HzC%;a&QRRqNJuD8ad_b^vbFEPSS`_FU;l z=`3UEcgI)V_;ZE&dUtb4RX0d*L9$KVD-)Zn@iP%JwP%0a);Do!`h?iFM?+X@1?USdcPi%+&bjR@Ly4m4%-TiLUlWaZ^R{=2VIl6(@YR#^>B zjCS#!YYB9A&G`LpaR+RCulG!6bp>@rhsA|Ilv^^JKl_Wyvc`H%6-Iw0$ZWPKG8?vt z2;A~GBg0)M%D+^EDMcn@QH-XqeVo@9ZfxGy1=4;Fv_5VLQgEw$)mOByC1YZw{q*2B z8OtSn$UF!{0#|@T!B!O{#t0 za1g1^!8(_gz}lD2U7g(x*QA9qf#llmIZj-g_OW?zQOb6p!d!GXNBGtA2*ZREi~TlP z#1DbS1UI;mb|~;18CZjn9Xhu=DOeUq-a;zq18g)-a%d@Nmd@V=u3n}2agZ>Jpwl6lHFkMRgkOyhRHbCX{Y-&<`js{{ z>HVK(&6!7vdUOi6C*%#RNnkNrU(%i0Qwo$=V>eph^2gNuw`8`&zU-{L_t{EeCsh(8 zAEvf9FnE=gk&$c3t8=5fL2qB0wG;$C3!+a9>)e`T9T;h3Z1xIYJLTWpgPsbLui`J-6z{^QJ|qN0YBIuEyEtGLWktI>fD;jDp5E2roTFJatI z;U+J8B|EJp$Sw8w$NSO&Zk=iS{I{IimKGX%lyehtk01pTv$G_WxsP_F=zAQbS9I3A zbURg*+|f|;(s`7a#VEW`$~n2$u#kk5)V3nHrm){;tK`JU$jC7;K$H7ji+-tuq{9W< zKDZpJCB3`8@Z0y#GIbfp20LrUlTX)F7OB%~f1Xli3z9@F=Rp3EB zy>%PxbXq%0k=IE4kAdzA*>h*5N>%cy(VR=dCPM~yR;o|vySNNLN$h*#ZN_!>@x4!J z;h}B*r_)=d2YyvJv=4Mt$${l(9+C1ZBK;-h2RYHY!k+%|{+cq-OLdoe=A4No{LD%U z1uMSnn(cP~-o4FrU?hX4uC3urkYxJVx>x544rjM)Z%6JOUFQpF(Pm4>#6vI7NEA9apI4#Nn{a~ z5KsiSgNY|!G=M@6DBUiZ(m$x zf_Y&bV#?qXb$`Fv1&_}Cv2+6Q7ALMMLTijxua%M-$%9BHzU*N6j_q7z?7WEECUx(n z3h8ErK?-$MU?)isc&ZnEjj^2ncL$Q<8r#-z-&g2)9{9%2fK3+KmnqCPA8NJ6zdA@4B+%-ro+kL$!I}i(5y*D$<}FEAkuvWa$Om832AB5h5k8${QE$r ztMmB1Ly`F(0n7h^oW2LBhJ~Pal1Sj|aIAg#EB+iX9$pGlVn+9SqdRzFq2bYWk%5Tr~?nB zfrM9K0`Y}6WNL>Bt5R%4)6R5M_6lgnZ}_X$l%cyc@B^k674c2Qb41;{XPV`4&KCvryJCZ(Xa4e^q__%&F?#unV|klRhjN5HO!n_z zAGkji_)jxueYWxFjrTS67-rQaTIgggkvkJ0Byl16FZHD5TEh{us*d$kp9in@T zx0@<0R{ODNJb{5xv)|bMxFIvq0RH}^3uGRjiPokc|FZ79(9iN0KTz7%H-T>ca{EqF zc?L!ec~ocraZL)kIK4ktKr>Ovww^`{w7NIt?#DP#j2j-u7E8YxYM{SDiQXfD%Wx=T ztM?MQ&l$!7n2$#ChNWjfX%ENlWk=^lNHt%H#CZ`JeQ zOgNXCYze1#SecGiK&GY-0VCZZ!KHTNl=T)O|DE2K^lNw#2OY%kur4n@(2`o%CIMNQz~Y?Y&A$s(E)(tW(4C z`Wr{wPw*M8fUG}r^r3msa3v&zi&T;$FR>jd?v=?qLO|Euq_f6)>IxpL|1m&aG%s=w zfp+z>f@|N;gU-Loe_yz_NSTIs2LgV%3GUJ4&yRHc-L9$zP z*S+&pTs0>dvp4roxtDiivGWJfiEht5i?>*(9)SSP)0_O_#+2h#lV}hN2AzL?`Tkw{ zTuSPX$;c%lO67xAQ&9l$f)0H~f2D=0z@D4;dQReXwOoR0#|GLh!>cTh6aJ zi1|GK%DZ^WFH@OQm7824r%~tEOSk{4(iGJzG@zr-qAa#1to2(2>-b(Hu23EVZO1$L z6Q3SPBei6uLs``c{Mndt@ss+)znW1b^(EEr$79op9$$T9_up^nTkRThmBNw=H}yB& zFe5&3>HRE2w}C_JARKmZa7yT3`Ma1*68nHz`(X6sVo-dlBSvgvUYsFL!c22GI!43TCOtj6RBbx z)PmAikGSkz_V-vuHYdlnGrv+>dOk|C>Oi%|IiH+G3pab8|KL3qcj)E7Ww>R@HA9OJ z|4dm%(<2I-oc;c&AW%-}mp(DR;X~C7Z?a=+(JG~}?l%u~!{<&> zBC@V#h(_SHds+P5jVRd)gn(Cfz-E5jRc*+R!cS~2OK-Y=@KvTuZ?mZ4aO0g&ipFBEa%+m-tC+D5?%Z#_DzO2(#E{`6*_v>?L>w=&zXFmf9^oA-e?bjrN1XJ{L~cxK&4nQxuG7`0{8Z$AYiM5V+^3@; zf;FC82Rb#5LO=abntfCF*2H}UM|61j;nD2Z7v6rPr+;KDC{S-$)F&~RmvRaPeCir61D=Gb_>R_k)KggR-RrbAs!mTi1qj|wP zT8S_P)2)j=oXsFO_^viAVvP^hwHVA&k^TcGpmo{rNg8q;iI3mDecRje+248meNcZ_ zo^bcOBtP-Tbk+p7u(U~aOa1E$CC0BmPE|Nsh!nBEtz^~DZxMx|!@Vto&px!vkE(!O z&ezNVU)&aJoM!($=MuNu5(U@9t8oLeQ{5lASUKeR^XDN@`s|BKYxaAEKpx{eNju!0 zVIf=sqH<@f-*i-^D2<4AO}OcPKfZJ-%^1^*V`QM?Wd>MfnL0A&+y8%Hy?85v8xoSoEd>8q;bz?0!8lf^Bmk6!-Y8?mrs>@ zzAp{%JH5n{Uw4pwI8w2CG1Yrny;(=!UcK%;Zf@r{wV=S5MwG}gCkAlecSwU#m?w+{ zG};cdm)X3By;oL&2B?@|eQ$+DPziG%-LVdZ= zq50Fo`yuhY+EFs8!AF-ObTm(tKK2CQhS9qjYS}BNa*RrON&}~*lmY02=mW5fXSdb{ zbiXBGd2_ojgSA)<_cWz+RU|}cWN=L3Q zSl`eBiO5J7c=?zvTU^)waFW);;U{C`@%&)QY1RfPBf7-1{QaD%n1qDHg01$AgJm&> zy^kSP4>EDkUNwECG07<9^tcIFr^g0J--GkQpHxOIAHwSIA3U9QswUmZ8P%hoDi@tU zHsxuMvk{%6Vx9%lMe&5%u{cXDSCl->Jn8?!6RB$Q;M^fm`iqut|;wA~ilKDTY z(|T9w8{54-1z=8Z^vm5vroz^kmY!beX~^w1>ctwnSQL3idQ24sMK+D?Cm|?&f!bYN zoWAXxCTSsd_x}B$k+I%FY14SK_xD;CKfO84IO~;(V9}Z;_U~6HNJ&X4lGP>iu@JCl z?}A_@)~ZK52eYOLPD|@c7M+#XTo_D0bW-Rw2e$L_+=rm~p$F^0$JV>@cG$N2PBo`I z0fZRe9BBP!{sHns6;Cn2V*h%+<-5lm-(HDN4U{1z3q-(C;42)eoLVo9$8NmQ$6#GTWk}CtN?=m2~#37gWY< zt4^i#p9NrgsR6*PyZ)JbzZ;S}C~}(*UTsvoH-aH_fCsADc>C6( z@FjQTR^I!IRDeUI#gZV$OnL>`ewiBbEwKY$Z_=OgVzcKiSm#`u;WDd%@oe_AYk5Uv z_m`O}*+76O%S%W<8MW65TW4MV-1NkThjO)WJDt$)@lz`TKyvrvImz>|u*O(Cskt$J zVBl^gu71M<&>*0{LZRr}Hu-Z?vkC^ykqG+i5oc8vTbBju=drLR>bIffJp#?bqO}CY zzMYzoe{uYNq{sQkN^r5|9;*MaD$afQ8q{9hD0zFESsGR9`vD6H3o}z+c_%GCYC+Jn zfI=XySg}GAuCUhPj_M(7-N`Q@bs%Z;Tu)Jt2g!!5o>SF<)I+uXAQ>Tuf>hnTwoY5z_WLE_rEL`nEt*d>0!Qv7Zu0fR4u5*E z{v6~wz1buE?bnJnh@6{ly&as6zsl`2LBOXGU>I5%y(~RYM*!<Y9q=jM`S|qI`(Kp!0^sLkzyd~7@Y;%h zRI|~)1)3ZGfOAcEM&;OTuh)TSa%LwOZsNcnw872$g!U+NT6AOIdv$G1Ygt%-#wLCq-8!~rW~ru z*cU(zZ-76y63!p~x^kMZ#Z*Jg3xoVYi%JzcSL7*T3!qV11HWJT;~?vY$K7I;DI_!3 zaY(VGvO~6CSc&5evfR1bFo`v5GCfM74)B8e3bza3E16PZ0r^L`0Y*Dv1BvzNsL03% zi}kM<%)y}epBQg=doAwjs-W~~L4uK;M}(&yM}IfGHN6?FYCVe)cdo-q$zO>+F*w*! zC0w=W$2MK3=?-Xu14or1lqj(}Z0MsG#=1OumK+;yb_KtBiq0CXlScJ&nz+qtfoV2H zqb?-|ZgX<&_-Hy||NA1Rz?Qtnp{$r?_X)^*vhu6b9Ok^^*{3jlQN0-vfzts{pL_J1 z6*i659TUdz5&;sqU-|$-9lhXwtlWE=@W{ow_1u7}F`+Z%(GEfSWyBM!UBe^g92lTA z<^5j~PZKC5 z@-_|nxLnXl?UUwJ9IAJY?#mHT6!FLE{p-e>k7u#QyL_LSX1y|;H=49*`qKPsw7i>3 zoaw_KO*@C%O<5xq8P;cSs~;7>kPmH!5^i=LVHeMIZcw?H%vo~^{2;&gktmI7UG##C z;aDRiRyt4$+Zm}Z?s1)}mlP^RmiIBj8v=Qc0H#|BVKIxO_AZuyrpixT2T8_^1zzf-WjBI5V7^j&QdKZd~dHxQ{s>!4Hp@IP0;)WP>>fWMo_0UE_KT$`mwWPtn>H<=d%92q`c z>M@;#A;}!Gp-q1d>i0Ljhg523F~C1tL0-5OHG*#R)WnQWw_n;RZsRi*$4AqlEu95@ z+T)5=@7*w{2N+wyp}$lNR7xdNS*RH#cp@7HmAB7Ob{%BkCIR^j3sr z&Nv}}WRzr?lm&_T1=R4}H5|1njz9yciUxw0(C;Y;I5%U3F~`VY7pr`1xHs;j{y7(J z3^g)PMA!&X%fslEUjpxROYn-OLF=xzt0cP22Qm|m;mGwi1V$Ch{rb11(-`C<#&q}K zqDm98e|~+u@4mFDVqORR_RU8|(?mrr!}}tL_ca+g4X~!0zOJtPt!M?Qvd=Eq zI@|he&t}MqR{N?OOpnrJF~h^C$ZCP7^1cdMyXet$aKJdGAT6iW9E%*>vX$8rBn%UUUZ_&okhyV@@^(O+6`!a0_J!!>1 zJwk8dsy_SU2CJ|B0p6=u|ITesp||*br7Ke`TCLKxskcS z@-98UB;0PuV+&B_3}rEoSG=m#v?zx$eMl(Zw}E`WOr^phj6j3-Z|{LGRLdiH~e9djw{1Ci8MFk@tf5@Mc5N;Hd&xD(fi6H^YqA*ynTpJYJDLYZ1u&Uv z+c#MXfTh&M7CbT8IVH11+|WIA80Oou+#zbd(&^V164aH$P=b7a{wh6Vy+HNMX;3Cy zEHWMM>7WLae^geyA0fLVrLNWiQlo38kn*ROR1{lHS8$9O#f|1nJm>H%Dn^jMiC&4m ze!BpI7ShbYX&ygN^JzvB!JvI6LB*i02(VLCRQlc|A_KaBrw%4r9{3n_x_K&SjSdSr zK?J0x>J@;gdIt`Le?V2R#cxO?g*#!UM&p1?ulx+rYi(qPhK4SO$(>&ZpA2qHd(Lb|N8MnU-a;^=%c#f z2ovgy0tfS~uY+~qpMllWiSg14>4_tNm#Ozd;{)-B()JRvA<{%&lGsnEn=b=ul2mxcu5h{wBM{U0AWr7JgOKB#f2KcX7DKKx^+yh1 zwKnG_hFdg0|L2m_#@qW8f7WHu+X0NU(%&LywgkHlz`6=*GzC|zZd*80YX^zTRc?-n zqUU?I$Q@sT^>tJ^S0LoqEFsc-rczXM_W>$8j8M@N2|A?&8YCWdF#qx6;eYRq&^9ZW zP&7ZDprc2~hAMV9f{q}KnrD{qp$R+;m z*z^J%CLv9kkoRc@0$iysUbQ(I3jsLsp#C$Xk~Xtz!_G5rn}w@fGN@dscvb&oVBp{u zvOWjX9BskY>sf#1lKwc?M*aHGYiqZa1KwK-EfP5;o0dDXG1PN-4dt4LUtLef^fHb_ zHd|%;OlCckl_s?B*zx1XJ5Ce^jTg7aWpxg+ zqN6%zxCkF}CYlDzhcY@ATG4fm8P7Bq;M98}>OJMltmI}=?}zJi72pyRiC~co!mQ>3 zm^Y$r9_-sN`xB#016qtu%@mWaP6%K?(lI8PBe%g8p`o@V z02cZqJW1=bLgw^t1(}kcfFnxCKg2+l@{841nm0ZV2S>J96otKaiT!qFhNq(OaO{Qx z*0LyUk4`)n2JNZKrU*_?DYMm_-Yo*P=vi(1hDWA)y!#C=eIsZ+K#^C}R(@MixVds# z+YCAXM>NRA{M1%D8~_uXuW$UK1r3}=Ty}HaI#a>SnW%T!9}(ml^>flXRF=A!tN^j=-X?;^yZI5%hVL#=>;W$=)(WA z=H;LODvWztW@8AiaDoui()@H~)%5g=K;+~8R~?-pR)Y`gr^6Ud)+qGl%dVZZ`8d-b zyYQ#Nw9dS9z5aFb#~}Tlqg2}?U`jMFGTIgibC`NTZ7+CcvU52T93BsNvYuZ**iNGr zISrr3nQJBoV*e$paSxu3z%XDXulHiim_FX0025B5CGsUl7td5&6&zMs4t4Mn$7(DD ziY0$2s61S54kXVAQ&b;u=28V$`2;{s&gAi6fLV3Fg8`p{4?1g-XQZhldof7jJ!NF9 z%WVhDi440R?@BVkdDYTk(4npQq|L|a)dga>>|nJ6Zyn6oM%)lpny_uUPoG$Yfw)>X zLOyfJCF;$VnIgT(f#xfm8*;B)Q{=;{9lxp5d$VWpj;+pfI~X**-lgQ<$yS%C>ANsz z>DhWwB|cP6Ircy#KmA#U=YOeRy$(ut$&DvIE$F|3g+KfWE=L9^DJm`ts1gV~TM9aSSO#01(3LjPn+o+G^SVBp1!V44$sd zNP90%WSscyx1=)DEk%j&AmX3$897;_Q3#+Bo{$zF>MF$ZF^?vIkjq3ATsM`u9h?E` z04HcijvRRn)na-z^mH_Z^g3Oj@p51N;fAIl`3{@^Q4vjI3N$ZH={!-BWJzGD5F%K= zU(XR8^9UM9{vSORXD+tM_}zl4$U1jHZ0@2>KV=ZIc`G9G)Xd})6cxs5z9Glm?@h#! zZqroT?<#cRFEgM4s?Sm;q_(jJ2ndItLVMN55XO#^YeuI}Sf=3}(LZR#o>71);x>51WOQDB@Fox#(PK7{&_XfVB zdHmN?#z=8p3)_beX;mY#(Zd8=G_kxAFgwY>HEq+zMBwhIF#4Gj2QG&W4rQ#_Ty|!>bmVde zf5h7U9VL}bk35%-*tA;j*PqEZ^i=;lM95BmxZvmi*ak^+B4=WJv=E?S50+zcxkvk26YQZ3ubJMah1<0XqB()692ig^XoUuH#?EaAi|5a9c4!*vw8SqcgOgFA zn>(}E48tR66*ol8u^!ZA)il4BK6Nk(u@Oxf>2t~baZ)+|u?c`CHbTjOly{*)zwm@z zhh}I8wHPl9KL@TuX%OP4Nj+{+s6s-f?lGj-?^k*ftS)7Z|Nz9WrFR3uRHLz1S$ zM@P8HSWZw@=QMnXN`avg>V?wj&6JC4V65M5B7IuVl1YQvVdh*T|7mQZB*S>>ui^IT zv4O}(!AKiUo|+bsXyq_$^SmX_ZQ2#d$}r^i{}cO{v(&xJhK2?uY5ie3z_$9Yq2;Rd z;w|7OTvDvONgqz(1+mFDhj8Jnq_{D?>-xOmb+P(#tlsRBH^lnKDjs7$M1z0r|5GwG zYSA#CoyPm{tc;XljUbCLAE(bs&34D!^40Nx*kf9$nMb?d_Fmh)E!JDVi8QX`yIMSYf|Z3 zrPLD~CmK9E-!~CQwyBPAji2H)7QS2k0;YS`Fzaml2TkJ8HNwqhxM#?J1`3w)N{_9J zSNI~I$*^`Xaz*$qA?2dwaKLc0Xh;vP)ign3(s;8enetZf8QV80n!?oiMLbe4vpITx zU!qs=j~>mo4wq+_5K(KXSY-M`efBSOIdwg%=*P(vUr|%81-^SCalcXoFS)2zDfhEO zZ4{N_>pekFH#6ljxb%s|}lQetw7egDkLTmIV$V=mF?zeZoU8eN1m|mWg!o?7h?K>pRqiBl7oE>GBd1Zf*a$vixI@#5hq+y?4-e7 zHh5k#U$w!X^}%F(1*1))x*gvk+Z@Q$;J}4Xrco_K$605^hzx9Agb)%kyqbXqYpd9! z4#hWV+BU3n#E>%u*AC&2GvRSxfu1>ArIs_pYTbeLPkov%28xL+9iNVDux-M$!Mk#c-3OAm+EX?? zg`G)U#J)LtH;b6QhJABgqQ$H6xJ~?+vXgcU+bX%ZKxYWJ;8Y`Nm33pFa<1#{;nTD) z=7Y%2Y~o^1mj_Gxlmgq6t%9*`Q}zlk4Y*%m_&qj>wWJ-XY$>+Y9pEQ70o%30k9|LaeonR3CJ?gsT#;udCJoCe zMgDXG&AlLSFFK661h#hkv|kwgl;TB`K)N3AinPw>0{gi9YoL@xr7?Xtc4UkTV;9{U`f=?`1BN#4 zDDh(}>JAif&a15y_|UKBhF5Ss(S_Hw0%qY{pd4dTz}J-94b5jP5Y-6B+1db#ab@wt z^SIRtGQS`*rry01tBIE6`@-I)MFZ@M9}pyuBeuSs^}~+MR(=A3v7B8k9F?daV0rKS zy$V||AAryDL+_f5@yl(ygqx8ON5(L#@I4n+izq0ucJ11HvkinZ6T@r3MRBsX41|ks z+qa5+mL+%UxRPQZ7E-D5fg4wTh`6NVFY-TPlivlv=CM6x!W*~Q@G>oz2opa`A~C9*xF-A3)uFNphLS)?GoCKFCk3fJ?aC%g8lLb!C zREN{JXVT#`TBJByFUM5#M4a#|{08&LJvJ{2Sv3+y@QOni{oOWXbxbliQE$q|YvCl!km|z5RHh}pA==r(tfoN`GS<`1 zZd$0O00}j{B}XJzGyN!4J5_y#Hug3Ts$t4v{hDUxiDs2;7Wn zm$eh#zhPf7fomTp`b@%pe7)_IAfU0V{xiK}2_J#hQBj3|mZC@;me;8{;dcQI`AN?W zIbfiHX>qLfW}6de_H5idX#La}a!8y?Q}Hxhz9dNB4| zSWiJm!Yl*)P>^FrBI{aVnBYXhHuv^gvTqmzw&}w5B*FaM%}R>!RTxUJVI(eq+tP5u ziMdX=Pnfg;tLFh&$n3KwxL%MHR{#xSY#P>se1ABJkg(r=o|foO)C#zQS)nF;@&pv} z1zf>vyHzSIzsxYf&5CprG6>mu=rOHc6D@ZZgysHMuHwtnV0mMKM$;q*WWYjpr{2Y= zJJj~!@x2?K*L!0ax=Ey9_gB{0pjl9~wuDNdzoryT)=9u#{-8lMBjN~VEsa{ zIG^iVpW+sE8!`yhcLBP1q6Mr^pz~d5J_+vQ=3#d0fc>ue%O|iEAS6|+#!rxp2N*uZ z%f$u|oEPgl`y~cG2=U$vHhF&J#O%GS@EAo#`Dk_iW^gpj=b)P6Rufb1YXgL3v^>TK z9K5u${Xz436M^rVUvSKwNrmva{kkC@4YefeAG6-efmJnrR(p%H3vynXyZPe{hzYQ~ zgMRUWi}7B}N5%ReF19vs%TGmL%tlOrr}l-wr?do}|4m?)K~^BT{^U|$0j!a)N)K)B z;|iO%sl;rD%`dKjm-~!;-kas@R<#`@GDy6dtvCU4@S9(|L#a3c+Hk~OzhP5&YnJXH zhdWDGAfPqySpn=v$$;G9y%JKiLajOYYEeY~iAVhSYJ=?H@oNhbh3*UdTOCl7;Mc`B z^-w8JJ_x&SYlaH!KjS(|9aQET?E}tRa@OL_EuX!xy;NS5Bz^%<3WZ)_EX@4=Hrzxj z!6Oz=%++NZ93GTT%mt^LdziczkCw%d8ftIcdjokiS~$adBBC7}V$o*J$yxzoj%8V0 zCw>vO&o~wjl3RH}3->u;J3z&*uD8T1sp^S{-np=Nt3HmP(mG&o%e_()apRaAhMpTfPveiSvdH=0;+!rQI9B{ z!EH!w5${_Sft!wn`p)Q7AEY`Sne2x8q|(2$2BrTrJ5&V8Lls4}uRnCyc4s>pJTc6T z6)7ECg3EOR^l~^MSpW|QtAV#izbsfQj(L68x{LehD-f~8&Ny<31MkYg+q`*@^*4m| zfH%$ePB?^{H#^*3ABJ%IfLGN8r%eKA3w;-Efat)zjcoxFqXrnZ;}&NR7MGv-V#(xk z6wOy2hGsiV z{gYOC8GuFPhHxI#fH%gOlKAZIQ2=|r`p*giAclJzPX*S8P7bU9rTTF8l-iieX1rQA zF#bQy88M#Tf-+HjEp;m1LL-4pm($eV^D7d3Zg025Oa7zaN4?%I5X6)2JWxpIrHT+C zWx0_|Y1C~XRLA>zs13m+qxi^p5V`_NtVK`s(DRT|P}d|yA_vl4$Z_GVERKh2v&l>) zC|gBlBl4-!avUD8_%h0ukDR*OLpB&c9WzXKY*k^IH?c;4u(G69|G@JKRUgoW?QgWD z;0{HrA+7L>h)xmXye_KrDx@b%MG5WdqX?9ZoQPTuXNaN9XNA4>RgD+4E8;yEfOA zXnvK*NL14n1#(qGz!izxp4_{#Ta>;{i(J0|G4~<-z^p(WK{%=z5TbIuwPq2xbtjw{ zA1K*4&MdYMh=_4{dLM{*$_MuBx8G7si(oo;!O!|6bp|aI*needa08-14N~Oy0sc{( z?nc>hM7)Mky$_5u?A1mig8b5x(SX0s6Ot~8p`X&kp#zvWs!84>pn{D!(4;`h5l654 zQM=6v!9?2yh*KiKDf<)Q+K5vo5bqG7B*o?RI_%o_Hn;}Wf*4(}1bAT@DgpfY%~j|* za;sBfH9Sup*Bm2nrqy30PAp@*0lj0If%)EEFMa@-?{Cmi0i_*06xl}%(^9K~HJ{uu@i20s&9!I?!7EkLT z0}YmjG)(K`tmkN8vuG{EWQ*YE6Xdb|P(g_ySxmqNLxrlB&;~I`7B@A%lcH8b4Zh(a z`pguy3x1GsrwXaVcjTbVOShri4&|>DIuJN!9<~D!h~nw8yeB7#C16In{sjm$@R5NZ z)JC2HFUz!@%tX~>d5~wUBj6Prl>pn5=I%k?ZBX@t^TX8DGD((e>kF~c>MvaI*Zd@q zWm!%sa?Xc|wPSO(*mm#_-GTIUKhni9(1hQ?x46)05(LvR-$*;j`!q2kym>4gIhQqt z43w6Ee#B1U;72K}D~o)Oaw-G3?sdIr2Rh>^oDq>Bgw8NP0l+WiG^hLO`X8(U1j$a7 zV>wEi#=Qhpsp9ZMha5{}xsA%8t^#`3kRVqhABxtCKo!9@xG8Fss|PjkF}X#T_~1+# z)+8pr47{kahUIqH>f#EBPcbg-_t_@LvG4wBCXS16B3(7p`hYyGF?f?arwwh(d=A2G z()yFO0hRXP(ef`vp($*N7jO~>4)gfBKeKKk?BF)4N@_ppqSVfSJQ<}8ibKf=R)I6K zYR>>nNz(;+M2bBFY?GM{7J+kHc6WA1)(_+g%^;7sy;bG_)V3jdP?*IX@bDLiJOD%_ zeXv0hqgWNxZMmZKO4#ZF06)^7(%SvtqPS1L4gOwrU`!V>Yi0)Uu~06JhckCK;>bp> zOCYHbBRJjig$p0hn=>DL57gO9Q4-7U-4($NCle}W2IyiTcY-=O7b)KAyif@^U3U+h zzG5vekViE}atO>U00-!dl+|Y*=;F5KNQ9p?8pD|m8rHLjF9adMo@|vwfnG-n`3plb z3Q|mA9r=7Pg>-YigruaTLA)8KAS|4Q)Z4ECwjs@=Xu_|52V9GSW3&#zF-}bxXxNI0 z?8WR$rI@VA@0`;>k!FGfo=BtUcVfr0vsaQG4~z%1Y< zIKodw$}F2uDha=f=LgR`|Bfd)49Q?Jc^w0XQ@;+G9RPH2G=;-uxadi%>#wnhOJBj4?xvPEnp|ufSh$)*CLyRV`;pZ{ zj2Qx)+wN`J3lw(01hobHx_ZJ>{#>R2l@bbGBKK)KO2_E@z}1^XrY(AkgFTd3c#7YW zK;(EeRQL~-@&rVKbDj*K9u7*8b%9YK7s|FU&z~oXjyumqUKG`%$V_0{)2u*H8^mnK z?n)3O?KozvrDb~;|AyRD*8o36Xv{_MF=Xd~ZQ}6N8qoJyOEQu<$MQaVSJCrus0p$W zcH^C_1*rmutm7Ji?&i9MARLU4MPz&6|4Y@$29Qoo@=X)SKD)7OJ)m+xHQx0Pfh?n5 z1gSIyv>)%z-&_uyW&_lJ?yEeh;-sQyTR16z6?SCxqQi-z+(RTr0{jyPIp?f1Tf;BQ zWK_{r@azme6^0a%$OqQxv)$?r4nxTafl#QnF$f(&od?x4hV;a~VMdZlJqZGX+#>Y2 z+X8`=D7p%wi2K_%X?MLCfM8u-NU0_PM;Z;;)S_g)zcz(2e{cdt4rt|F5P)Xu;iO$u zZn)Yc>O=t7$4;tMvp}w`!7t`<1^|4Wm4K&^;((~gE=E*6Le-aTq|q{;OYpltt3eWM zUaCfbnL8F7&?m+7O}dsnTfALy%$W#HI1&;P)#o&v+!_?uNuGMfU;LT69d017X>~Y1 zB-X;LU2N>Njdqn3e_y?WmqMX8-|$MqstZ<8ChgMKabIfXMH zP(tl3S{?BGKJxs{&kh1@U>3mJ$SdF$SGt<)144|zb7u+tKBXRVA8fFUjcmbps-Pm; zCVLzl1-N)vbp#d5&DBmQ8I}O=vtPmpX_yOSw!!AMReWBz(Col#7;-qOdrKP>i1zQt zJmJ8}$VWvv=A~cPthq-_YfEfHL@K-8$$n%%Ob%EkZ7ekrq zV$g^w8Cy0xNd)d>kMJMg^wO0S8`$eljzWSR_@9KKSCPj&L5Xe>rZ^fJ6+s1e{Yt^t z(dL0Tw&o^nVlLo@d;a(?fk16e&=WE1!yi_lC|EZ(8AZJ(JQe6Q$b(KqT=ql z=i^O4u$V5W&zIMd{!r5azwsbC8F^?_Vc5<%(&!El$v5eLACmPGp*v$I{KpuoKkDLN z?-Wymjqp;D*$o?g|7@Ih1oV}cJsm__3pY1|s-;8)`%DtU#|KMlwOG5OPU90@oDSvaSi`aYumEi#%#+ z;AA0lBHJ`?o2LV}oCM~2Bq7;~NX`T{XsHwEv!5gM6&j(nW8#xJ+TZaI^3IT+LQKzt zAJsOk2#>_gA1Yjb@cgF*_M?*L=JM_zqO4RjZ)}Y_kt&wpyaW4yJTH`-KDn?T;V3uf zcPn58vzUFzL$MKRW-1s=!xp}NP+jnY9z5A!QIQDVV=_T72R;NuJl+DbfebJ`W#;3XAwB5cy5oFW~kI(OX1d zEKJuZa@234sCptt#%>`)>W)z@1azbc#Mj(I*Ra>31Wb|w!oceh5C#p1AbLJA10taK zGH1~2!Dz*u&q#^G=0&jip5TxX2+-P4!jZO~QIAjtUhkd#6txS)kXOroB#GkgJ>m|m zg+c29wH3EHA+-`#`+;&;_&qU!!dC-2seoPv9EMmZATJ*-K<%biz3i2J_F<#u(1Cc) zcyj#vP;0L9T*Ms|3KzHG-CgTQ%$yx}&!{~yFY34+=x9#9)2T7~>P$q0xSh42WQCYA zUnFfiO@d@bQ&!$i+cJxP%XXptq{efi0^in$>h3SJzb_r^9lScgp{}ey^E68~tG8H( zv}g#_lc(NQq-^R-b)n}WmL!L6Ce>+0x#>SO!6$yD)L>J z5z?|>#cuX7Z3GRHtsmOHT{wK3rlFysXJ@yb{e@Es8Kg*k7Xl>%sO5Q}l1oR;gtvA= zys&65{Asda7J0S($G*~?M>=2&t(x~}h%T#uiOr=P{B_iM5yM6RrpVEpLA$&+d3~cB&$%6e7 z6bNr{VfTPiZEqQ>nsb0E@|(NG_!*DDS}q<{lU!9)#kxC*s@)a?viZYccni-ksISaH zmO}+5EkFnv3RSAwH|TVz9Kn8E1WdR+>F8Vw$($-y;$>37JPc$1Hv-mGvHL&)0`q@vl zZ|L&PG5df@WxyOLZ=bf|Ga4JfMDodQl`{QbS&3|Pfi#nkBZ!Yb3k^Jxw#5ToAHJ2~ zZ$5<`fcxroUbk-Upzoff!3N0JJl7tE1_d@aHsq!rKmXHy208fZul!R-fbdbywzip+weae8=~| zEU^nl$ZkmukkotC{>mKpuEm>(Oj!_6173oF1ho;ZO`nF2)=rqaleqp{g4+@XyF~*H z+i;z-jgp?CxjVt;>#&M|ju$@{6-5`q{0(LsGr)dCMfKK*bbO)HYy(t){HFxOjAGs%*7-Bzv^i{Q{)z zzY+O=Y<$D4k7q}N362MG=~V*B7UC)Ypt~r8YEWw5k2GX;Pc{g(?BUe+&y4rrKIs)w z!h%^19dy~2CwQw<0_F=t`c8DR_ZJ31UFNo(g>y*_xb#{y!;@9dM*FBf2;**>94bJ= zg(Q0Y)~#C&0n2_Zt&0Fb&~y$sKrqAzZ#|;_d;)o(I&++Nz+G1b-S{csR$;Z}c%zmg z0NXvE({YV;UO}`3w;8;lHyL7bmEr2u+ucS*t1BY$m*Vg?d2mu7eoh0NQ$Me~idA2* z`#7#FIL9qmsMzZpOldYCU z;jOq&aPkJFS@Ut>D#K2%7lx&(P8psI8DI;3`wEj3@-DZKv1ZR&Jz5z%fN+J z=HOd;t6ehw;UHxs)Xax%_;B3@lnf|<|NN6%DYN5A8(Zg5fK*8Re?47;p0W{LvN9Mj zWqe^fo*fmTotGqfvWh1=f(^?dP83c~;z$DO9yYIBg{8x;;wfIY#>Y(JvI1$J#~ktcP1yG@EqY)LK)te$MY9sEne39{=`LS zGe{Bs%wO#RQHX7yIP!3k-HmH4@ewO3<1 zgj`Gt>$prJ9tJ5x+tA3_7Pb-I1}PuE(ARCcMadZ{KgPTUO0afiV;6EN3>qis+=`1M zejnZYvaCHiSK`r07)-!eEx(SvfFEpNFWHItF=IT-QO+58VT^ej{ZXok{gDV;J^~>- zT$Im-C0$bxh^(6y5(XsS-kXMJ0w__ipPezT$r-&1KQ>z*$eNtHRJ{!sy}PvN4ZZbq z^>%gug6L=a)V`BFZ%l`LTVniH{`>durC{K5{KjOABkf%S4>mtsjNdf-65J$QZei_& zZ^1{@k^>rIfQERkboMI)V~{;*r9TE}Z*NNzdpbx<59DXaQKN~U3dL$>xlz<3wF2JaIj zNA@FHsL8O|HSgLtthYCe*|*eQ0fuld#tHZCXa2wDuKSGW~hwgXwjl5m8YnP6iUh{qe-M~g@{UJCnMvW@B6wB4xaviuOE8JeeQE# z_jP@)&%WMo`f~AduCnbHQ6f%q{=~tL+8T`{5dmJ707y_Ke5sN&$x&pLXze06PvR@R zjQjBNrRr*MSlsR#(HinC1Xz2^sbqS!D6BxLr!drinU?~czuFAz;L|ybqSyNcA`Xl7 zSb?}T=Se~ca#4s^vumcvy41lxfS5G&94`W464BN5 zjvw5$8{Pj5-q&7N+STMc9g}nW+m%sds)X-gcv?p)Ygj(waip5dl#9c3X7YMiNDfjU zz3!PtM@0LVLLO94Aat+(tSKf$5n@L`mOb^pT;ob>C@7J?2ndeEaJL>RxlRTuu@t*4 zfxA{c{{U^Xzhg45&R)q&C>+7W!-H)V{@CT@VXD#vcoU>6>ElL~ZRZB;^`E|ko-wjn zbx5X(jL}ojVJM~Ekx}=sitcK73~Mi3N>{ePO(f;xMEM5<1Q=OZjCu27Y8O(L4xZiB z67Db0P8^75Px(Qa(gvTW9XD^xrj49CqkI(AdXLqxpkg8I%-a*EOhYS|zUsv&m!aSt zh7O3N>p^m43wQT}pOKfypx_db-ql6TFqBgwGcBOu zl|^P+Ku{XXFnK z@9npikf-PbFh2~S#JM{c>d+lt$nHYo!$`i6tgjY3auC(!GB%99MR%lnF^Nw!Y-AX% zN97s?*AB(QUw8a55TFM>^(GwNTi&%(=y9?nd>koKG@!dpXr*+o3Ee zIbHvAJONuP2KiBY&zm1obKKF@PwBsi9VtYiE&S1wtu|x^eVDJ^496@pwvG?W$aj%w1G^vOGDhymtrp{tTaaysuu(R1q)fe~Hi zwfPC-?24#>*HP~9F@sm~?1UF2M{t&~HLX-yg{~r7$%{yjvShl^Hd7JG+E3-_oimKb zaX&x;EGuXGuZdzV7hE69kgx%5G+ufqBE7{uM z$NG0J=zQd(&t0}w?}+GX6K2W@WA5ZuEuw9Pga}(@Am&}x_gVx`QT30k2mt4;r+xy5 zzWN~*e<5%oZ`B>K>MtDPc`&@0qUKngv?aWEDxU;Ro&V68{@=6_^I07CW3D={eqrZl zyj9?>{VI$>W%bHv`c3w+RXQd^+SyvmNIw&{(KKwMutT|5>B}a#EYp<`bFf$*#)CSP zFvmRZULNZ}<_+-o{;a{!0EVkec&Ki2t$0Fr#3eC{EytBX(MqgvlYtUF(>hl%)4u1= z@wTl9JCq}XWq37(w+som*p-EXlUA0Y#c{}MZKzlmf8a;Z>W-JJ^6lKtE}KFp4eGku z(x;6Oe&;8MhLqvrMZb7&%D^}zczqUf5^LC@9aB-JI@;I~Ne8*Y^5}Z^-@Ueqr`WZI zW$qO&`&{b4n+8TIaq7A{qn-&~5EZ~W?z?@Cw};0Qzn`u)K%Ll&hPAI0(C9eY_;1%Y zhoNBLzhZ+8&nMFQM<+viK-*)j44#7{k%*qB?js93N9g262uP;=RBrgD)ewR#CA|s^ z4$_llt9Tl9u!y{(aD7?G6@ia*0zvo?Op-)gS4Y{^DgMHOlC0-`vR38aQulq=D=s_9 z@-4KWV+;nb->j>%chNsO9nZaFGSR(Jt8Gky2Q<KG+ zzO|3@UQcU?=e@stO(-BOObdaF%BimK(^}&$?M1iu@n^(~$q-U7iE-C_0I1H8OYw&Du8o0pORm6V6o4i~j|kAk?OX z)qo=)6peOE{{u%bhrkj1-su7s06pO4-E^GuUzi8N_2qY$Q2JE>>*VGU-oZ-#wD?>n zTt|9aOMyEnOUN$nUHuC%0)ZRRaRGQ(3jlUGyZ$ngj|q0%2h|`8sRKg%>bpDsf?5Ii z8@0P2-E2-EteB%6_=Dh0%-i9Kcz)@t)QVGTmL8xSd7%9lpogTSq?>m!67zWb&yQ38 zgP*ZOlmMb_Y6g(hb?Ff!PWf#6FKPu4gy&3j14%8n4UA$O>A$=VV2s+*;?f!~)pg`3 z12#Bh*IeE7CdT)m!|usy`XW$9nmryf{3dtZ+O zhe|HttxJ6$q4!Uh?dZ`76DI55FbKZ|vZS;{E>~j#4{=!o^b<)wO_98_;XUGsga72J z8m|6%J-`ciI{y{}(nacyi9pY1c)v`T{04{{et?>^Y-*%J0jV=+-tGc@)y*2a`eMx^ z5~QyJWk|mcp9HKH!E1B&sK)88UJ?x}<0As-9p@s5zM1DZoMHYJV-dGiH~+80EG{lK zLKEGOX5hMRO!u+^+{Mu2DWde@5dODH%w!7;82D0Q`t*)hl>+I&#BJ9jM6_FmRjpOy zYf&;&a?OGu#I36!mrLEpQ75oBUPAaUpZwdm0_RCX1;8UOLIzQQ(b;$u*yhn{@I{_} z{6NRQ{8#gdWxBYyzvT zk*IGBM}`7u=g$IwpF}OcX;jktlbEtrQF65vS9iGsU=XAggyUw8uk!I%v?tZ0>Y4G3 zg|4*^-9Ue|I&C=h8`!wHz!|qYZhSMjpS(0k5m>F8fYqvP3#T>$7k3?jAXZ47j)o-K z>U59yt)$F+`!Yh)^wnb;l?Vhd&Nsq$yEEP_w;w)_Pi#7Hj9Ly>f>SaibvQ-npHE@Z zX8d)iU=s{h18!OnhD!RRJHXY^)BqU44u$j`Ip*{ssh*)oEMWu8>5@n5=c9^x=T$ye z2MKyTr|}D0M+6J|EfY+UF>P0GRBP4507Wp^1e_jA!WQQ7 z!l|A+e6A1($xo(9j@DIT11Z2{D_h*&_^4#&-nV=vHQJc?1QM)eR?5al-U8BtDZp+u z`vEnkZPDMpG#w}~o(paAZpm7zP?_1;qX{wL{=a^yrRAq|0Njnh2W7N8-JeB-c(#4P zwp7rLvdQTu5HccwR4jW5P)b8*Pz;CQ9-WR5X!oKtpw1ka>G+neE%NWepn)$q6bu+w zI3e?z!b=7HTaN?TG#ZRpKcY|nw3yvbJeLWAq~^dLf_s`elaj&Xz5vAVTSn&6oBcj% zAPDv(rSrf)KW{gff|?t55k#I(b(~{(u}sgJNK7P}&H*5Q9&Mv(9)>we)2A0Z1NGE# zX54>9e29tnc|Al%CG^La7XdBqTPXk5Y(i}M2D}0`%da}E=D!|UMz)MdF(7-6LP@T# zd~MFj3Zf<}A%C`i52!B2ZuxSJ&YkuwxE#V307S(oDk;4TNY2_s@X9=8hkzY1S1abno@k z5;B_Zt)+lFZ_dwoUqau9Og5TeZ(-tUdIeK6b>HGvR>r841O4T(FoXwgvCv zJO{)_aRG3Zp8+mCYX}nbB{y@JEFOrgh zbrs*9bFu0DEFsEI{g+M!I8SN_DVfZNWx2crYtIstD-Z$^y+K3A6qLW15>mYGIwj}e z1Ui-bv6d;x`z>ZRp>M9&SQy?4Z^BmwjRoGZt;3x*rM9t@FG06oEcs$M)zu5o+5u9k zBUZa-vUus?o`(sCnGhQ>8#cnV^qI?ywUmEQkg7WfVP1zo%r{s4>5I<v(FRsS)Gk6RGWfk|WV;366%o&};3*q@MKlK3<;GC~Y0Q8+CBqa1K+%)=w0CRJL zmivcL)1C7i`#(b8g{`z9x%y$CY))DjMeqk-BQkJZRhWIBm-p*=Upf6z>_rC$uD-11 zYI}n-JqEa(uRc2nb`lkrv@|2i@1af%#yHzAA1)Aoxt220OAK*!gkRV zUh<}?&-3x)%P6^2QY|O}Ju|EkSKscb4<&)~%HQ z?*%_{1=>+93E-eoTHwhNUJ9H)vw;?6A8leg!|w7iGGGNl$1W*g)gV5e2;3OGe*5uzH~YPSQLlCDQtyGQQv>V*jkj5 z3i0iTG!Ut+20#Pki!!zkL8-4zuX(>V3iUgv^rLEi>+^o&NYSpgUMX7+F z_c=7ZryBzI+yJ9cUWu#69x#gui-_|>0N<;wf$7$ZJD|6V@}C+7dTcRZpSMlA$*kg! zgor#Egn$zs8;Kv*toAhrXbVsVIx0^G=trqDrjewvxR?E7j$weA+Ws5Z$j6xJ>+7F? z=^22q=vX$KG7mUqH35IS1uYeM%I5cPEs1kmtxKtZnziorh1r!M5R|iLngUL4)aWZ2 z5)2yNxC3aJoC83i)mrJIG0lxxhbZY4iIP^kJj&zMRD0%aElvDK$51PmvZjeqwTKy~ zfQa{S+M1Bms$T{|F>Q&%>f30tXVW(I(aLYnC&B zu2W|j4O!ohGUw0+!htI50#4%!V9dIPMFrt5->Y>%9+yT2OiTW6_mssvsJ*Dc;&fcO zQ9bc14rY6DqCq!i)&@2J{dTF=~CD5SS!K z;gzborI1T4P-`?7a=_lJ4H#S|5~IM>D{ z3UrDCLSR6H)+Ocyom*o=`4<%0F?8~G)buq7i==e&Rk_@q+@0bC`Q3@6MMl^b-H$fY zA9O{jub}%Yuz{)0(FWIWJGShfT*DfNE6(r9jv}-v#{u}c0E_zf1|5Hs!(X z+x9z>ZC>b*UF%wb$+IGyH#vJ%cWd=~%rwI7IA%|2YG)0J)znJnZk4BW<}9!;OZSH1 zC<&=OQbTu;_3S6ctVt>m?Mg661cdl@&K^OX0IFm&Ov$_$WCDM#nqy1DbMeOoxK?;C z+?XvHr&PhS-C)14rE_!a3Ao^bWG}La>)rzG9_;(6hOXZ6pY?8n{*}dtZ^0kvtj3mX zy&1oIAA9wzQIz~oM61~Zr`9TH8r9w}WHJ;Z&L2p zG$_bg^6@ABW5_JZNeP;4aL(QC(e``)ZY+;p*yheNF&7$O7k0qM&YQj=Iewk)ahI`u z&5!VLCbtjwPdaxe6oYy`X(C%y^ey|rVT-1%sz0dR}BqAl8FeL;CgwSlz!Iu)P zc%hb~r6C0iGyQkH;c{GwRhLO^)Pb=zi_h52ZrAbcP&xRh1>g(#2Po!NW zL>YgMBk!Rnb|C)>gwzMX+W!JQ{o(0FgN*vF%G2c}Xnap676*?#I+?bd*2w4& zyvfUGVxRGAo70(NM<=Hu5WRlBtFm|0MU4MK>sl`mJ$1Ijq3oE-lvxurs<%0iSj+yF z6CCmrDAiW0F3g%#4?M)pj4NI|)pF`PVv)y?i+4VKl5|w;p0x$TdWBUBRat zh~(BM*?NOCLrWm5X*Ut+3no&Uv&q|-?*H#ms@hA+nv3j8TfyBLKg3-1%3z8RJnSIJ zDrB0%mldSv@@m-K;__K)M{ZQ^9VGutw5+oNlQvaN2*iP!{ou&s-rxG-Ox;*5PxW|> zLp8S}`$H0vg)JibX|i*!GR@puvmmK{m3 z{KTkEoVl{q&x^bUG0eikE!RB5CUOCn*k}A*+=jy;N zU6D@}N6J^|3BnNRvm<}>Es?#OstO7Us9+@M*AQypn^LE|!VB1mf zrt4Yj5>{?AKDCwd4;=%Y>PnckTVYgVGY3c0UAsNAC$~y)hFN@TBwQB1)ghXl(wm|l zgxy;{jnXtRCSXJfo_ICDuirgtd zsPp{^(5?r;-STk3sm=_@#b7mg7HPSDL7ej#`M~fzZ&p5?p?>=}NzFr_#{cNmOgVi(;qO8hX(dNxvR}e&DRw&-4#{Gpy3m zk?SH}C87AP(TGHuhI_-!3Wvi~+3RoW@P&7x>m;oXdFG&J6~f=?Ll)ht_^l)ov(LA~ zpWEjNy25K$b4?4Ahfx;kb%?Sqr@%dUgoJ4My8De^`%prM8HKJmt5E6W=m@=|#nLfT z9r|e7IQz71IwZM&8@9p|R$b(g%voMfO!xDo6bv)7a&l&XR{p1610yz_cfGXKA(!dU;ESVXQDOi7l|H;8M7TqqgD4jh@mH^s_JTuXvJyrs`K{q_T>si zOb}p-Y?}#6Oa($1k%LTG^0_j{pq_hZJ$I1-`)`8Kz4{gCtuC*4f9^`jGD`pP+ewSP zP=W5;25#iVUXmA+`p`4o7+zm*wQz#~bJ{~ry{^|%&oQCHcPRDFuroy&lCte~kO5gc z&dO7Y)YPQhN|VW{|CA9nZSHKZNg5q4UIyh!|MmzEez6{|<6gwFwXOYCWcy%7{_BpQ zq3bC9g|S+eC6M4r$e*NCpqLVr=v~yk9Q3?}&JiTb0{1S+s>x(jyjGGiW$$b6TLkr( zuo1z42ND|3?hHO<+k1@Z&lpjLHyl&Z)g*QtSs1HoGZt%;fOw3x`bbw~R;$r;GISwm zg{`eEvTFJ)ki7sK#ccOH(C;Rp14dTxJ2{#gWZROVh7!U2JxKgM5l8A)NhI5pcOtBN z+BtIisC_z>w&f*kBeYt>1LM$`ob|0G<3SzRS0nJ1s$hoEkP3+JIw;vSK@v7>=m^$F z5NEwW9{JsqJ4bX38p*kV$>&CrZHaUdx0cu>o$|P@HaXsynbi!l_#F*vQSxHVM0qr0 zbH4krQ3%LKx*&(|N6erfG{uc=XM*mNqxL;hyLtQ@6m+ktmh63Bff&(UeZ-K68_4a7 zhAt|wooYByw9FIErkr)lR6~5nq29yB5w7z$e9AnV`YtvjqTXuO{K=M zmL409-M>wPJ9babgh`}m;tCD=CkR<>v5C3snHPrvB|!pS&(0K>foV8&HP#^UWOyGH zd4P?B6Z!BxW*s2u#3iMxn4{5~*ca*KD zbX7=%dZ%7MdLR-(XV`!|lTF~a=sqNBY^LyDP+Z$V2TWcC3cw4@;ll$aFsP9-zXyn` zzu5!#OUo+NeSgv5-FD;{Y4@ZGZ(T+Bt?Uprbhy3NY_HBbAvNs%Ng-pX~JfUG0yW>io<4k0|wc5cDdM^@uRxN9gK`=C%cke|II!Z@sN`TJt z&ofA79rBXxQX`d~HTQb7gpMyEH`Yvs+YmzNbzIlQv6rqylls(Q>vk!RBEA%yhF?!W za-OnimunosXX+XGJ4M2#UmuPp;H5O+d~T2akwY&3xd|6pZHKSD{qwnFF9&=8Mi zQ)ZMwpOmiB>aw_OrR-#MFj?7wFs?;ZRFqQ}9$g+ogEQT@aOfU5ZtJr_g;d%I+FcW! z$ByTEJGRV^JnsX_U0G_x2+t`s!A12psRozt`f7^PZK)G8f*ve*S-ncqL0QvvFj|G2gPTO_MH@# zIqCrU|4dnP-dyL~^;P`!St~`yOz_cTlbb)(BXQzGd_0*=Xi)L+oKZX))>*|W%@UOtg{bkDIq(fHyS3LO)Ro-Xviz(>!j-+f z?p-YR21VC(gz{MoE_#X<_@1MB<(t_lE0oa zUq(7=?$#T|yc5HcPS~p?88WZ{i4$>4kEYzb)X{Q_59adPw8^5f=ZMLB=5g;A*%lSU zc@8;H6|${kxGj(+$J<^%2R@_v*^t6l2{0;V8tdj3^m0?tV@&i+D_9QeZ=z7EF7s-r zVW%@**5~p`1V9|pBEjl_6YH$*P=08^^1ZJ<^4hD(T>DRFBW-Xg4b}l^v7EVDN=S_O z-*S%rXN-XOkw+Jnu2A^XoRHS}9wEK!OL(C_!?zAN-pk*WI6(Nk$C8xxWbRLsw6;ax zgWsm-<0#B#`}t59`MXf=(R+yRPa@9b@Pp^>_42fsA`BAqw!!T+dj}71`II&PCWbQ- z66K%b1ti2{dH>>N!4c7kn z?)g&au9@l+i0&#xT(ayT^po?Jf>as_lo!iAH6x7HgJ7aHw~#EH-(2m%!P3-I`|rNz zWWi}&)q;vJmoU|}KoS))0v8qWMRfB-y2}&VKP3^#Q*O%Ysf}hWATaI4jm*Pv% zfaI?;;dD4fiDv6+FES*#?mO7kd(|DkWLnRxe8ZPLcBaUOD5I*oc*kg)g4>@fNP&NB z(!jZfoAF(~H?Deo_8B&DTnUt6BE-nj6{FpuN}uQN+A%XFB9X>5hJh%Bo&Q0&e$|Iz zy7$pXa(+!>L(6iq@fJrAYI1hX;lGFQp>?59e`z+j?Io#s#Xc@%N39!=Lq>;pW#*4T z&aVdTTUvLpm)K||b;g;!q%HQFO`Jc4;SS2Vg z_p6Ru;JEeigSylrH(12n@*APN10}QtdL*5Q0A0@l{@8Jw+^D&pSWLS?X`7|C;e@!q zNRxlf0>oTJ%}7=z;O?p2CwNQC!AtJhj#)6DdtfZ@ge!TqJ+j=Px;6%{*pqwfm-R+d=7hVEw}N_-!V^y{Ap(#Np4@X*x(I2bGZ za_VENOgI7;WuZ*!T=_m=uSo9IAQlzm!|rWzXuS79>b)bW*|iu9yz%`MC@FFU3fGoMv)QneP5NkcWN z9~~u!ME=nfad{Rc7VI_X?(Nmnw>F)UnOWYR)?QM#Y!95p>kvUBVbedrY3k0i_&c)E zGyLyPtctmbl}A%NWV&Pe*U=y8V~_=>G*eDie*HRRKAKQipvp7J$MPHM)z!N2(;*V- zY(!II?I7AwWMM-vE04>JBCmG6l2;21;`!$pezL6H&{VEu>wFBDqPOs%K*)t(J=W;V z6n!)~G~(+1Som*^Bsy*-O?2{h&WgiYw+V<_t!% z0Xx!VW8|I*!p~C6g>`DeM=m#aG`{A#9ZR^Y&K85p9CPA?7-&1MH^Dw}vXM+E^c(17Xugr-? z;z;p*M<3}xaE`YhAq~I3_SV`d^L0S@utazVT7pj|1S6b?Za3u$uQa^Q-STaBP%9LdM%#UT=M{!Jz#2zFy>`oYe z?WSxpT`AEGva#lH`ayBcf=)P)ThPp3um#_iKW^dPNDGDLJCl3tI$KNxS&m8z_^6+5 zB7^k{<%w0E-!5!7_Madz0_t`BtbVy0X=#z%=S-}zz~>6-PdF1WMk=+Lpr&Im(KVtnQ^7bu$Z&Wz(5GnWZ@L!ILLKdh9)3@FXVzN_D^Us%YmW*12==kc=*i%L5zEL7` zTr@L&w79nHr2kspKFyFSs3Qj$}*xBf8JO|9UVF7m)a ztwwlm@;-twLF>SfE>B57TIXSj-J8%yZxJom#F*awc0|D_YI<7vF~x19SZkQR(qtU_ z1hnj|Wp)*}MTrtJ$zWoY300Pjse43#-|kodS;_aFqKn`(5s%MNTgG19rb3WnE@ zcn}E)@=DPVbI*ii)9j%VJUz8C53VyWgC${)_Vs=|RRv1?7}7=6PLLxdlBE2Bs$33g zE@pM$NzElqa+)VJZglTQe_OgyJbJ@}<_09M#}Umcj1%2pir@-&l3{m<@)MWr;`pJ-Jo-~`tF#fL?= zlV6ez!dJ+W%Z4a4>jmQD>%&)^Jw<-e$6s1d7vgk^N-0z;pvCJHvW6TbLy2RP5J+;x Yv%?nK?>QseLg7bu=^E{r#T&i<2Z(>~BLDyZ literal 0 HcmV?d00001 diff --git a/static/favicon.ico b/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..54a5336357ae91659ba3f8ab5a0394889cf1670d GIT binary patch literal 43853 zcmeFa`9IX}_douaY1AMYOSCBSEM;lYPT8hXRJM>U(V}E2d$MF^+GwR!*%J|>>_j9* z8zuXmEFnwweZJ>s)6)_j4YP=M^mt6&|idTo{J&>`_(L z#xN56lZ26H!~fBjthxaIM{v+q*@49th_qrD72Bh{UH7cXuNuw=_nIDxk4aSs9=DS; zz7j7%Rwf^%5S|{rLOsGa_ce$1ipytis}+XH)%j|qZ@jq2xoBAN<+pF&-j^%K@N04I zH@mgVa9+O9GJY+Zme71Vg1**W^(P;5H%a9`jT#Jats5LJaq8d9-Q^NAI#J_YR?@-> z*ml66-V{{H!$|blQ(6X<4>(?7v}JqoE_@^-aBupyn*A-FOL%DE!kGw_~p^g z8}meDzjVVtB9_RXeg69Og*oj@wmC+2KHBNoCL({f`|eK9d4cw@GzT^l7TIba)}4QA zIftmn(2>_)AFCXzNogQcs5Az~tmYIH5)ulq{~*cX)K>g-Ecmfgd+D0UgDIU~UHjY4 z8Dx2k^R|~nJBOKMx_Y*k#nB@@#z*F~l|&DET$s%>P!_FkW18vOU;9))uGYW5y{t|r z<#dfc{4gsTR>-xLK6`m8=OSlZW31&W0jib`GskajeLsn;Yg#{0Ws}u&fmboL0siNt zLwz<=s4I&#aS8s%@#eC%CiLxO&EIj6fs-^*(c$vw0Q? zZn7w9RQ>VphLY@$>lYxqk!?pJVK+ILB3#!O%fxo>lg=!*Voi+eXAai7b=1M(_q$$=c5-xO^cEP}yc$07 zM!{{M*hwJx4Z*;VeZJQ{T9Y{M)vH(M1v`}*j{0x1=xJVP9`L)TH6r*GP@!2@{`|-r z@wMqcL&JaMc`J!ThtA(g-9;oa1bK8;C5f*)U7b22ozb1+G&!|x{ z4k~W*7#q6qzEwG#)F4U4yjqNP=O1In$HzC%;a&QRRqNJuD8ad_b^vbFEPSS`_FU;l z=`3UEcgI)V_;ZE&dUtb4RX0d*L9$KVD-)Zn@iP%JwP%0a);Do!`h?iFM?+X@1?USdcPi%+&bjR@Ly4m4%-TiLUlWaZ^R{=2VIl6(@YR#^>B zjCS#!YYB9A&G`LpaR+RCulG!6bp>@rhsA|Ilv^^JKl_Wyvc`H%6-Iw0$ZWPKG8?vt z2;A~GBg0)M%D+^EDMcn@QH-XqeVo@9ZfxGy1=4;Fv_5VLQgEw$)mOByC1YZw{q*2B z8OtSn$UF!{0#|@T!B!O{#t0 za1g1^!8(_gz}lD2U7g(x*QA9qf#llmIZj-g_OW?zQOb6p!d!GXNBGtA2*ZREi~TlP z#1DbS1UI;mb|~;18CZjn9Xhu=DOeUq-a;zq18g)-a%d@Nmd@V=u3n}2agZ>Jpwl6lHFkMRgkOyhRHbCX{Y-&<`js{{ z>HVK(&6!7vdUOi6C*%#RNnkNrU(%i0Qwo$=V>eph^2gNuw`8`&zU-{L_t{EeCsh(8 zAEvf9FnE=gk&$c3t8=5fL2qB0wG;$C3!+a9>)e`T9T;h3Z1xIYJLTWpgPsbLui`J-6z{^QJ|qN0YBIuEyEtGLWktI>fD;jDp5E2roTFJatI z;U+J8B|EJp$Sw8w$NSO&Zk=iS{I{IimKGX%lyehtk01pTv$G_WxsP_F=zAQbS9I3A zbURg*+|f|;(s`7a#VEW`$~n2$u#kk5)V3nHrm){;tK`JU$jC7;K$H7ji+-tuq{9W< zKDZpJCB3`8@Z0y#GIbfp20LrUlTX)F7OB%~f1Xli3z9@F=Rp3EB zy>%PxbXq%0k=IE4kAdzA*>h*5N>%cy(VR=dCPM~yR;o|vySNNLN$h*#ZN_!>@x4!J z;h}B*r_)=d2YyvJv=4Mt$${l(9+C1ZBK;-h2RYHY!k+%|{+cq-OLdoe=A4No{LD%U z1uMSnn(cP~-o4FrU?hX4uC3urkYxJVx>x544rjM)Z%6JOUFQpF(Pm4>#6vI7NEA9apI4#Nn{a~ z5KsiSgNY|!G=M@6DBUiZ(m$x zf_Y&bV#?qXb$`Fv1&_}Cv2+6Q7ALMMLTijxua%M-$%9BHzU*N6j_q7z?7WEECUx(n z3h8ErK?-$MU?)isc&ZnEjj^2ncL$Q<8r#-z-&g2)9{9%2fK3+KmnqCPA8NJ6zdA@4B+%-ro+kL$!I}i(5y*D$<}FEAkuvWa$Om832AB5h5k8${QE$r ztMmB1Ly`F(0n7h^oW2LBhJ~Pal1Sj|aIAg#EB+iX9$pGlVn+9SqdRzFq2bYWk%5Tr~?nB zfrM9K0`Y}6WNL>Bt5R%4)6R5M_6lgnZ}_X$l%cyc@B^k674c2Qb41;{XPV`4&KCvryJCZ(Xa4e^q__%&F?#unV|klRhjN5HO!n_z zAGkji_)jxueYWxFjrTS67-rQaTIgggkvkJ0Byl16FZHD5TEh{us*d$kp9in@T zx0@<0R{ODNJb{5xv)|bMxFIvq0RH}^3uGRjiPokc|FZ79(9iN0KTz7%H-T>ca{EqF zc?L!ec~ocraZL)kIK4ktKr>Ovww^`{w7NIt?#DP#j2j-u7E8YxYM{SDiQXfD%Wx=T ztM?MQ&l$!7n2$#ChNWjfX%ENlWk=^lNHt%H#CZ`JeQ zOgNXCYze1#SecGiK&GY-0VCZZ!KHTNl=T)O|DE2K^lNw#2OY%kur4n@(2`o%CIMNQz~Y?Y&A$s(E)(tW(4C z`Wr{wPw*M8fUG}r^r3msa3v&zi&T;$FR>jd?v=?qLO|Euq_f6)>IxpL|1m&aG%s=w zfp+z>f@|N;gU-Loe_yz_NSTIs2LgV%3GUJ4&yRHc-L9$zP z*S+&pTs0>dvp4roxtDiivGWJfiEht5i?>*(9)SSP)0_O_#+2h#lV}hN2AzL?`Tkw{ zTuSPX$;c%lO67xAQ&9l$f)0H~f2D=0z@D4;dQReXwOoR0#|GLh!>cTh6aJ zi1|GK%DZ^WFH@OQm7824r%~tEOSk{4(iGJzG@zr-qAa#1to2(2>-b(Hu23EVZO1$L z6Q3SPBei6uLs``c{Mndt@ss+)znW1b^(EEr$79op9$$T9_up^nTkRThmBNw=H}yB& zFe5&3>HRE2w}C_JARKmZa7yT3`Ma1*68nHz`(X6sVo-dlBSvgvUYsFL!c22GI!43TCOtj6RBbx z)PmAikGSkz_V-vuHYdlnGrv+>dOk|C>Oi%|IiH+G3pab8|KL3qcj)E7Ww>R@HA9OJ z|4dm%(<2I-oc;c&AW%-}mp(DR;X~C7Z?a=+(JG~}?l%u~!{<&> zBC@V#h(_SHds+P5jVRd)gn(Cfz-E5jRc*+R!cS~2OK-Y=@KvTuZ?mZ4aO0g&ipFBEa%+m-tC+D5?%Z#_DzO2(#E{`6*_v>?L>w=&zXFmf9^oA-e?bjrN1XJ{L~cxK&4nQxuG7`0{8Z$AYiM5V+^3@; zf;FC82Rb#5LO=abntfCF*2H}UM|61j;nD2Z7v6rPr+;KDC{S-$)F&~RmvRaPeCir61D=Gb_>R_k)KggR-RrbAs!mTi1qj|wP zT8S_P)2)j=oXsFO_^viAVvP^hwHVA&k^TcGpmo{rNg8q;iI3mDecRje+248meNcZ_ zo^bcOBtP-Tbk+p7u(U~aOa1E$CC0BmPE|Nsh!nBEtz^~DZxMx|!@Vto&px!vkE(!O z&ezNVU)&aJoM!($=MuNu5(U@9t8oLeQ{5lASUKeR^XDN@`s|BKYxaAEKpx{eNju!0 zVIf=sqH<@f-*i-^D2<4AO}OcPKfZJ-%^1^*V`QM?Wd>MfnL0A&+y8%Hy?85v8xoSoEd>8q;bz?0!8lf^Bmk6!-Y8?mrs>@ zzAp{%JH5n{Uw4pwI8w2CG1Yrny;(=!UcK%;Zf@r{wV=S5MwG}gCkAlecSwU#m?w+{ zG};cdm)X3By;oL&2B?@|eQ$+DPziG%-LVdZ= zq50Fo`yuhY+EFs8!AF-ObTm(tKK2CQhS9qjYS}BNa*RrON&}~*lmY02=mW5fXSdb{ zbiXBGd2_ojgSA)<_cWz+RU|}cWN=L3Q zSl`eBiO5J7c=?zvTU^)waFW);;U{C`@%&)QY1RfPBf7-1{QaD%n1qDHg01$AgJm&> zy^kSP4>EDkUNwECG07<9^tcIFr^g0J--GkQpHxOIAHwSIA3U9QswUmZ8P%hoDi@tU zHsxuMvk{%6Vx9%lMe&5%u{cXDSCl->Jn8?!6RB$Q;M^fm`iqut|;wA~ilKDTY z(|T9w8{54-1z=8Z^vm5vroz^kmY!beX~^w1>ctwnSQL3idQ24sMK+D?Cm|?&f!bYN zoWAXxCTSsd_x}B$k+I%FY14SK_xD;CKfO84IO~;(V9}Z;_U~6HNJ&X4lGP>iu@JCl z?}A_@)~ZK52eYOLPD|@c7M+#XTo_D0bW-Rw2e$L_+=rm~p$F^0$JV>@cG$N2PBo`I z0fZRe9BBP!{sHns6;Cn2V*h%+<-5lm-(HDN4U{1z3q-(C;42)eoLVo9$8NmQ$6#GTWk}CtN?=m2~#37gWY< zt4^i#p9NrgsR6*PyZ)JbzZ;S}C~}(*UTsvoH-aH_fCsADc>C6( z@FjQTR^I!IRDeUI#gZV$OnL>`ewiBbEwKY$Z_=OgVzcKiSm#`u;WDd%@oe_AYk5Uv z_m`O}*+76O%S%W<8MW65TW4MV-1NkThjO)WJDt$)@lz`TKyvrvImz>|u*O(Cskt$J zVBl^gu71M<&>*0{LZRr}Hu-Z?vkC^ykqG+i5oc8vTbBju=drLR>bIffJp#?bqO}CY zzMYzoe{uYNq{sQkN^r5|9;*MaD$afQ8q{9hD0zFESsGR9`vD6H3o}z+c_%GCYC+Jn zfI=XySg}GAuCUhPj_M(7-N`Q@bs%Z;Tu)Jt2g!!5o>SF<)I+uXAQ>Tuf>hnTwoY5z_WLE_rEL`nEt*d>0!Qv7Zu0fR4u5*E z{v6~wz1buE?bnJnh@6{ly&as6zsl`2LBOXGU>I5%y(~RYM*!<Y9q=jM`S|qI`(Kp!0^sLkzyd~7@Y;%h zRI|~)1)3ZGfOAcEM&;OTuh)TSa%LwOZsNcnw872$g!U+NT6AOIdv$G1Ygt%-#wLCq-8!~rW~ru z*cU(zZ-76y63!p~x^kMZ#Z*Jg3xoVYi%JzcSL7*T3!qV11HWJT;~?vY$K7I;DI_!3 zaY(VGvO~6CSc&5evfR1bFo`v5GCfM74)B8e3bza3E16PZ0r^L`0Y*Dv1BvzNsL03% zi}kM<%)y}epBQg=doAwjs-W~~L4uK;M}(&yM}IfGHN6?FYCVe)cdo-q$zO>+F*w*! zC0w=W$2MK3=?-Xu14or1lqj(}Z0MsG#=1OumK+;yb_KtBiq0CXlScJ&nz+qtfoV2H zqb?-|ZgX<&_-Hy||NA1Rz?Qtnp{$r?_X)^*vhu6b9Ok^^*{3jlQN0-vfzts{pL_J1 z6*i659TUdz5&;sqU-|$-9lhXwtlWE=@W{ow_1u7}F`+Z%(GEfSWyBM!UBe^g92lTA z<^5j~PZKC5 z@-_|nxLnXl?UUwJ9IAJY?#mHT6!FLE{p-e>k7u#QyL_LSX1y|;H=49*`qKPsw7i>3 zoaw_KO*@C%O<5xq8P;cSs~;7>kPmH!5^i=LVHeMIZcw?H%vo~^{2;&gktmI7UG##C z;aDRiRyt4$+Zm}Z?s1)}mlP^RmiIBj8v=Qc0H#|BVKIxO_AZuyrpixT2T8_^1zzf-WjBI5V7^j&QdKZd~dHxQ{s>!4Hp@IP0;)WP>>fWMo_0UE_KT$`mwWPtn>H<=d%92q`c z>M@;#A;}!Gp-q1d>i0Ljhg523F~C1tL0-5OHG*#R)WnQWw_n;RZsRi*$4AqlEu95@ z+T)5=@7*w{2N+wyp}$lNR7xdNS*RH#cp@7HmAB7Ob{%BkCIR^j3sr z&Nv}}WRzr?lm&_T1=R4}H5|1njz9yciUxw0(C;Y;I5%U3F~`VY7pr`1xHs;j{y7(J z3^g)PMA!&X%fslEUjpxROYn-OLF=xzt0cP22Qm|m;mGwi1V$Ch{rb11(-`C<#&q}K zqDm98e|~+u@4mFDVqORR_RU8|(?mrr!}}tL_ca+g4X~!0zOJtPt!M?Qvd=Eq zI@|he&t}MqR{N?OOpnrJF~h^C$ZCP7^1cdMyXet$aKJdGAT6iW9E%*>vX$8rBn%UUUZ_&okhyV@@^(O+6`!a0_J!!>1 zJwk8dsy_SU2CJ|B0p6=u|ITesp||*br7Ke`TCLKxskcS z@-98UB;0PuV+&B_3}rEoSG=m#v?zx$eMl(Zw}E`WOr^phj6j3-Z|{LGRLdiH~e9djw{1Ci8MFk@tf5@Mc5N;Hd&xD(fi6H^YqA*ynTpJYJDLYZ1u&Uv z+c#MXfTh&M7CbT8IVH11+|WIA80Oou+#zbd(&^V164aH$P=b7a{wh6Vy+HNMX;3Cy zEHWMM>7WLae^geyA0fLVrLNWiQlo38kn*ROR1{lHS8$9O#f|1nJm>H%Dn^jMiC&4m ze!BpI7ShbYX&ygN^JzvB!JvI6LB*i02(VLCRQlc|A_KaBrw%4r9{3n_x_K&SjSdSr zK?J0x>J@;gdIt`Le?V2R#cxO?g*#!UM&p1?ulx+rYi(qPhK4SO$(>&ZpA2qHd(Lb|N8MnU-a;^=%c#f z2ovgy0tfS~uY+~qpMllWiSg14>4_tNm#Ozd;{)-B()JRvA<{%&lGsnEn=b=ul2mxcu5h{wBM{U0AWr7JgOKB#f2KcX7DKKx^+yh1 zwKnG_hFdg0|L2m_#@qW8f7WHu+X0NU(%&LywgkHlz`6=*GzC|zZd*80YX^zTRc?-n zqUU?I$Q@sT^>tJ^S0LoqEFsc-rczXM_W>$8j8M@N2|A?&8YCWdF#qx6;eYRq&^9ZW zP&7ZDprc2~hAMV9f{q}KnrD{qp$R+;m z*z^J%CLv9kkoRc@0$iysUbQ(I3jsLsp#C$Xk~Xtz!_G5rn}w@fGN@dscvb&oVBp{u zvOWjX9BskY>sf#1lKwc?M*aHGYiqZa1KwK-EfP5;o0dDXG1PN-4dt4LUtLef^fHb_ zHd|%;OlCckl_s?B*zx1XJ5Ce^jTg7aWpxg+ zqN6%zxCkF}CYlDzhcY@ATG4fm8P7Bq;M98}>OJMltmI}=?}zJi72pyRiC~co!mQ>3 zm^Y$r9_-sN`xB#016qtu%@mWaP6%K?(lI8PBe%g8p`o@V z02cZqJW1=bLgw^t1(}kcfFnxCKg2+l@{841nm0ZV2S>J96otKaiT!qFhNq(OaO{Qx z*0LyUk4`)n2JNZKrU*_?DYMm_-Yo*P=vi(1hDWA)y!#C=eIsZ+K#^C}R(@MixVds# z+YCAXM>NRA{M1%D8~_uXuW$UK1r3}=Ty}HaI#a>SnW%T!9}(ml^>flXRF=A!tN^j=-X?;^yZI5%hVL#=>;W$=)(WA z=H;LODvWztW@8AiaDoui()@H~)%5g=K;+~8R~?-pR)Y`gr^6Ud)+qGl%dVZZ`8d-b zyYQ#Nw9dS9z5aFb#~}Tlqg2}?U`jMFGTIgibC`NTZ7+CcvU52T93BsNvYuZ**iNGr zISrr3nQJBoV*e$paSxu3z%XDXulHiim_FX0025B5CGsUl7td5&6&zMs4t4Mn$7(DD ziY0$2s61S54kXVAQ&b;u=28V$`2;{s&gAi6fLV3Fg8`p{4?1g-XQZhldof7jJ!NF9 z%WVhDi440R?@BVkdDYTk(4npQq|L|a)dga>>|nJ6Zyn6oM%)lpny_uUPoG$Yfw)>X zLOyfJCF;$VnIgT(f#xfm8*;B)Q{=;{9lxp5d$VWpj;+pfI~X**-lgQ<$yS%C>ANsz z>DhWwB|cP6Ircy#KmA#U=YOeRy$(ut$&DvIE$F|3g+KfWE=L9^DJm`ts1gV~TM9aSSO#01(3LjPn+o+G^SVBp1!V44$sd zNP90%WSscyx1=)DEk%j&AmX3$897;_Q3#+Bo{$zF>MF$ZF^?vIkjq3ATsM`u9h?E` z04HcijvRRn)na-z^mH_Z^g3Oj@p51N;fAIl`3{@^Q4vjI3N$ZH={!-BWJzGD5F%K= zU(XR8^9UM9{vSORXD+tM_}zl4$U1jHZ0@2>KV=ZIc`G9G)Xd})6cxs5z9Glm?@h#! zZqroT?<#cRFEgM4s?Sm;q_(jJ2ndItLVMN55XO#^YeuI}Sf=3}(LZR#o>71);x>51WOQDB@Fox#(PK7{&_XfVB zdHmN?#z=8p3)_beX;mY#(Zd8=G_kxAFgwY>HEq+zMBwhIF#4Gj2QG&W4rQ#_Ty|!>bmVde zf5h7U9VL}bk35%-*tA;j*PqEZ^i=;lM95BmxZvmi*ak^+B4=WJv=E?S50+zcxkvk26YQZ3ubJMah1<0XqB()692ig^XoUuH#?EaAi|5a9c4!*vw8SqcgOgFA zn>(}E48tR66*ol8u^!ZA)il4BK6Nk(u@Oxf>2t~baZ)+|u?c`CHbTjOly{*)zwm@z zhh}I8wHPl9KL@TuX%OP4Nj+{+s6s-f?lGj-?^k*ftS)7Z|Nz9WrFR3uRHLz1S$ zM@P8HSWZw@=QMnXN`avg>V?wj&6JC4V65M5B7IuVl1YQvVdh*T|7mQZB*S>>ui^IT zv4O}(!AKiUo|+bsXyq_$^SmX_ZQ2#d$}r^i{}cO{v(&xJhK2?uY5ie3z_$9Yq2;Rd z;w|7OTvDvONgqz(1+mFDhj8Jnq_{D?>-xOmb+P(#tlsRBH^lnKDjs7$M1z0r|5GwG zYSA#CoyPm{tc;XljUbCLAE(bs&34D!^40Nx*kf9$nMb?d_Fmh)E!JDVi8QX`yIMSYf|Z3 zrPLD~CmK9E-!~CQwyBPAji2H)7QS2k0;YS`Fzaml2TkJ8HNwqhxM#?J1`3w)N{_9J zSNI~I$*^`Xaz*$qA?2dwaKLc0Xh;vP)ign3(s;8enetZf8QV80n!?oiMLbe4vpITx zU!qs=j~>mo4wq+_5K(KXSY-M`efBSOIdwg%=*P(vUr|%81-^SCalcXoFS)2zDfhEO zZ4{N_>pekFH#6ljxb%s|}lQetw7egDkLTmIV$V=mF?zeZoU8eN1m|mWg!o?7h?K>pRqiBl7oE>GBd1Zf*a$vixI@#5hq+y?4-e7 zHh5k#U$w!X^}%F(1*1))x*gvk+Z@Q$;J}4Xrco_K$605^hzx9Agb)%kyqbXqYpd9! z4#hWV+BU3n#E>%u*AC&2GvRSxfu1>ArIs_pYTbeLPkov%28xL+9iNVDux-M$!Mk#c-3OAm+EX?? zg`G)U#J)LtH;b6QhJABgqQ$H6xJ~?+vXgcU+bX%ZKxYWJ;8Y`Nm33pFa<1#{;nTD) z=7Y%2Y~o^1mj_Gxlmgq6t%9*`Q}zlk4Y*%m_&qj>wWJ-XY$>+Y9pEQ70o%30k9|LaeonR3CJ?gsT#;udCJoCe zMgDXG&AlLSFFK661h#hkv|kwgl;TB`K)N3AinPw>0{gi9YoL@xr7?Xtc4UkTV;9{U`f=?`1BN#4 zDDh(}>JAif&a15y_|UKBhF5Ss(S_Hw0%qY{pd4dTz}J-94b5jP5Y-6B+1db#ab@wt z^SIRtGQS`*rry01tBIE6`@-I)MFZ@M9}pyuBeuSs^}~+MR(=A3v7B8k9F?daV0rKS zy$V||AAryDL+_f5@yl(ygqx8ON5(L#@I4n+izq0ucJ11HvkinZ6T@r3MRBsX41|ks z+qa5+mL+%UxRPQZ7E-D5fg4wTh`6NVFY-TPlivlv=CM6x!W*~Q@G>oz2opa`A~C9*xF-A3)uFNphLS)?GoCKFCk3fJ?aC%g8lLb!C zREN{JXVT#`TBJByFUM5#M4a#|{08&LJvJ{2Sv3+y@QOni{oOWXbxbliQE$q|YvCl!km|z5RHh}pA==r(tfoN`GS<`1 zZd$0O00}j{B}XJzGyN!4J5_y#Hug3Ts$t4v{hDUxiDs2;7Wn zm$eh#zhPf7fomTp`b@%pe7)_IAfU0V{xiK}2_J#hQBj3|mZC@;me;8{;dcQI`AN?W zIbfiHX>qLfW}6de_H5idX#La}a!8y?Q}Hxhz9dNB4| zSWiJm!Yl*)P>^FrBI{aVnBYXhHuv^gvTqmzw&}w5B*FaM%}R>!RTxUJVI(eq+tP5u ziMdX=Pnfg;tLFh&$n3KwxL%MHR{#xSY#P>se1ABJkg(r=o|foO)C#zQS)nF;@&pv} z1zf>vyHzSIzsxYf&5CprG6>mu=rOHc6D@ZZgysHMuHwtnV0mMKM$;q*WWYjpr{2Y= zJJj~!@x2?K*L!0ax=Ey9_gB{0pjl9~wuDNdzoryT)=9u#{-8lMBjN~VEsa{ zIG^iVpW+sE8!`yhcLBP1q6Mr^pz~d5J_+vQ=3#d0fc>ue%O|iEAS6|+#!rxp2N*uZ z%f$u|oEPgl`y~cG2=U$vHhF&J#O%GS@EAo#`Dk_iW^gpj=b)P6Rufb1YXgL3v^>TK z9K5u${Xz436M^rVUvSKwNrmva{kkC@4YefeAG6-efmJnrR(p%H3vynXyZPe{hzYQ~ zgMRUWi}7B}N5%ReF19vs%TGmL%tlOrr}l-wr?do}|4m?)K~^BT{^U|$0j!a)N)K)B z;|iO%sl;rD%`dKjm-~!;-kas@R<#`@GDy6dtvCU4@S9(|L#a3c+Hk~OzhP5&YnJXH zhdWDGAfPqySpn=v$$;G9y%JKiLajOYYEeY~iAVhSYJ=?H@oNhbh3*UdTOCl7;Mc`B z^-w8JJ_x&SYlaH!KjS(|9aQET?E}tRa@OL_EuX!xy;NS5Bz^%<3WZ)_EX@4=Hrzxj z!6Oz=%++NZ93GTT%mt^LdziczkCw%d8ftIcdjokiS~$adBBC7}V$o*J$yxzoj%8V0 zCw>vO&o~wjl3RH}3->u;J3z&*uD8T1sp^S{-np=Nt3HmP(mG&o%e_()apRaAhMpTfPveiSvdH=0;+!rQI9B{ z!EH!w5${_Sft!wn`p)Q7AEY`Sne2x8q|(2$2BrTrJ5&V8Lls4}uRnCyc4s>pJTc6T z6)7ECg3EOR^l~^MSpW|QtAV#izbsfQj(L68x{LehD-f~8&Ny<31MkYg+q`*@^*4m| zfH%$ePB?^{H#^*3ABJ%IfLGN8r%eKA3w;-Efat)zjcoxFqXrnZ;}&NR7MGv-V#(xk z6wOy2hGsiV z{gYOC8GuFPhHxI#fH%gOlKAZIQ2=|r`p*giAclJzPX*S8P7bU9rTTF8l-iieX1rQA zF#bQy88M#Tf-+HjEp;m1LL-4pm($eV^D7d3Zg025Oa7zaN4?%I5X6)2JWxpIrHT+C zWx0_|Y1C~XRLA>zs13m+qxi^p5V`_NtVK`s(DRT|P}d|yA_vl4$Z_GVERKh2v&l>) zC|gBlBl4-!avUD8_%h0ukDR*OLpB&c9WzXKY*k^IH?c;4u(G69|G@JKRUgoW?QgWD z;0{HrA+7L>h)xmXye_KrDx@b%MG5WdqX?9ZoQPTuXNaN9XNA4>RgD+4E8;yEfOA zXnvK*NL14n1#(qGz!izxp4_{#Ta>;{i(J0|G4~<-z^p(WK{%=z5TbIuwPq2xbtjw{ zA1K*4&MdYMh=_4{dLM{*$_MuBx8G7si(oo;!O!|6bp|aI*needa08-14N~Oy0sc{( z?nc>hM7)Mky$_5u?A1mig8b5x(SX0s6Ot~8p`X&kp#zvWs!84>pn{D!(4;`h5l654 zQM=6v!9?2yh*KiKDf<)Q+K5vo5bqG7B*o?RI_%o_Hn;}Wf*4(}1bAT@DgpfY%~j|* za;sBfH9Sup*Bm2nrqy30PAp@*0lj0If%)EEFMa@-?{Cmi0i_*06xl}%(^9K~HJ{uu@i20s&9!I?!7EkLT z0}YmjG)(K`tmkN8vuG{EWQ*YE6Xdb|P(g_ySxmqNLxrlB&;~I`7B@A%lcH8b4Zh(a z`pguy3x1GsrwXaVcjTbVOShri4&|>DIuJN!9<~D!h~nw8yeB7#C16In{sjm$@R5NZ z)JC2HFUz!@%tX~>d5~wUBj6Prl>pn5=I%k?ZBX@t^TX8DGD((e>kF~c>MvaI*Zd@q zWm!%sa?Xc|wPSO(*mm#_-GTIUKhni9(1hQ?x46)05(LvR-$*;j`!q2kym>4gIhQqt z43w6Ee#B1U;72K}D~o)Oaw-G3?sdIr2Rh>^oDq>Bgw8NP0l+WiG^hLO`X8(U1j$a7 zV>wEi#=Qhpsp9ZMha5{}xsA%8t^#`3kRVqhABxtCKo!9@xG8Fss|PjkF}X#T_~1+# z)+8pr47{kahUIqH>f#EBPcbg-_t_@LvG4wBCXS16B3(7p`hYyGF?f?arwwh(d=A2G z()yFO0hRXP(ef`vp($*N7jO~>4)gfBKeKKk?BF)4N@_ppqSVfSJQ<}8ibKf=R)I6K zYR>>nNz(;+M2bBFY?GM{7J+kHc6WA1)(_+g%^;7sy;bG_)V3jdP?*IX@bDLiJOD%_ zeXv0hqgWNxZMmZKO4#ZF06)^7(%SvtqPS1L4gOwrU`!V>Yi0)Uu~06JhckCK;>bp> zOCYHbBRJjig$p0hn=>DL57gO9Q4-7U-4($NCle}W2IyiTcY-=O7b)KAyif@^U3U+h zzG5vekViE}atO>U00-!dl+|Y*=;F5KNQ9p?8pD|m8rHLjF9adMo@|vwfnG-n`3plb z3Q|mA9r=7Pg>-YigruaTLA)8KAS|4Q)Z4ECwjs@=Xu_|52V9GSW3&#zF-}bxXxNI0 z?8WR$rI@VA@0`;>k!FGfo=BtUcVfr0vsaQG4~z%1Y< zIKodw$}F2uDha=f=LgR`|Bfd)49Q?Jc^w0XQ@;+G9RPH2G=;-uxadi%>#wnhOJBj4?xvPEnp|ufSh$)*CLyRV`;pZ{ zj2Qx)+wN`J3lw(01hobHx_ZJ>{#>R2l@bbGBKK)KO2_E@z}1^XrY(AkgFTd3c#7YW zK;(EeRQL~-@&rVKbDj*K9u7*8b%9YK7s|FU&z~oXjyumqUKG`%$V_0{)2u*H8^mnK z?n)3O?KozvrDb~;|AyRD*8o36Xv{_MF=Xd~ZQ}6N8qoJyOEQu<$MQaVSJCrus0p$W zcH^C_1*rmutm7Ji?&i9MARLU4MPz&6|4Y@$29Qoo@=X)SKD)7OJ)m+xHQx0Pfh?n5 z1gSIyv>)%z-&_uyW&_lJ?yEeh;-sQyTR16z6?SCxqQi-z+(RTr0{jyPIp?f1Tf;BQ zWK_{r@azme6^0a%$OqQxv)$?r4nxTafl#QnF$f(&od?x4hV;a~VMdZlJqZGX+#>Y2 z+X8`=D7p%wi2K_%X?MLCfM8u-NU0_PM;Z;;)S_g)zcz(2e{cdt4rt|F5P)Xu;iO$u zZn)Yc>O=t7$4;tMvp}w`!7t`<1^|4Wm4K&^;((~gE=E*6Le-aTq|q{;OYpltt3eWM zUaCfbnL8F7&?m+7O}dsnTfALy%$W#HI1&;P)#o&v+!_?uNuGMfU;LT69d017X>~Y1 zB-X;LU2N>Njdqn3e_y?WmqMX8-|$MqstZ<8ChgMKabIfXMH zP(tl3S{?BGKJxs{&kh1@U>3mJ$SdF$SGt<)144|zb7u+tKBXRVA8fFUjcmbps-Pm; zCVLzl1-N)vbp#d5&DBmQ8I}O=vtPmpX_yOSw!!AMReWBz(Col#7;-qOdrKP>i1zQt zJmJ8}$VWvv=A~cPthq-_YfEfHL@K-8$$n%%Ob%EkZ7ekrq zV$g^w8Cy0xNd)d>kMJMg^wO0S8`$eljzWSR_@9KKSCPj&L5Xe>rZ^fJ6+s1e{Yt^t z(dL0Tw&o^nVlLo@d;a(?fk16e&=WE1!yi_lC|EZ(8AZJ(JQe6Q$b(KqT=ql z=i^O4u$V5W&zIMd{!r5azwsbC8F^?_Vc5<%(&!El$v5eLACmPGp*v$I{KpuoKkDLN z?-Wymjqp;D*$o?g|7@Ih1oV}cJsm__3pY1|s-;8)`%DtU#|KMlwOG5OPU90@oDSvaSi`aYumEi#%#+ z;AA0lBHJ`?o2LV}oCM~2Bq7;~NX`T{XsHwEv!5gM6&j(nW8#xJ+TZaI^3IT+LQKzt zAJsOk2#>_gA1Yjb@cgF*_M?*L=JM_zqO4RjZ)}Y_kt&wpyaW4yJTH`-KDn?T;V3uf zcPn58vzUFzL$MKRW-1s=!xp}NP+jnY9z5A!QIQDVV=_T72R;NuJl+DbfebJ`W#;3XAwB5cy5oFW~kI(OX1d zEKJuZa@234sCptt#%>`)>W)z@1azbc#Mj(I*Ra>31Wb|w!oceh5C#p1AbLJA10taK zGH1~2!Dz*u&q#^G=0&jip5TxX2+-P4!jZO~QIAjtUhkd#6txS)kXOroB#GkgJ>m|m zg+c29wH3EHA+-`#`+;&;_&qU!!dC-2seoPv9EMmZATJ*-K<%biz3i2J_F<#u(1Cc) zcyj#vP;0L9T*Ms|3KzHG-CgTQ%$yx}&!{~yFY34+=x9#9)2T7~>P$q0xSh42WQCYA zUnFfiO@d@bQ&!$i+cJxP%XXptq{efi0^in$>h3SJzb_r^9lScgp{}ey^E68~tG8H( zv}g#_lc(NQq-^R-b)n}WmL!L6Ce>+0x#>SO!6$yD)L>J z5z?|>#cuX7Z3GRHtsmOHT{wK3rlFysXJ@yb{e@Es8Kg*k7Xl>%sO5Q}l1oR;gtvA= zys&65{Asda7J0S($G*~?M>=2&t(x~}h%T#uiOr=P{B_iM5yM6RrpVEpLA$&+d3~cB&$%6e7 z6bNr{VfTPiZEqQ>nsb0E@|(NG_!*DDS}q<{lU!9)#kxC*s@)a?viZYccni-ksISaH zmO}+5EkFnv3RSAwH|TVz9Kn8E1WdR+>F8Vw$($-y;$>37JPc$1Hv-mGvHL&)0`q@vl zZ|L&PG5df@WxyOLZ=bf|Ga4JfMDodQl`{QbS&3|Pfi#nkBZ!Yb3k^Jxw#5ToAHJ2~ zZ$5<`fcxroUbk-Upzoff!3N0JJl7tE1_d@aHsq!rKmXHy208fZul!R-fbdbywzip+weae8=~| zEU^nl$ZkmukkotC{>mKpuEm>(Oj!_6173oF1ho;ZO`nF2)=rqaleqp{g4+@XyF~*H z+i;z-jgp?CxjVt;>#&M|ju$@{6-5`q{0(LsGr)dCMfKK*bbO)HYy(t){HFxOjAGs%*7-Bzv^i{Q{)z zzY+O=Y<$D4k7q}N362MG=~V*B7UC)Ypt~r8YEWw5k2GX;Pc{g(?BUe+&y4rrKIs)w z!h%^19dy~2CwQw<0_F=t`c8DR_ZJ31UFNo(g>y*_xb#{y!;@9dM*FBf2;**>94bJ= zg(Q0Y)~#C&0n2_Zt&0Fb&~y$sKrqAzZ#|;_d;)o(I&++Nz+G1b-S{csR$;Z}c%zmg z0NXvE({YV;UO}`3w;8;lHyL7bmEr2u+ucS*t1BY$m*Vg?d2mu7eoh0NQ$Me~idA2* z`#7#FIL9qmsMzZpOldYCU z;jOq&aPkJFS@Ut>D#K2%7lx&(P8psI8DI;3`wEj3@-DZKv1ZR&Jz5z%fN+J z=HOd;t6ehw;UHxs)Xax%_;B3@lnf|<|NN6%DYN5A8(Zg5fK*8Re?47;p0W{LvN9Mj zWqe^fo*fmTotGqfvWh1=f(^?dP83c~;z$DO9yYIBg{8x;;wfIY#>Y(JvI1$J#~ktcP1yG@EqY)LK)te$MY9sEne39{=`LS zGe{Bs%wO#RQHX7yIP!3k-HmH4@ewO3<1 zgj`Gt>$prJ9tJ5x+tA3_7Pb-I1}PuE(ARCcMadZ{KgPTUO0afiV;6EN3>qis+=`1M zejnZYvaCHiSK`r07)-!eEx(SvfFEpNFWHItF=IT-QO+58VT^ej{ZXok{gDV;J^~>- zT$Im-C0$bxh^(6y5(XsS-kXMJ0w__ipPezT$r-&1KQ>z*$eNtHRJ{!sy}PvN4ZZbq z^>%gug6L=a)V`BFZ%l`LTVniH{`>durC{K5{KjOABkf%S4>mtsjNdf-65J$QZei_& zZ^1{@k^>rIfQERkboMI)V~{;*r9TE}Z*NNzdpbx<59DXaQKN~U3dL$>xlz<3wF2JaIj zNA@FHsL8O|HSgLtthYCe*|*eQ0fuld#tHZCXa2wDuKSGW~hwgXwjl5m8YnP6iUh{qe-M~g@{UJCnMvW@B6wB4xaviuOE8JeeQE# z_jP@)&%WMo`f~AduCnbHQ6f%q{=~tL+8T`{5dmJ707y_Ke5sN&$x&pLXze06PvR@R zjQjBNrRr*MSlsR#(HinC1Xz2^sbqS!D6BxLr!drinU?~czuFAz;L|ybqSyNcA`Xl7 zSb?}T=Se~ca#4s^vumcvy41lxfS5G&94`W464BN5 zjvw5$8{Pj5-q&7N+STMc9g}nW+m%sds)X-gcv?p)Ygj(waip5dl#9c3X7YMiNDfjU zz3!PtM@0LVLLO94Aat+(tSKf$5n@L`mOb^pT;ob>C@7J?2ndeEaJL>RxlRTuu@t*4 zfxA{c{{U^Xzhg45&R)q&C>+7W!-H)V{@CT@VXD#vcoU>6>ElL~ZRZB;^`E|ko-wjn zbx5X(jL}ojVJM~Ekx}=sitcK73~Mi3N>{ePO(f;xMEM5<1Q=OZjCu27Y8O(L4xZiB z67Db0P8^75Px(Qa(gvTW9XD^xrj49CqkI(AdXLqxpkg8I%-a*EOhYS|zUsv&m!aSt zh7O3N>p^m43wQT}pOKfypx_db-ql6TFqBgwGcBOu zl|^P+Ku{XXFnK z@9npikf-PbFh2~S#JM{c>d+lt$nHYo!$`i6tgjY3auC(!GB%99MR%lnF^Nw!Y-AX% zN97s?*AB(QUw8a55TFM>^(GwNTi&%(=y9?nd>koKG@!dpXr*+o3Ee zIbHvAJONuP2KiBY&zm1obKKF@PwBsi9VtYiE&S1wtu|x^eVDJ^496@pwvG?W$aj%w1G^vOGDhymtrp{tTaaysuu(R1q)fe~Hi zwfPC-?24#>*HP~9F@sm~?1UF2M{t&~HLX-yg{~r7$%{yjvShl^Hd7JG+E3-_oimKb zaX&x;EGuXGuZdzV7hE69kgx%5G+ufqBE7{uM z$NG0J=zQd(&t0}w?}+GX6K2W@WA5ZuEuw9Pga}(@Am&}x_gVx`QT30k2mt4;r+xy5 zzWN~*e<5%oZ`B>K>MtDPc`&@0qUKngv?aWEDxU;Ro&V68{@=6_^I07CW3D={eqrZl zyj9?>{VI$>W%bHv`c3w+RXQd^+SyvmNIw&{(KKwMutT|5>B}a#EYp<`bFf$*#)CSP zFvmRZULNZ}<_+-o{;a{!0EVkec&Ki2t$0Fr#3eC{EytBX(MqgvlYtUF(>hl%)4u1= z@wTl9JCq}XWq37(w+som*p-EXlUA0Y#c{}MZKzlmf8a;Z>W-JJ^6lKtE}KFp4eGku z(x;6Oe&;8MhLqvrMZb7&%D^}zczqUf5^LC@9aB-JI@;I~Ne8*Y^5}Z^-@Ueqr`WZI zW$qO&`&{b4n+8TIaq7A{qn-&~5EZ~W?z?@Cw};0Qzn`u)K%Ll&hPAI0(C9eY_;1%Y zhoNBLzhZ+8&nMFQM<+viK-*)j44#7{k%*qB?js93N9g262uP;=RBrgD)ewR#CA|s^ z4$_llt9Tl9u!y{(aD7?G6@ia*0zvo?Op-)gS4Y{^DgMHOlC0-`vR38aQulq=D=s_9 z@-4KWV+;nb->j>%chNsO9nZaFGSR(Jt8Gky2Q<KG+ zzO|3@UQcU?=e@stO(-BOObdaF%BimK(^}&$?M1iu@n^(~$q-U7iE-C_0I1H8OYw&Du8o0pORm6V6o4i~j|kAk?OX z)qo=)6peOE{{u%bhrkj1-su7s06pO4-E^GuUzi8N_2qY$Q2JE>>*VGU-oZ-#wD?>n zTt|9aOMyEnOUN$nUHuC%0)ZRRaRGQ(3jlUGyZ$ngj|q0%2h|`8sRKg%>bpDsf?5Ii z8@0P2-E2-EteB%6_=Dh0%-i9Kcz)@t)QVGTmL8xSd7%9lpogTSq?>m!67zWb&yQ38 zgP*ZOlmMb_Y6g(hb?Ff!PWf#6FKPu4gy&3j14%8n4UA$O>A$=VV2s+*;?f!~)pg`3 z12#Bh*IeE7CdT)m!|usy`XW$9nmryf{3dtZ+O zhe|HttxJ6$q4!Uh?dZ`76DI55FbKZ|vZS;{E>~j#4{=!o^b<)wO_98_;XUGsga72J z8m|6%J-`ciI{y{}(nacyi9pY1c)v`T{04{{et?>^Y-*%J0jV=+-tGc@)y*2a`eMx^ z5~QyJWk|mcp9HKH!E1B&sK)88UJ?x}<0As-9p@s5zM1DZoMHYJV-dGiH~+80EG{lK zLKEGOX5hMRO!u+^+{Mu2DWde@5dODH%w!7;82D0Q`t*)hl>+I&#BJ9jM6_FmRjpOy zYf&;&a?OGu#I36!mrLEpQ75oBUPAaUpZwdm0_RCX1;8UOLIzQQ(b;$u*yhn{@I{_} z{6NRQ{8#gdWxBYyzvT zk*IGBM}`7u=g$IwpF}OcX;jktlbEtrQF65vS9iGsU=XAggyUw8uk!I%v?tZ0>Y4G3 zg|4*^-9Ue|I&C=h8`!wHz!|qYZhSMjpS(0k5m>F8fYqvP3#T>$7k3?jAXZ47j)o-K z>U59yt)$F+`!Yh)^wnb;l?Vhd&Nsq$yEEP_w;w)_Pi#7Hj9Ly>f>SaibvQ-npHE@Z zX8d)iU=s{h18!OnhD!RRJHXY^)BqU44u$j`Ip*{ssh*)oEMWu8>5@n5=c9^x=T$ye z2MKyTr|}D0M+6J|EfY+UF>P0GRBP4507Wp^1e_jA!WQQ7 z!l|A+e6A1($xo(9j@DIT11Z2{D_h*&_^4#&-nV=vHQJc?1QM)eR?5al-U8BtDZp+u z`vEnkZPDMpG#w}~o(paAZpm7zP?_1;qX{wL{=a^yrRAq|0Njnh2W7N8-JeB-c(#4P zwp7rLvdQTu5HccwR4jW5P)b8*Pz;CQ9-WR5X!oKtpw1ka>G+neE%NWepn)$q6bu+w zI3e?z!b=7HTaN?TG#ZRpKcY|nw3yvbJeLWAq~^dLf_s`elaj&Xz5vAVTSn&6oBcj% zAPDv(rSrf)KW{gff|?t55k#I(b(~{(u}sgJNK7P}&H*5Q9&Mv(9)>we)2A0Z1NGE# zX54>9e29tnc|Al%CG^La7XdBqTPXk5Y(i}M2D}0`%da}E=D!|UMz)MdF(7-6LP@T# zd~MFj3Zf<}A%C`i52!B2ZuxSJ&YkuwxE#V307S(oDk;4TNY2_s@X9=8hkzY1S1abno@k z5;B_Zt)+lFZ_dwoUqau9Og5TeZ(-tUdIeK6b>HGvR>r841O4T(FoXwgvCv zJO{)_aRG3Zp8+mCYX}nbB{y@JEFOrgh zbrs*9bFu0DEFsEI{g+M!I8SN_DVfZNWx2crYtIstD-Z$^y+K3A6qLW15>mYGIwj}e z1Ui-bv6d;x`z>ZRp>M9&SQy?4Z^BmwjRoGZt;3x*rM9t@FG06oEcs$M)zu5o+5u9k zBUZa-vUus?o`(sCnGhQ>8#cnV^qI?ywUmEQkg7WfVP1zo%r{s4>5I<v(FRsS)Gk6RGWfk|WV;366%o&};3*q@MKlK3<;GC~Y0Q8+CBqa1K+%)=w0CRJL zmivcL)1C7i`#(b8g{`z9x%y$CY))DjMeqk-BQkJZRhWIBm-p*=Upf6z>_rC$uD-11 zYI}n-JqEa(uRc2nb`lkrv@|2i@1af%#yHzAA1)Aoxt220OAK*!gkRV zUh<}?&-3x)%P6^2QY|O}Ju|EkSKscb4<&)~%HQ z?*%_{1=>+93E-eoTHwhNUJ9H)vw;?6A8leg!|w7iGGGNl$1W*g)gV5e2;3OGe*5uzH~YPSQLlCDQtyGQQv>V*jkj5 z3i0iTG!Ut+20#Pki!!zkL8-4zuX(>V3iUgv^rLEi>+^o&NYSpgUMX7+F z_c=7ZryBzI+yJ9cUWu#69x#gui-_|>0N<;wf$7$ZJD|6V@}C+7dTcRZpSMlA$*kg! zgor#Egn$zs8;Kv*toAhrXbVsVIx0^G=trqDrjewvxR?E7j$weA+Ws5Z$j6xJ>+7F? z=^22q=vX$KG7mUqH35IS1uYeM%I5cPEs1kmtxKtZnziorh1r!M5R|iLngUL4)aWZ2 z5)2yNxC3aJoC83i)mrJIG0lxxhbZY4iIP^kJj&zMRD0%aElvDK$51PmvZjeqwTKy~ zfQa{S+M1Bms$T{|F>Q&%>f30tXVW(I(aLYnC&B zu2W|j4O!ohGUw0+!htI50#4%!V9dIPMFrt5->Y>%9+yT2OiTW6_mssvsJ*Dc;&fcO zQ9bc14rY6DqCq!i)&@2J{dTF=~CD5SS!K z;gzborI1T4P-`?7a=_lJ4H#S|5~IM>D{ z3UrDCLSR6H)+Ocyom*o=`4<%0F?8~G)buq7i==e&Rk_@q+@0bC`Q3@6MMl^b-H$fY zA9O{jub}%Yuz{)0(FWIWJGShfT*DfNE6(r9jv}-v#{u}c0E_zf1|5Hs!(X z+x9z>ZC>b*UF%wb$+IGyH#vJ%cWd=~%rwI7IA%|2YG)0J)znJnZk4BW<}9!;OZSH1 zC<&=OQbTu;_3S6ctVt>m?Mg661cdl@&K^OX0IFm&Ov$_$WCDM#nqy1DbMeOoxK?;C z+?XvHr&PhS-C)14rE_!a3Ao^bWG}La>)rzG9_;(6hOXZ6pY?8n{*}dtZ^0kvtj3mX zy&1oIAA9wzQIz~oM61~Zr`9TH8r9w}WHJ;Z&L2p zG$_bg^6@ABW5_JZNeP;4aL(QC(e``)ZY+;p*yheNF&7$O7k0qM&YQj=Iewk)ahI`u z&5!VLCbtjwPdaxe6oYy`X(C%y^ey|rVT-1%sz0dR}BqAl8FeL;CgwSlz!Iu)P zc%hb~r6C0iGyQkH;c{GwRhLO^)Pb=zi_h52ZrAbcP&xRh1>g(#2Po!NW zL>YgMBk!Rnb|C)>gwzMX+W!JQ{o(0FgN*vF%G2c}Xnap676*?#I+?bd*2w4& zyvfUGVxRGAo70(NM<=Hu5WRlBtFm|0MU4MK>sl`mJ$1Ijq3oE-lvxurs<%0iSj+yF z6CCmrDAiW0F3g%#4?M)pj4NI|)pF`PVv)y?i+4VKl5|w;p0x$TdWBUBRat zh~(BM*?NOCLrWm5X*Ut+3no&Uv&q|-?*H#ms@hA+nv3j8TfyBLKg3-1%3z8RJnSIJ zDrB0%mldSv@@m-K;__K)M{ZQ^9VGutw5+oNlQvaN2*iP!{ou&s-rxG-Ox;*5PxW|> zLp8S}`$H0vg)JibX|i*!GR@puvmmK{m3 z{KTkEoVl{q&x^bUG0eikE!RB5CUOCn*k}A*+=jy;N zU6D@}N6J^|3BnNRvm<}>Es?#OstO7Us9+@M*AQypn^LE|!VB1mf zrt4Yj5>{?AKDCwd4;=%Y>PnckTVYgVGY3c0UAsNAC$~y)hFN@TBwQB1)ghXl(wm|l zgxy;{jnXtRCSXJfo_ICDuirgtd zsPp{^(5?r;-STk3sm=_@#b7mg7HPSDL7ej#`M~fzZ&p5?p?>=}NzFr_#{cNmOgVi(;qO8hX(dNxvR}e&DRw&-4#{Gpy3m zk?SH}C87AP(TGHuhI_-!3Wvi~+3RoW@P&7x>m;oXdFG&J6~f=?Ll)ht_^l)ov(LA~ zpWEjNy25K$b4?4Ahfx;kb%?Sqr@%dUgoJ4My8De^`%prM8HKJmt5E6W=m@=|#nLfT z9r|e7IQz71IwZM&8@9p|R$b(g%voMfO!xDo6bv)7a&l&XR{p1610yz_cfGXKA(!dU;ESVXQDOi7l|H;8M7TqqgD4jh@mH^s_JTuXvJyrs`K{q_T>si zOb}p-Y?}#6Oa($1k%LTG^0_j{pq_hZJ$I1-`)`8Kz4{gCtuC*4f9^`jGD`pP+ewSP zP=W5;25#iVUXmA+`p`4o7+zm*wQz#~bJ{~ry{^|%&oQCHcPRDFuroy&lCte~kO5gc z&dO7Y)YPQhN|VW{|CA9nZSHKZNg5q4UIyh!|MmzEez6{|<6gwFwXOYCWcy%7{_BpQ zq3bC9g|S+eC6M4r$e*NCpqLVr=v~yk9Q3?}&JiTb0{1S+s>x(jyjGGiW$$b6TLkr( zuo1z42ND|3?hHO<+k1@Z&lpjLHyl&Z)g*QtSs1HoGZt%;fOw3x`bbw~R;$r;GISwm zg{`eEvTFJ)ki7sK#ccOH(C;Rp14dTxJ2{#gWZROVh7!U2JxKgM5l8A)NhI5pcOtBN z+BtIisC_z>w&f*kBeYt>1LM$`ob|0G<3SzRS0nJ1s$hoEkP3+JIw;vSK@v7>=m^$F z5NEwW9{JsqJ4bX38p*kV$>&CrZHaUdx0cu>o$|P@HaXsynbi!l_#F*vQSxHVM0qr0 zbH4krQ3%LKx*&(|N6erfG{uc=XM*mNqxL;hyLtQ@6m+ktmh63Bff&(UeZ-K68_4a7 zhAt|wooYByw9FIErkr)lR6~5nq29yB5w7z$e9AnV`YtvjqTXuO{K=M zmL409-M>wPJ9babgh`}m;tCD=CkR<>v5C3snHPrvB|!pS&(0K>foV8&HP#^UWOyGH zd4P?B6Z!BxW*s2u#3iMxn4{5~*ca*KD zbX7=%dZ%7MdLR-(XV`!|lTF~a=sqNBY^LyDP+Z$V2TWcC3cw4@;ll$aFsP9-zXyn` zzu5!#OUo+NeSgv5-FD;{Y4@ZGZ(T+Bt?Uprbhy3NY_HBbAvNs%Ng-pX~JfUG0yW>io<4k0|wc5cDdM^@uRxN9gK`=C%cke|II!Z@sN`TJt z&ofA79rBXxQX`d~HTQb7gpMyEH`Yvs+YmzNbzIlQv6rqylls(Q>vk!RBEA%yhF?!W za-OnimunosXX+XGJ4M2#UmuPp;H5O+d~T2akwY&3xd|6pZHKSD{qwnFF9&=8Mi zQ)ZMwpOmiB>aw_OrR-#MFj?7wFs?;ZRFqQ}9$g+ogEQT@aOfU5ZtJr_g;d%I+FcW! z$ByTEJGRV^JnsX_U0G_x2+t`s!A12psRozt`f7^PZK)G8f*ve*S-ncqL0QvvFj|G2gPTO_MH@# zIqCrU|4dnP-dyL~^;P`!St~`yOz_cTlbb)(BXQzGd_0*=Xi)L+oKZX))>*|W%@UOtg{bkDIq(fHyS3LO)Ro-Xviz(>!j-+f z?p-YR21VC(gz{MoE_#X<_@1MB<(t_lE0oa zUq(7=?$#T|yc5HcPS~p?88WZ{i4$>4kEYzb)X{Q_59adPw8^5f=ZMLB=5g;A*%lSU zc@8;H6|${kxGj(+$J<^%2R@_v*^t6l2{0;V8tdj3^m0?tV@&i+D_9QeZ=z7EF7s-r zVW%@**5~p`1V9|pBEjl_6YH$*P=08^^1ZJ<^4hD(T>DRFBW-Xg4b}l^v7EVDN=S_O z-*S%rXN-XOkw+Jnu2A^XoRHS}9wEK!OL(C_!?zAN-pk*WI6(Nk$C8xxWbRLsw6;ax zgWsm-<0#B#`}t59`MXf=(R+yRPa@9b@Pp^>_42fsA`BAqw!!T+dj}71`II&PCWbQ- z66K%b1ti2{dH>>N!4c7kn z?)g&au9@l+i0&#xT(ayT^po?Jf>as_lo!iAH6x7HgJ7aHw~#EH-(2m%!P3-I`|rNz zWWi}&)q;vJmoU|}KoS))0v8qWMRfB-y2}&VKP3^#Q*O%Ysf}hWATaI4jm*Pv% zfaI?;;dD4fiDv6+FES*#?mO7kd(|DkWLnRxe8ZPLcBaUOD5I*oc*kg)g4>@fNP&NB z(!jZfoAF(~H?Deo_8B&DTnUt6BE-nj6{FpuN}uQN+A%XFB9X>5hJh%Bo&Q0&e$|Iz zy7$pXa(+!>L(6iq@fJrAYI1hX;lGFQp>?59e`z+j?Io#s#Xc@%N39!=Lq>;pW#*4T z&aVdTTUvLpm)K||b;g;!q%HQFO`Jc4;SS2Vg z_p6Ru;JEeigSylrH(12n@*APN10}QtdL*5Q0A0@l{@8Jw+^D&pSWLS?X`7|C;e@!q zNRxlf0>oTJ%}7=z;O?p2CwNQC!AtJhj#)6DdtfZ@ge!TqJ+j=Px;6%{*pqwfm-R+d=7hVEw}N_-!V^y{Ap(#Np4@X*x(I2bGZ za_VENOgI7;WuZ*!T=_m=uSo9IAQlzm!|rWzXuS79>b)bW*|iu9yz%`MC@FFU3fGoMv)QneP5NkcWN z9~~u!ME=nfad{Rc7VI_X?(Nmnw>F)UnOWYR)?QM#Y!95p>kvUBVbedrY3k0i_&c)E zGyLyPtctmbl}A%NWV&Pe*U=y8V~_=>G*eDie*HRRKAKQipvp7J$MPHM)z!N2(;*V- zY(!II?I7AwWMM-vE04>JBCmG6l2;21;`!$pezL6H&{VEu>wFBDqPOs%K*)t(J=W;V z6n!)~G~(+1Som*^Bsy*-O?2{h&WgiYw+V<_t!% z0Xx!VW8|I*!p~C6g>`DeM=m#aG`{A#9ZR^Y&K85p9CPA?7-&1MH^Dw}vXM+E^c(17Xugr-? z;z;p*M<3}xaE`YhAq~I3_SV`d^L0S@utazVT7pj|1S6b?Za3u$uQa^Q-STaBP%9LdM%#UT=M{!Jz#2zFy>`oYe z?WSxpT`AEGva#lH`ayBcf=)P)ThPp3um#_iKW^dPNDGDLJCl3tI$KNxS&m8z_^6+5 zB7^k{<%w0E-!5!7_Madz0_t`BtbVy0X=#z%=S-}zz~>6-PdF1WMk=+Lpr&Im(KVtnQ^7bu$Z&Wz(5GnWZ@L!ILLKdh9)3@FXVzN_D^Us%YmW*12==kc=*i%L5zEL7` zTr@L&w79nHr2kspKFyFSs3Qj$}*xBf8JO|9UVF7m)a ztwwlm@;-twLF>SfE>B57TIXSj-J8%yZxJom#F*awc0|D_YI<7vF~x19SZkQR(qtU_ z1hnj|Wp)*}MTrtJ$zWoY300Pjse43#-|kodS;_aFqKn`(5s%MNTgG19rb3WnE@ zcn}E)@=DPVbI*ii)9j%VJUz8C53VyWgC${)_Vs=|RRv1?7}7=6PLLxdlBE2Bs$33g zE@pM$NzElqa+)VJZglTQe_OgyJbJ@}<_09M#}Umcj1%2pir@-&l3{m<@)MWr;`pJ-Jo-~`tF#fL?= zlV6ez!dJ+W%Z4a4>jmQD>%&)^Jw<-e$6s1d7vgk^N-0z;pvCJHvW6TbLy2RP5J+;x Yv%?nK?>QseLg7bu=^E{r#T&i<2Z(>~BLDyZ literal 0 HcmV?d00001 From 33c707c658d799321af7f98c2413b0e65e65d582 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 15:05:05 +0000 Subject: [PATCH 329/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.119?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 424a70f3..42e3846e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.118 +0.0.119 From 943e510d770e0de31a16db849a240bf6984e78c0 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 23:06:59 +0800 Subject: [PATCH 330/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Remove=20redunda?= =?UTF-8?q?nt=20logo=20routing=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/main.py b/main.py index c39a1939..d737b1dc 100644 --- a/main.py +++ b/main.py @@ -2133,15 +2133,9 @@ async def delete_row(row_id: str): # return await asgi.fetch(app, request, env) from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse - # 添加静态文件挂载 app.mount("/", StaticFiles(directory="./static", html=True), name="static") -@app.get('/favicon.ico', include_in_schema=False) -async def favicon(): - return FileResponse('favicon.ico') - if __name__ == '__main__': import uvicorn uvicorn.run( From 7534fec1d65ebe1b6bcbec7991c39a061cfb6c0e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 15:07:26 +0000 Subject: [PATCH 331/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.120?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 42e3846e..2f29dcd3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.119 +0.0.120 From 5bec9fdb76dc9206b9b5abb592b61170894b983e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 26 Nov 2024 23:16:52 +0800 Subject: [PATCH 332/476] =?UTF-8?q?=E2=9C=A8=20Feature:=201.=20Support=20o?= =?UTF-8?q?1=20model=20streaming=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Support felo o1 model reverse API --- main.py | 5 +---- request.py | 63 ++---------------------------------------------------- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/main.py b/main.py index d737b1dc..c18ae9ac 100644 --- a/main.py +++ b/main.py @@ -834,6 +834,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if "claude" not in original_model \ and "gpt" not in original_model \ + and "o1" not in original_model \ and "gemini" not in original_model \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': @@ -845,10 +846,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if "gemini" in original_model and engine == "vertex": engine = "vertex-gemini" - if "o1-preview" in original_model or "o1-mini" in original_model: - engine = "o1" - request.stream = False - if endpoint == "/v1/images/generations": engine = "dalle" request.stream = False diff --git a/request.py b/request.py index 4ebe9724..f19efdad 100644 --- a/request.py +++ b/request.py @@ -618,7 +618,7 @@ async def get_gpt_payload(request, engine, provider): model_dict = get_model_dict(provider) model = model_dict[request.model] if provider.get("api"): - if provider['base_url'] == "https://api-ext.felo.ai/one-ai/completions": + if provider['base_url'] == "https://api-ext.felo.ai/one-ai/completions" or provider['base_url'] == "https://api-ext.felo.ai/trail/v1/chat/completions": headers['Authorization'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" else: headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" @@ -679,7 +679,7 @@ async def get_gpt_payload(request, engine, provider): if field not in miss_fields and value is not None: payload[field] = value - if provider.get("tools") == False: + if provider.get("tools") == False or "o1" in model: payload.pop("tools", None) payload.pop("tool_choice", None) @@ -869,63 +869,6 @@ async def get_cloudflare_payload(request, engine, provider): return url, headers, payload -async def get_o1_payload(request, engine, provider): - headers = { - 'Content-Type': 'application/json' - } - model_dict = get_model_dict(provider) - model = model_dict[request.model] - if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" - - url = provider['base_url'] - - messages = [] - for msg in request.messages: - if isinstance(msg.content, list): - content = [] - for item in msg.content: - if item.type == "text": - text_message = await get_text_message(msg.role, item.text, engine) - content.append(text_message) - else: - content = msg.content - - if isinstance(content, list) and msg.role != "system": - for item in content: - if item["type"] == "text": - messages.append({"role": msg.role, "content": item["text"]}) - elif msg.role != "system": - messages.append({"role": msg.role, "content": content}) - - payload = { - "model": model, - "messages": messages, - } - - miss_fields = [ - 'model', - 'messages', - 'tools', - 'tool_choice', - 'temperature', - 'top_p', - 'max_tokens', - 'presence_penalty', - 'frequency_penalty', - 'n', - 'user', - 'include_usage', - 'logprobs', - 'top_logprobs' - ] - - for field, value in request.model_dump(exclude_unset=True).items(): - if field not in miss_fields and value is not None: - payload[field] = value - - return url, headers, payload - async def gpt2claude_tools_json(json_dict): import copy json_dict = copy.deepcopy(json_dict) @@ -1213,8 +1156,6 @@ async def get_payload(request: RequestModel, engine, provider): return await get_openrouter_payload(request, engine, provider) elif engine == "cloudflare": return await get_cloudflare_payload(request, engine, provider) - elif engine == "o1": - return await get_o1_payload(request, engine, provider) elif engine == "cohere": return await get_cohere_payload(request, engine, provider) elif engine == "dalle": From 644ed2c9e2812d4259f5b9125f8676404f48489c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 26 Nov 2024 15:17:17 +0000 Subject: [PATCH 333/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.121?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2f29dcd3..674d1095 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.120 +0.0.121 From 98d75825a590b699e1fd2391e7ac106462a618ac Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 27 Nov 2024 18:10:29 +0800 Subject: [PATCH 334/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=201.=20Fix=20the=20?= =?UTF-8?q?bug=20where=20the=20OpenRouter=20channel=20cannot=20use=20image?= =?UTF-8?q?=20Q&A.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Fix the bug where the grok model was assigned to the openrouter channel. 💰 Sponsors: Thanks to @PowerHunter for the ¥2000 sponsorship, sponsorship information has been added to the README. --- main.py | 1 + request.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index c18ae9ac..2f69a6bb 100644 --- a/main.py +++ b/main.py @@ -836,6 +836,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A and "gpt" not in original_model \ and "o1" not in original_model \ and "gemini" not in original_model \ + and "grok" not in original_model \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" diff --git a/request.py b/request.py index f19efdad..abe6d818 100644 --- a/request.py +++ b/request.py @@ -78,7 +78,7 @@ async def get_image_message(base64_image, engine = None): semicolon_index = base64_image.index(";") image_type = base64_image[colon_index + 1:semicolon_index] - if "gpt" == engine: + if "gpt" == engine or "openrouter" == engine: return { "type": "image_url", "image_url": { From afbea4f17bfd5d9c2ffe1a12b2225856f186c805 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 27 Nov 2024 10:10:58 +0000 Subject: [PATCH 335/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.122?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 674d1095..5a05f0a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.121 +0.0.122 From 76396bf5dd14e8d254b661c700b165ad8e97e8db Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 27 Nov 2024 19:33:51 +0800 Subject: [PATCH 336/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20grok=20cannot=20read=20webp=20format=20images.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- README_CN.md | 4 ++-- request.py | 20 ++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c0a84de9..6aed1de5 100644 --- a/README.md +++ b/README.md @@ -373,8 +373,8 @@ pex -r requirements.txt \ ## Sponsors We thank the following sponsors for their support: - -- @PowerHunter: ¥1800 + +- @PowerHunter: ¥2000 - @ioi：¥50 ## How to sponsor us diff --git a/README_CN.md b/README_CN.md index 2ccc135a..3d39e24c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -373,8 +373,8 @@ pex -r requirements.txt \ ## 赞助商 我们感谢以下赞助商的支持： - -- @PowerHunter：¥1800 + +- @PowerHunter：¥2000 - @ioi：¥50 ## 如何赞助我们 diff --git a/request.py b/request.py index abe6d818..358e2840 100644 --- a/request.py +++ b/request.py @@ -4,6 +4,8 @@ import httpx import base64 import urllib.parse +from PIL import Image +import io from models import RequestModel from utils import c35s, c3s, c3o, c3h, gem, BaseAPI, get_model_dict, provider_api_circular_list, safe_get @@ -78,6 +80,24 @@ async def get_image_message(base64_image, engine = None): semicolon_index = base64_image.index(";") image_type = base64_image[colon_index + 1:semicolon_index] + if image_type == "image/webp": + # 将webp转换为png + + # 解码base64获取图片数据 + image_data = base64.b64decode(base64_image.split(",")[1]) + + # 使用PIL打开webp图片 + image = Image.open(io.BytesIO(image_data)) + + # 转换为PNG格式 + png_buffer = io.BytesIO() + image.save(png_buffer, format="PNG") + png_base64 = base64.b64encode(png_buffer.getvalue()).decode('utf-8') + + # 返回PNG格式的base64 + base64_image = f"data:image/png;base64,{png_base64}" + image_type = "image/png" + if "gpt" == engine or "openrouter" == engine: return { "type": "image_url", diff --git a/requirements.txt b/requirements.txt index 6611b75b..9fe9f05b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ xue pytest +pillow uvicorn fastapi aiofiles From d0addcac7490573a1a0240c3506c196baffde64f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 27 Nov 2024 11:34:12 +0000 Subject: [PATCH 337/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.123?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5a05f0a4..dd5a402b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.122 +0.0.123 From c3f97d2159e1f9f50b27af6ebe4c6d34ed1f0bc8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 28 Nov 2024 10:45:39 +0800 Subject: [PATCH 338/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6aed1de5..9edef9fc 100644 --- a/README.md +++ b/README.md @@ -434,7 +434,7 @@ All scheduling algorithms need to be enabled by setting api_keys.(api).preferenc - How should the base_url be filled in correctly? -Except for some special channels shown in the advanced configuration, all OpenAI format providers need to fill in the base_url completely, which means the base_url must end with /v1/chat/completions. If you are using GitHub models, the base_url should be filled in as https://models.inference.ai.azure.com/chat/completion, not Azure's URL. +Except for some special channels shown in the advanced configuration, all OpenAI format providers need to fill in the base_url completely, which means the base_url must end with /v1/chat/completions. If you are using GitHub models, the base_url should be filled in as https://models.inference.ai.azure.com/chat/completions, not Azure's URL. - How does the model timeout time work? What is the priority of the channel-level timeout setting and the global model timeout setting? diff --git a/README_CN.md b/README_CN.md index 3d39e24c..636582e8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -434,7 +434,7 @@ api_keys: - 应该怎么正确填写 base_url？ -除了高级配置里面所展示的一些特殊的渠道，所有 OpenAI 格式的提供商需要把 base_url 填完整，也就是说 base_url 必须以 /v1/chat/completions 结尾。如果你使用的 GitHub models，base_url 应该填写为 https://models.inference.ai.azure.com/chat/completion，而不是 Azure 的 URL。 +除了高级配置里面所展示的一些特殊的渠道，所有 OpenAI 格式的提供商需要把 base_url 填完整，也就是说 base_url 必须以 /v1/chat/completions 结尾。如果你使用的 GitHub models，base_url 应该填写为 https://models.inference.ai.azure.com/chat/completions，而不是 Azure 的 URL。 - 模型超时时间是如何确认的？渠道级别的超时设置和全局模型超时设置的优先级是什么？ From a13ac27d32983115c86d27caf44e03a2b7682536 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 29 Nov 2024 10:59:13 +0800 Subject: [PATCH 339/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20API=20key=20to=20use=20fixed=20priority=20scheduli?= =?UTF-8?q?ng,=20i.e.,=20always=20use=20the=20first=20available=20API=20ke?= =?UTF-8?q?y.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💻 Code: Optimize log display, show the API key used for the current request. --- README.md | 2 +- README_CN.md | 2 +- main.py | 8 +++++--- utils.py | 11 +++++++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9edef9fc..f5925f11 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ providers: # gemini-1.5-pro: 2/min,50/day # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. - api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. + api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random, fixed_priority. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. fixed_priority is fixed priority scheduling, always use the first available API key. model_timeout: # Model timeout, in seconds, default 100 seconds, optional gemini-1.5-pro: 10 # Model gemini-1.5-pro timeout is 10 seconds gemini-1.5-flash: 10 # Model gemini-1.5-flash timeout is 10 seconds diff --git a/README_CN.md b/README_CN.md index 636582e8..bc8e906a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -96,7 +96,7 @@ providers: # gemini-1.5-pro: 2/min,50/day # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 - api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡。 + api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random，fixed_priority。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡，fixed_priority 是固定优先级调度，永远使用第一个可用的 API key。 model_timeout: # 模型超时时间，单位为秒，默认 100 秒，选填 gemini-1.5-pro: 10 # 模型 gemini-1.5-pro 的超时时间为 10 秒 gemini-1.5-flash: 10 # 模型 gemini-1.5-flash 的超时时间为 10 秒 diff --git a/main.py b/main.py index 2f69a6bb..febf0e48 100644 --- a/main.py +++ b/main.py @@ -808,7 +808,7 @@ def get_timeout_value(provider_timeouts, original_model): return timeout_value # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, role=None): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -870,7 +870,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = provider["engine"] channel_id = f"{provider['provider']}" - logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine}") + if engine != "moderation": + logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine} role: {role}") url, headers, payload = await get_payload(request, engine, provider) if is_debug: @@ -1152,6 +1153,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques start_index = self.last_provider_indices[request_model] auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) + role = safe_get(config, 'api_keys', api_index, "role", default=safe_get(config, 'api_keys', api_index, "api", default="None")[:8]) index = 0 if num_matching_providers == 1 and (count := provider_api_circular_list[matching_providers[0]['provider']].get_items_count()) > 1: @@ -1170,7 +1172,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques index += 1 provider = matching_providers[current_index] try: - response = await process_request(request, provider, endpoint) + response = await process_request(request, provider, endpoint, role) return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: diff --git a/utils.py b/utils.py index 2a5e6059..3370fea8 100644 --- a/utils.py +++ b/utils.py @@ -70,12 +70,17 @@ def __init__(self, items = [], rate_limit={"default": "999999/min"}, schedule_al if schedule_algorithm == "random": import random self.items = random.sample(items, len(items)) + self.schedule_algorithm = "random" elif schedule_algorithm == "round_robin": self.items = items + self.schedule_algorithm = "round_robin" + elif schedule_algorithm == "fixed_priority": + self.items = items + self.schedule_algorithm = "fixed_priority" else: self.items = items - logger.warning(f"Unknown schedule algorithm: {schedule_algorithm}, use (round_robin, random) instead") - + logger.warning(f"Unknown schedule algorithm: {schedule_algorithm}, use (round_robin, random, fixed_priority) instead") + self.schedule_algorithm = "round_robin" self.index = 0 self.lock = asyncio.Lock() # 修改为二级字典，第一级是item，第二级是model @@ -152,6 +157,8 @@ async def is_rate_limited(self, item, model: str = None) -> bool: async def next(self, model: str = None): async with self.lock: + if self.schedule_algorithm == "fixed_priority": + self.index = 0 start_index = self.index while True: item = self.items[self.index] From c59c4987a58d056205e3f5c36a84682e67e66d73 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 29 Nov 2024 02:59:32 +0000 Subject: [PATCH 340/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.124?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dd5a402b..8624408d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.123 +0.0.124 From 4ce2a61b77829de3949b36256be82230ad7783a0 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 29 Nov 2024 11:13:14 +0800 Subject: [PATCH 341/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20cryptography=20package=20version=20is=20not=20?= =?UTF-8?q?specified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9fe9f05b..fe5b13d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ watchfiles ruamel.yaml httpx[http2] httpx-socks -cryptography +cryptography==43.0.3 python-multipart \ No newline at end of file From dbfe93f7a821cc7ffc67a42d1814dc90e0102938 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 29 Nov 2024 03:13:38 +0000 Subject: [PATCH 342/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.125?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8624408d..26445619 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.124 +0.0.125 From 224c64cabdd7cebf8bb868b8258c227e4ff53156 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 29 Nov 2024 12:10:21 +0800 Subject: [PATCH 343/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20httpx-socks=20does=20not=20have=20a=20fixed=20versio?= =?UTF-8?q?n=20to=200.9.2.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe5b13d6..b0717ad8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ sqlalchemy watchfiles ruamel.yaml httpx[http2] -httpx-socks +httpx-socks==0.9.2 cryptography==43.0.3 python-multipart \ No newline at end of file From 0d9915a93c5576f0c773ded3893f919a775ed5d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 29 Nov 2024 04:10:40 +0000 Subject: [PATCH 344/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.126?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 26445619..f8ee4528 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.125 +0.0.126 From 2daaae3182c7b57a4d1c742ccf21692cc65dc393 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 1 Dec 2024 11:42:43 +0800 Subject: [PATCH 345/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20GitHub=20o1=20model=20cannot=20use=20streaming?= =?UTF-8?q?=20output.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + models.py | 1 + request.py | 11 +++++-- response.py | 83 +++++------------------------------------------------ utils.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 78 deletions(-) diff --git a/main.py b/main.py index febf0e48..b582402c 100644 --- a/main.py +++ b/main.py @@ -874,6 +874,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine} role: {role}") url, headers, payload = await get_payload(request, engine, provider) + # print("url", url) if is_debug: logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) if payload.get("file"): diff --git a/models.py b/models.py index 97871e38..f9fec42c 100644 --- a/models.py +++ b/models.py @@ -85,6 +85,7 @@ class RequestModel(BaseRequest): temperature: Optional[float] = 0.5 top_p: Optional[float] = 1.0 max_tokens: Optional[int] = None + max_completion_tokens: Optional[int] = None presence_penalty: Optional[float] = 0.0 frequency_penalty: Optional[float] = 0.0 n: Optional[int] = 1 diff --git a/request.py b/request.py index 358e2840..b3b6dc18 100644 --- a/request.py +++ b/request.py @@ -692,16 +692,23 @@ async def get_gpt_payload(request, engine, provider): miss_fields = [ 'model', - 'messages' + 'messages', ] for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: - payload[field] = value + if field == "max_tokens" and "o1" in model: + payload["max_completion_tokens"] = value + else: + payload[field] = value if provider.get("tools") == False or "o1" in model: payload.pop("tools", None) payload.pop("tool_choice", None) + if "o1" in model and "models.inference.ai.azure.com" in url: + payload["stream"] = False + # request.stream = False + payload.pop("stream_options", None) return url, headers, payload diff --git a/response.py b/response.py index eca415b6..2f735f37 100644 --- a/response.py +++ b/response.py @@ -6,81 +6,7 @@ from log_config import logger -from utils import safe_get - -# end_of_line = "\n\r\n" -# end_of_line = "\r\n" -# end_of_line = "\n\r" -end_of_line = "\n\n" -# end_of_line = "\r" -# end_of_line = "\n" - -async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): - random.seed(timestamp) - random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) - sample_data = { - "id": f"chatcmpl-{random_str}", - "object": "chat.completion.chunk", - "created": timestamp, - "model": model, - "choices": [ - { - "index": 0, - "delta": {"content": content}, - "logprobs": None, - "finish_reason": None - } - ], - "usage": None, - "system_fingerprint": "fp_d576307f90", - } - if function_call_content: - sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]} - if tools_id and function_call_name: - sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"id": tools_id,"type":"function","function":{"name": function_call_name, "arguments":""}}]} - # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]} - if role: - sample_data["choices"][0]["delta"] = {"role": role, "content": ""} - if total_tokens: - total_tokens = prompt_tokens + completion_tokens - sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} - sample_data["choices"] = [] - json_data = json.dumps(sample_data, ensure_ascii=False) - - # 构建SSE响应 - sse_response = f"data: {json_data}" + end_of_line - - return sse_response - -async def generate_no_stream_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): - random.seed(timestamp) - random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) - sample_data = { - "id": f"chatcmpl-{random_str}", - "object": "chat.completion", - "created": timestamp, - "model": model, - "choices": [ - { - "index": 0, - "message": { - "role": role, - "content": content, - "refusal": None - }, - "logprobs": None, - "finish_reason": "stop" - } - ], - "usage": None, - "system_fingerprint": "fp_a7d06e42a7" - } - if total_tokens: - total_tokens = prompt_tokens + completion_tokens - sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} - json_data = json.dumps(sample_data, ensure_ascii=False) - - return json_data +from utils import safe_get, generate_sse_response, generate_no_stream_response, end_of_line async def check_response(response, error_log): if response and not (200 <= response.status_code < 300): @@ -208,7 +134,12 @@ async def fetch_gpt_response_stream(client, url, headers, payload): return line = json.loads(result) line['id'] = f"chatcmpl-{random_str}" - yield "data: " + json.dumps(line).strip() + end_of_line + no_stream_content = safe_get(line, "choices", 0, "message", "content", default=None) + if no_stream_content: + sse_string = await generate_sse_response(safe_get(line, "created", default=None), safe_get(line, "model", default=None), content=no_stream_content) + yield sse_string + else: + yield "data: " + json.dumps(line).strip() + end_of_line async def fetch_cloudflare_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) diff --git a/utils.py b/utils.py index 3370fea8..85c01794 100644 --- a/utils.py +++ b/utils.py @@ -468,6 +468,8 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr # 如果不是错误，创建一个新的生成器，首先yield第一个项，然后yield剩余的项 async def new_generator(): + # print("type(first_item)", type(first_item)) + # print("first_item", ensure_string(first_item)) yield ensure_string(first_item) try: async for item in generator: @@ -649,3 +651,80 @@ def safe_get(data, *keys, default=None): except (KeyError, IndexError, AttributeError, TypeError): return default return data + + +# end_of_line = "\n\r\n" +# end_of_line = "\r\n" +# end_of_line = "\n\r" +end_of_line = "\n\n" +# end_of_line = "\r" +# end_of_line = "\n" + +import random +import string +async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) + sample_data = { + "id": f"chatcmpl-{random_str}", + "object": "chat.completion.chunk", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"content": content}, + "logprobs": None, + "finish_reason": None + } + ], + "usage": None, + "system_fingerprint": "fp_d576307f90", + } + if function_call_content: + sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]} + if tools_id and function_call_name: + sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"id": tools_id,"type":"function","function":{"name": function_call_name, "arguments":""}}]} + # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]} + if role: + sample_data["choices"][0]["delta"] = {"role": role, "content": ""} + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + sample_data["choices"] = [] + json_data = json.dumps(sample_data, ensure_ascii=False) + + # 构建SSE响应 + sse_response = f"data: {json_data}" + end_of_line + + return sse_response + +async def generate_no_stream_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) + sample_data = { + "id": f"chatcmpl-{random_str}", + "object": "chat.completion", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": role, + "content": content, + "refusal": None + }, + "logprobs": None, + "finish_reason": "stop" + } + ], + "usage": None, + "system_fingerprint": "fp_a7d06e42a7" + } + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + json_data = json.dumps(sample_data, ensure_ascii=False) + + return json_data From 8e8a326d3630fb7a8730689180e08632672b7cce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 1 Dec 2024 03:43:05 +0000 Subject: [PATCH 346/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.127?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f8ee4528..3ac9c0dc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.126 +0.0.127 From 905893b66be59232f79b6ab2ca951b37e87ab772 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 1 Dec 2024 12:28:50 +0800 Subject: [PATCH 347/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20OpenRouter=20from=20using=20images.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/main.py b/main.py index b582402c..6d02760f 100644 --- a/main.py +++ b/main.py @@ -821,8 +821,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "cloudflare" elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): engine = "claude" - elif parsed_url.netloc == 'openrouter.ai': - engine = "openrouter" elif parsed_url.netloc == 'api.cohere.com': engine = "cohere" request.stream = True From a75a84abcdaf75e95022391d5fcb0380b4618b93 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 1 Dec 2024 04:29:08 +0000 Subject: [PATCH 348/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.128?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3ac9c0dc..de20a4db 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.127 +0.0.128 From 541d965b0c33e6cb69b047963975a57c535df61e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 1 Dec 2024 17:16:21 +0800 Subject: [PATCH 349/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20no=20error=20is=20returned=20immediately=20when=20th?= =?UTF-8?q?e=20API=20key=20is=20invalid.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.py b/main.py index 6d02760f..e1b2faef 100644 --- a/main.py +++ b/main.py @@ -450,6 +450,11 @@ async def dispatch(self, request: Request, call_next): if api_index is not None: enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) + else: + return JSONResponse( + status_code=403, + content={"error": "Invalid or missing API Key"} + ) else: # 如果token为None，检查全局设置 enable_moderation = config.get('ENABLE_MODERATION', False) @@ -643,6 +648,7 @@ async def get_client(self, timeout_value, base_url, proxy=None): proxy = proxy.replace('socks5h://', 'socks5://') transport = AsyncProxyTransport.from_url(proxy) client_config["transport"] = transport + # print("proxy", proxy) except ImportError: logger.error("httpx-socks package is required for SOCKS proxy support") raise ImportError("Please install httpx-socks package for SOCKS proxy support: pip install httpx-socks") From 97bc62b7c90e6c042054fb2951472337461a6059 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 1 Dec 2024 09:16:50 +0000 Subject: [PATCH 350/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.129?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index de20a4db..5e7d8c61 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.128 +0.0.129 From 24a1d96cc11ed26577d0e4f5663c93a2cf839783 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 1 Dec 2024 20:43:46 +0800 Subject: [PATCH 351/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Add=20feature:=20?= =?UTF-8?q?Support=20chatgpt-4o-latest=20automatic=20deletion=20of=20tool?= =?UTF-8?q?=20request=20body.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index b3b6dc18..2ace333a 100644 --- a/request.py +++ b/request.py @@ -702,7 +702,7 @@ async def get_gpt_payload(request, engine, provider): else: payload[field] = value - if provider.get("tools") == False or "o1" in model: + if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model: payload.pop("tools", None) payload.pop("tool_choice", None) if "o1" in model and "models.inference.ai.azure.com" in url: From c06beece94c8fd32b957661730100ebfed6e1b81 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 1 Dec 2024 12:44:20 +0000 Subject: [PATCH 352/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.130?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5e7d8c61..ee358a33 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.129 +0.0.130 From dfd5704c4ddc9c7d7b3cf3e2a8706ebd25837ea4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 2 Dec 2024 15:58:05 +0800 Subject: [PATCH 353/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20update=20openai?= =?UTF-8?q?=20UA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index e1b2faef..8b6e8a83 100644 --- a/main.py +++ b/main.py @@ -713,7 +713,7 @@ async def ensure_config(request: Request, call_next): default_config = { "headers": { - "User-Agent": "curl/7.68.0", + "User-Agent": "OpenAI/Python 1.55.3", "Accept": "*/*", }, "http2": True, From c20bc2da2714d1054a9677bb2ef2d5f80281e578 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 2 Dec 2024 07:58:26 +0000 Subject: [PATCH 354/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.131?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee358a33..1ad6b812 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.130 +0.0.131 From ef9371824a5ffb3d300524cbd84de558337a67b6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 07:52:01 +0800 Subject: [PATCH 355/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20grok=20model=20does=20not=20automatically=20de?= =?UTF-8?q?lete=20the=20tool=20request=20body.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index 2ace333a..5528db83 100644 --- a/request.py +++ b/request.py @@ -702,7 +702,7 @@ async def get_gpt_payload(request, engine, provider): else: payload[field] = value - if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model: + if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model or "grok" in model: payload.pop("tools", None) payload.pop("tool_choice", None) if "o1" in model and "models.inference.ai.azure.com" in url: From db74559adeba96483ea059f30569dcaf5fc8b0c3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 2 Dec 2024 23:52:32 +0000 Subject: [PATCH 356/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.132?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1ad6b812..9ab8fdd1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.131 +0.0.132 From 4bc229014c5edeb26db5044e70f1156740b5397b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 08:27:53 +0800 Subject: [PATCH 357/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20error=20string=20was=20not=20recognized=20as?= =?UTF-8?q?=20a=20failed=20response.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/provider_test.py | 11 ++++++++++- utils.py | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/provider_test.py b/test/provider_test.py index b1014e72..557ac1b8 100644 --- a/test/provider_test.py +++ b/test/provider_test.py @@ -1,4 +1,5 @@ import os +import json import pytest from fastapi.testclient import TestClient import sys @@ -81,7 +82,15 @@ def test_request_model(test_client, api_key, get_model): response = test_client.post("/v1/chat/completions", json=request_data, headers=headers) for line in response.iter_lines(): - print(line.lstrip("data: ")) + line = line.lstrip("data: ") + if line == "[DONE]": + print("DONE") + break + try: + data = json.loads(line) + print(data) + except json.JSONDecodeError: + print(line) assert 200 <= response.status_code < 300 if __name__ == "__main__": diff --git a/utils.py b/utils.py index 85c01794..47ae183c 100644 --- a/utils.py +++ b/utils.py @@ -462,6 +462,9 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) if isinstance(first_item_str, dict) and engine not in ["tts", "embedding", "dalle", "moderation", "whisper"] and stream == False: + if any(x in str(first_item_str) for x in error_triggers): + logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) + raise StopAsyncIteration content = safe_get(first_item_str, "choices", 0, "message", "content", default=None) if content == "" or content is None: raise StopAsyncIteration From 2b72824695d331e9d8ac810ac003b0f975b3523a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 00:28:11 +0000 Subject: [PATCH 358/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.133?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9ab8fdd1..6067817e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.132 +0.0.133 From 1967fff59f2c92fb13cafbec90bd39fb2170f13e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 13:47:26 +0800 Subject: [PATCH 359/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20DALL-E=20model=20cannot=20customize=20the=20re?= =?UTF-8?q?sponse=5Fformat.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 1 + request.py | 1 + 2 files changed, 2 insertions(+) diff --git a/models.py b/models.py index f9fec42c..3e0e2a89 100644 --- a/models.py +++ b/models.py @@ -109,6 +109,7 @@ class ImageGenerationRequest(BaseRequest): prompt: str model: Optional[str] = "dall-e-3" n: Optional[int] = 1 + response_format: Optional[str] = "url" size: Optional[str] = "1024x1024" stream: bool = False diff --git a/request.py b/request.py index 5528db83..137d0dac 100644 --- a/request.py +++ b/request.py @@ -1071,6 +1071,7 @@ async def get_dalle_payload(request, engine, provider): "model": model, "prompt": request.prompt, "n": request.n, + "response_format": request.response_format, "size": request.size } From 626f713807bb2ea81dd13da55c28b8df5c37d2f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 05:47:49 +0000 Subject: [PATCH 360/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.134?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6067817e..297b2ecb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.133 +0.0.134 From a675cc95fd30001df1492a3968a8f53a7fe1ab2c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 14:13:32 +0800 Subject: [PATCH 361/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20Doubao=20model=20cannot=20use=20tools.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 8b6e8a83..ea0a3168 100644 --- a/main.py +++ b/main.py @@ -841,6 +841,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A and "o1" not in original_model \ and "gemini" not in original_model \ and "grok" not in original_model \ + and "doubao" not in original_model.lower() \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" From d2ce8e8e68ff2278756ed44e012fa4ecff33eaba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 06:13:51 +0000 Subject: [PATCH 362/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.135?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 297b2ecb..3954434a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.134 +0.0.135 From f5a26ecf1bb74044df82e5c914b6fbd4282b75a4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 17:46:14 +0800 Subject: [PATCH 363/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Delete=20the=20co?= =?UTF-8?q?de=20for=20the=20Doubao=20model=20to=20use=20the=20GPT=20channe?= =?UTF-8?q?l.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index ea0a3168..8b6e8a83 100644 --- a/main.py +++ b/main.py @@ -841,7 +841,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A and "o1" not in original_model \ and "gemini" not in original_model \ and "grok" not in original_model \ - and "doubao" not in original_model.lower() \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': engine = "openrouter" From b5304499479bc9480136e2b916356f2442462629 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 09:46:52 +0000 Subject: [PATCH 364/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.136?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3954434a..e4711634 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.135 +0.0.136 From ede1c039470c7e788276f766dd1a8022032ee4f7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 3 Dec 2024 19:55:21 +0800 Subject: [PATCH 365/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20error=20strings=20cannot=20be=20captured=20correctly?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 47ae183c..de7ad298 100644 --- a/utils.py +++ b/utils.py @@ -447,8 +447,13 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr if first_item_str.startswith("[DONE]"): logger.error(f"provider: {channel_id:<11} error_handling_wrapper [DONE]!") raise StopAsyncIteration - if any(x in first_item_str for x in error_triggers): - logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) + try: + encode_first_item_str = first_item_str.encode().decode('unicode-escape') + except UnicodeDecodeError: + encode_first_item_str = first_item_str + logger.error(f"provider: {channel_id:<11} error UnicodeDecodeError: %s", first_item_str) + if any(x in encode_first_item_str for x in error_triggers): + logger.error(f"provider: {channel_id:<11} error const string: %s", encode_first_item_str) raise StopAsyncIteration try: first_item_str = json.loads(first_item_str) From 89ab406206b196756269f69b9c1a6c3902b32541 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 11:55:46 +0000 Subject: [PATCH 366/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.137?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e4711634..ccefc022 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.136 +0.0.137 From c211d80a02398de009fdc2b87797a6e82c4cbe07 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 7 Dec 2024 12:03:23 +0800 Subject: [PATCH 367/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20image=20inp?= =?UTF-8?q?ut=20bug=20in=20o1-mini=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💻 Code: Change request header to curl/7.68.0 --- main.py | 2 +- request.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 8b6e8a83..e1b2faef 100644 --- a/main.py +++ b/main.py @@ -713,7 +713,7 @@ async def ensure_config(request: Request, call_next): default_config = { "headers": { - "User-Agent": "OpenAI/Python 1.55.3", + "User-Agent": "curl/7.68.0", "Accept": "*/*", }, "http2": True, diff --git a/request.py b/request.py index 137d0dac..e704a8a3 100644 --- a/request.py +++ b/request.py @@ -658,7 +658,7 @@ async def get_gpt_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url" and provider.get("image", True): + elif item.type == "image_url" and provider.get("image", True) and "o1-mini" not in model: image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: From 359accc5ece6f43b02834add6e01e1928e7df885 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 7 Dec 2024 04:03:49 +0000 Subject: [PATCH 368/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.138?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ccefc022..202169af 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.137 +0.0.138 From a67f66cb605c9554139af8d068d04248977377bf Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 7 Dec 2024 14:19:07 +0800 Subject: [PATCH 369/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20Jina=20embedding=20model=20cannot=20be=20used.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/request.py b/request.py index e704a8a3..b0d81e95 100644 --- a/request.py +++ b/request.py @@ -1138,7 +1138,10 @@ async def get_embedding_payload(request, engine, provider): } if request.encoding_format: - payload["encoding_format"] = request.encoding_format + if url.startswith("https://api.jina.ai"): + payload["embedding_type"] = request.encoding_format + else: + payload["encoding_format"] = request.encoding_format return url, headers, payload From a4f312669584ef6c233af69381600232b4aca469 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 7 Dec 2024 06:19:32 +0000 Subject: [PATCH 370/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.139?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 202169af..39056676 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.138 +0.0.139 From 970435967f677b83f0a8d441554bdd7cfca9aef9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 9 Dec 2024 12:58:10 +0800 Subject: [PATCH 371/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20makes=20the=20learnlm=20model=20unusable.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index e1b2faef..4c6c192d 100644 --- a/main.py +++ b/main.py @@ -840,6 +840,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A and "gpt" not in original_model \ and "o1" not in original_model \ and "gemini" not in original_model \ + and "learnlm" not in original_model \ and "grok" not in original_model \ and parsed_url.netloc != 'api.cloudflare.com' \ and parsed_url.netloc != 'api.cohere.com': @@ -878,8 +879,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine} role: {role}") url, headers, payload = await get_payload(request, engine, provider) - # print("url", url) if is_debug: + logger.info(url) logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) if payload.get("file"): pass From deea261b0b85de99cbab80b95818ef3fec9a1cda Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 9 Dec 2024 04:58:30 +0000 Subject: [PATCH 372/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.140?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 39056676..fd3c1fec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.139 +0.0.140 From 5f0c2f2b91c9e4b298b59f88063eec27a74285ef Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 11 Dec 2024 13:49:11 +0800 Subject: [PATCH 373/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20Gemini=20API=20does=20not=20support=20the=20to?= =?UTF-8?q?ol=20default=20field.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/request.py b/request.py index b0d81e95..7bafc958 100644 --- a/request.py +++ b/request.py @@ -261,9 +261,25 @@ async def get_gemini_payload(request, engine, provider): for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: if field == "tools": + # 处理每个工具的 function 定义 + processed_tools = [] + for tool in value: + function_def = tool["function"] + # 处理 parameters.properties 中的 default 字段 + if safe_get(function_def, "parameters", "properties", default=None): + for prop_value in function_def["parameters"]["properties"].values(): + if "default" in prop_value: + # 将 default 值添加到 description 中 + default_value = prop_value["default"] + description = prop_value.get("description", "") + prop_value["description"] = f"{description}\nDefault: {default_value}" + # 删除 default 字段 + del prop_value["default"] + processed_tools.append({"function": function_def}) + payload.update({ "tools": [{ - "function_declarations": [tool["function"] for tool in value] + "function_declarations": [tool["function"] for tool in processed_tools] }], "tool_config": { "function_calling_config": { From e96e3bfcd8a18552e0df5bd21cbf628ffb52849c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 11 Dec 2024 05:49:38 +0000 Subject: [PATCH 374/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.141?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd3c1fec..021629c4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.140 +0.0.141 From 6226845233c6e5f497aaa82530d561d1d2998006 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Dec 2024 12:23:27 +0800 Subject: [PATCH 375/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Add=20support=20for=20the=20built-in=20search=20tool=20of=20?= =?UTF-8?q?gemini-2.0-flash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/request.py b/request.py index 7bafc958..0d07a8ad 100644 --- a/request.py +++ b/request.py @@ -290,6 +290,16 @@ async def get_gemini_payload(request, engine, provider): else: payload[field] = value + if request.model.endswith("-search"): + if "tools" not in payload: + payload["tools"] = [{ + "googleSearch": {} + }] + else: + payload["tools"].append({ + "googleSearch": {} + }) + return url, headers, payload import time From 608830f222f86b3781ceb547e9eafa826d45553a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Dec 2024 04:23:53 +0000 Subject: [PATCH 376/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.142?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 021629c4..9f45cffb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.141 +0.0.142 From fbaa40f8626c31a4828eca9b475b2bfb47dea306 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 12 Dec 2024 22:58:58 +0800 Subject: [PATCH 377/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20Gemini=20API=20where=20an=20empty=20message=20was?= =?UTF-8?q?=20not=20automatically=20added=20when=20no=20role=20message=20w?= =?UTF-8?q?as=20passed=20in.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 ++ request.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 4c6c192d..96878852 100644 --- a/main.py +++ b/main.py @@ -490,6 +490,8 @@ async def dispatch(self, request: Request, call_next): if parsed_body: try: request_model = UnifiedRequest.model_validate(parsed_body).data + if is_debug: + logger.info("request_model: %s", json.dumps(request_model.model_dump(exclude_unset=True), indent=2, ensure_ascii=False)) model = request_model.model current_info["model"] = model diff --git a/request.py b/request.py index 0d07a8ad..beadb1c8 100644 --- a/request.py +++ b/request.py @@ -212,7 +212,7 @@ async def get_gemini_payload(request, engine, provider): payload = { - "contents": messages, + "contents": messages or [{"role": "user", "parts": [{"text": "No messages"}]}], "safetySettings": [ { "category": "HARM_CATEGORY_HARASSMENT", From 540f3c2ddcaf04a83902a99666fea07765713cad Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Dec 2024 14:59:20 +0000 Subject: [PATCH 378/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.143?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9f45cffb..bbb8b6ed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.142 +0.0.143 From c6061be491d04115767c6b80dd205d895e9be86f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 13 Dec 2024 22:13:21 +0800 Subject: [PATCH 379/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20Gemini=20channel=20cannot=20use=20third-party?= =?UTF-8?q?=20proxy=20URLs=20properly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- request.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 96878852..c8519f67 100644 --- a/main.py +++ b/main.py @@ -821,7 +821,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A parsed_url = urlparse(url) # print("parsed_url", parsed_url) engine = None - if parsed_url.path.startswith("/v1beta") or parsed_url.path.endswith("/v1"): + if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" diff --git a/request.py b/request.py index beadb1c8..5a6bde8a 100644 --- a/request.py +++ b/request.py @@ -145,12 +145,13 @@ async def get_gemini_payload(request, engine, provider): gemini_stream = "streamGenerateContent" url = provider['base_url'] parsed_url = urllib.parse.urlparse(url) - if parsed_url.path.startswith("/v1beta") or parsed_url.path.startswith("/v1"): + # print("parsed_url", parsed_url) + if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): api_version = parsed_url.path.split('/')[-1] # 获取 v1 或 v1beta else: api_version = "v1beta" # https://generativelanguage.googleapis.com/v1beta/models/ - url = f"{parsed_url.scheme}://{parsed_url.netloc}/{api_version}/models/{model}:{gemini_stream}?key={await provider_api_circular_list[provider['provider']].next(model)}" + url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}/models/{model}:{gemini_stream}?key={await provider_api_circular_list[provider['provider']].next(model)}" messages = [] systemInstruction = None From 910c44a69e566bbbdca04df161023d9a4040dc83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 13 Dec 2024 14:13:40 +0000 Subject: [PATCH 380/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.144?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bbb8b6ed..798a352e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.143 +0.0.144 From 56e2b93aa003ec933770944a1b436fa82026ebf8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 14 Dec 2024 16:58:04 +0800 Subject: [PATCH 381/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Try=20to=20fix=20?= =?UTF-8?q?vercel=20500=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index c8519f67..fdb86460 100644 --- a/main.py +++ b/main.py @@ -551,7 +551,7 @@ async def dispatch(self, request: Request, call_next): try: response = await call_next(request) - if request.url.path.startswith("/v1"): + if request.url.path.startswith("/v1") and not DISABLE_DATABASE: if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)) or type(response).__name__ == '_StreamingResponse': response = LoggingStreamingResponse( content=response.body_iterator, From ee65c13d0644997a9ddee6bea0c5ec4fa9e47218 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 14 Dec 2024 08:58:25 +0000 Subject: [PATCH 382/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.145?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 798a352e..b054b1a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.144 +0.0.145 From ace877f1bd6ae829ed28646f8dd98a71953e858f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 16 Dec 2024 11:49:02 +0800 Subject: [PATCH 383/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20incomplete=20response=5Fformat=20parameter=20passing.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/models.py b/models.py index 3e0e2a89..7b58c65c 100644 --- a/models.py +++ b/models.py @@ -61,16 +61,15 @@ class ToolChoice(BaseModel): class BaseRequest(BaseModel): request_type: Optional[Literal["chat", "image", "audio", "moderation"]] = Field(default=None, exclude=True) -def create_json_schema_class(): - class JsonSchema(BaseModel): - name: str +import warnings +warnings.filterwarnings("ignore", category=UserWarning, message=".*shadows an attribute.*") - model_config = ConfigDict(protected_namespaces=()) +class JsonSchema(BaseModel): + name: str + schema: Dict[str, Any] = Field(validation_alias='schema') - JsonSchema.__annotations__['schema'] = Dict[str, Any] - return JsonSchema + model_config = ConfigDict(protected_namespaces=()) -JsonSchema = create_json_schema_class() class ResponseFormat(BaseModel): type: Literal["text", "json_object", "json_schema"] json_schema: Optional[JsonSchema] = None From f8a81c0c630dc6ad1e5e6edb4eedd39544af4ba1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Dec 2024 03:49:27 +0000 Subject: [PATCH 384/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.146?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b054b1a5..c3fe93e2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.145 +0.0.146 From 38f6502d98e1ebb444b3bae00f79a0d84c6e75d7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 16 Dec 2024 18:14:16 +0800 Subject: [PATCH 385/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20streaming=20messages=20did=20not=20fully=20comply=20?= =?UTF-8?q?with=20the=20OpenAI=20format.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add support for Azure channel 📖 Docs: Update documentation --- README.md | 14 +++++--- README_CN.md | 14 +++++--- main.py | 2 ++ request.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++-- response.py | 49 ++++++++++++++++++++++++++- utils.py | 4 +-- 6 files changed, 164 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f5925f11..bae5a389 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,15 @@ ## Introduction -For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cohere, Groq, Cloudflare, OpenRouter, and more. +For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Azure, Cohere, Groq, Cloudflare, OpenRouter, and more. ## ✨ Features - No front-end, pure configuration file to configure API channels. You can run your own API station just by writing a file, and the documentation has a detailed configuration guide, beginner-friendly. - Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. -- Simultaneously supports Anthropic, Gemini, Vertex AI, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. -- Support OpenAI, Anthropic, Gemini, Vertex native tool use function calls. -- Support OpenAI, Anthropic, Gemini, Vertex native image recognition API. +- Simultaneously supports Anthropic, Gemini, Vertex AI, Azure, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure native tool use function calls. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure native image recognition API. - Support four types of load balancing. 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. @@ -125,6 +125,12 @@ providers: - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a simple name to replace the original complex name, optional - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes, otherwise yaml syntax error + - provider: azure + base_url: https://your-endpoint.openai.azure.com + api: your-api-key + model: + - gpt-4o + - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA diff --git a/README_CN.md b/README_CN.md index bc8e906a..605c0b8a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,15 +13,15 @@ ## 介绍 -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Cohere、Groq、Cloudflare、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Azure、Cohere、Groq、Cloudflare、OpenRouter 等。 ## ✨ 特性 - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 - 统一管理多个后端服务，支持 OpenAI、Deepseek、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex AI、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 -- 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 -- 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 +- 同时支持 Anthropic、Gemini、Vertex AI、Azure、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 +- 支持 OpenAI、 Anthropic、Gemini、Vertex、Azure 原生 tool use 函数调用。 +- 支持 OpenAI、Anthropic、Gemini、Vertex、Azure 原生识图 API。 - 支持四种负载均衡。 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 @@ -125,6 +125,12 @@ providers: - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # 重命名模型，@cf/meta/llama-3.1-8b-instruct 是服务商的原始的模型名称，必须使用引号包裹模型名，否则yaml语法错误，llama-3.1-8b 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 - '@cf/meta/llama-3.1-8b-instruct' # 必须使用引号包裹模型名，否则yaml语法错误 + - provider: azure + base_url: https://your-endpoint.openai.azure.com + api: your-api-key + model: + - gpt-4o + - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA diff --git a/main.py b/main.py index fdb86460..4e15c3c8 100644 --- a/main.py +++ b/main.py @@ -825,6 +825,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" + elif parsed_url.netloc.rstrip('/').endswith('openai.azure.com'): + engine = "azure" elif parsed_url.netloc == 'api.cloudflare.com': engine = "cloudflare" elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): diff --git a/request.py b/request.py index 5a6bde8a..e39b72ac 100644 --- a/request.py +++ b/request.py @@ -98,7 +98,7 @@ async def get_image_message(base64_image, engine = None): base64_image = f"data:image/png;base64,{png_base64}" image_type = "image/png" - if "gpt" == engine or "openrouter" == engine: + if "gpt" == engine or "openrouter" == engine or "azure" == engine: return { "type": "image_url", "image_url": { @@ -126,7 +126,7 @@ async def get_image_message(base64_image, engine = None): raise ValueError("Unknown engine") async def get_text_message(role, message, engine = None): - if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine or "o1" == engine: + if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine or "o1" == engine or "azure" == engine: return {"type": "text", "text": message} if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} @@ -739,6 +739,94 @@ async def get_gpt_payload(request, engine, provider): return url, headers, payload +def build_azure_endpoint(base_url, deployment_id, api_version="2024-10-21"): + # 移除base_url末尾的斜杠(如果有) + base_url = base_url.rstrip('/') + + # 构建路径 + path = f"/openai/deployments/{deployment_id}/chat/completions" + + # 使用urljoin拼接base_url和path + full_url = urllib.parse.urljoin(base_url, path) + + # 添加api-version查询参数 + final_url = f"{full_url}?api-version={api_version}" + + return final_url + +async def get_azure_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json', + } + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers['api-key'] = f"{provider['api']}" + + url = build_azure_endpoint( + base_url=provider['base_url'], + deployment_id=model, + ) + + messages = [] + for msg in request.messages: + tool_calls = None + tool_call_id = None + if isinstance(msg.content, list): + content = [] + for item in msg.content: + if item.type == "text": + text_message = await get_text_message(msg.role, item.text, engine) + content.append(text_message) + elif item.type == "image_url" and provider.get("image", True) and "o1-mini" not in model: + image_message = await get_image_message(item.image_url.url, engine) + content.append(image_message) + else: + content = msg.content + tool_calls = msg.tool_calls + tool_call_id = msg.tool_call_id + + if tool_calls: + tool_calls_list = [] + for tool_call in tool_calls: + tool_calls_list.append({ + "id": tool_call.id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments + } + }) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_calls": tool_calls_list}) + elif tool_call_id: + if provider.get("tools"): + messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) + else: + messages.append({"role": msg.role, "content": content}) + + payload = { + "model": model, + "messages": messages, + } + + miss_fields = [ + 'model', + 'messages', + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + if field == "max_tokens" and "o1" in model: + payload["max_completion_tokens"] = value + else: + payload[field] = value + + if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model or "grok" in model: + payload.pop("tools", None) + payload.pop("tool_choice", None) + + return url, headers, payload + async def get_openrouter_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' @@ -1206,6 +1294,8 @@ async def get_payload(request: RequestModel, engine, provider): return await get_vertex_gemini_payload(request, engine, provider) elif engine == "vertex-claude": return await get_vertex_claude_payload(request, engine, provider) + elif engine == "azure": + return await get_azure_payload(request, engine, provider) elif engine == "claude": return await get_claude_payload(request, engine, provider) elif engine == "gpt": diff --git a/response.py b/response.py index 2f735f37..fc2fe17b 100644 --- a/response.py +++ b/response.py @@ -141,10 +141,40 @@ async def fetch_gpt_response_stream(client, url, headers, payload): else: yield "data: " + json.dumps(line).strip() + end_of_line +async def fetch_azure_response_stream(client, url, headers, payload): + timestamp = int(datetime.timestamp(datetime.now())) + async with client.stream('POST', url, headers=headers, json=payload) as response: + error_message = await check_response(response, "fetch_azure_response_stream") + if error_message: + yield error_message + return + + buffer = "" + sse_string = "" + async for chunk in response.aiter_text(): + buffer += chunk + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + # logger.info("line: %s", repr(line)) + if line and line != "data: " and line != "data:" and not line.startswith(": "): + result = line.lstrip("data: ") + if result.strip() == "[DONE]": + yield "data: [DONE]" + end_of_line + return + line = json.loads(result) + no_stream_content = safe_get(line, "choices", 0, "message", "content", default=None) + stream_content = safe_get(line, "choices", 0, "delta", "content", default=None) + if no_stream_content or stream_content or sse_string: + sse_string = await generate_sse_response(timestamp, safe_get(line, "model", default=None), content=no_stream_content or stream_content) + yield sse_string + if no_stream_content: + yield "data: [DONE]" + end_of_line + return + async def fetch_cloudflare_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: - error_message = await check_response(response, "fetch_gpt_response_stream") + error_message = await check_response(response, "fetch_cloudflare_response_stream") if error_message: yield error_message return @@ -299,6 +329,20 @@ async def fetch_response(client, url, headers, payload, engine, model): timestamp = int(datetime.timestamp(datetime.now())) yield await generate_no_stream_response(timestamp, model, content=content, tools_id=None, function_call_name=None, function_call_content=None, role=role, total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=candidates_tokens) + elif engine == "azure": + response_json = response.json() + # 删除 content_filter_results + if "choices" in response_json: + for choice in response_json["choices"]: + if "content_filter_results" in choice: + del choice["content_filter_results"] + + # 删除 prompt_filter_results + if "prompt_filter_results" in response_json: + del response_json["prompt_filter_results"] + + yield response_json + else: response_json = response.json() yield response_json @@ -314,6 +358,9 @@ async def fetch_response_stream(client, url, headers, payload, engine, model): elif engine == "gpt": async for chunk in fetch_gpt_response_stream(client, url, headers, payload): yield chunk + elif engine == "azure": + async for chunk in fetch_azure_response_stream(client, url, headers, payload): + yield chunk elif engine == "openrouter": async for chunk in fetch_gpt_response_stream(client, url, headers, payload): yield chunk diff --git a/utils.py b/utils.py index de7ad298..db194422 100644 --- a/utils.py +++ b/utils.py @@ -681,9 +681,9 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f "choices": [ { "index": 0, - "delta": {"content": content}, + "delta": {"content": content} if content else {}, "logprobs": None, - "finish_reason": None + "finish_reason": None if content else "stop" } ], "usage": None, From d6eb53be395bef4919d89e242ac5a3fb0a1ff71e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Dec 2024 10:14:41 +0000 Subject: [PATCH 386/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.147?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c3fe93e2..cc5ed63e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.146 +0.0.147 From 22a17e1fe1e39fa076aedd46f2866d3660b2843c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 18 Dec 2024 17:09:03 +0800 Subject: [PATCH 387/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20OpenRouter=20channel=20cannot=20use=20images.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index e39b72ac..efb4bc4e 100644 --- a/request.py +++ b/request.py @@ -862,7 +862,7 @@ async def get_openrouter_payload(request, engine, provider): if item["type"] == "text": messages.append({"role": msg.role, "content": item["text"]}) elif item["type"] == "image_url": - messages.append({"role": msg.role, "content": item["url"]}) + messages.append({"role": msg.role, "content": [await get_image_message(item["image_url"]["url"], engine)]}) else: messages.append({"role": msg.role, "content": content}) From 375348814f21bec9bed81e5e46067af336f2b738 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Dec 2024 09:09:36 +0000 Subject: [PATCH 388/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.148?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cc5ed63e..7547e448 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.147 +0.0.148 From a637729cac2df99a0c9b0c9d6fd7af70007bd032 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Dec 2024 13:04:42 +0800 Subject: [PATCH 389/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20new-api=20is=20mistakenly=20identified=20as=20an=20e?= =?UTF-8?q?rror=20message=20when=20it=20is=20not=20streaming=20output.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add support for new-api Gemini search. 📖 Docs: Update documentation --- README.md | 8 ++++---- README_CN.md | 8 ++++---- request.py | 19 +++++++++++++++++++ utils.py | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bae5a389..b9c9048b 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,15 @@ ## Introduction -For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Azure, Cohere, Groq, Cloudflare, OpenRouter, and more. +For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Azure, xai, Cohere, Groq, Cloudflare, OpenRouter, and more. ## ✨ Features - No front-end, pure configuration file to configure API channels. You can run your own API station just by writing a file, and the documentation has a detailed configuration guide, beginner-friendly. - Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. -- Simultaneously supports Anthropic, Gemini, Vertex AI, Azure, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. -- Support OpenAI, Anthropic, Gemini, Vertex, Azure native tool use function calls. -- Support OpenAI, Anthropic, Gemini, Vertex, Azure native image recognition API. +- Simultaneously supports Anthropic, Gemini, Vertex AI, Azure, xai, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure, xai native tool use function calls. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure, xai native image recognition API. - Support four types of load balancing. 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. diff --git a/README_CN.md b/README_CN.md index 605c0b8a..37a4612e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,15 +13,15 @@ ## 介绍 -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Azure、Cohere、Groq、Cloudflare、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Azure、xai、Cohere、Groq、Cloudflare、OpenRouter 等。 ## ✨ 特性 - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 - 统一管理多个后端服务，支持 OpenAI、Deepseek、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex AI、Azure、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 -- 支持 OpenAI、 Anthropic、Gemini、Vertex、Azure 原生 tool use 函数调用。 -- 支持 OpenAI、Anthropic、Gemini、Vertex、Azure 原生识图 API。 +- 同时支持 Anthropic、Gemini、Vertex AI、Azure、xai、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 +- 支持 OpenAI、 Anthropic、Gemini、Vertex、Azure、xai 原生 tool use 函数调用。 +- 支持 OpenAI、Anthropic、Gemini、Vertex、Azure、xai 原生识图 API。 - 支持四种负载均衡。 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 diff --git a/request.py b/request.py index efb4bc4e..aefbef91 100644 --- a/request.py +++ b/request.py @@ -737,6 +737,25 @@ async def get_gpt_payload(request, engine, provider): # request.stream = False payload.pop("stream_options", None) + if request.model.endswith("-search") and "gemini" in request.model: + if "tools" not in payload: + payload["tools"] = [{ + "type": "function", + "function": { + "name": "googleSearch", + "description": "googleSearch" + } + }] + else: + if not any(tool["function"]["name"] == "googleSearch" for tool in payload["tools"]): + payload["tools"].append({ + "type": "function", + "function": { + "name": "googleSearch", + "description": "googleSearch" + } + }) + return url, headers, payload def build_azure_endpoint(base_url, deployment_id, api_version="2024-10-21"): diff --git a/utils.py b/utils.py index db194422..baca0a36 100644 --- a/utils.py +++ b/utils.py @@ -460,7 +460,7 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr except json.JSONDecodeError: logger.error(f"provider: {channel_id:<11} error_handling_wrapper JSONDecodeError! {repr(first_item_str)}") raise StopAsyncIteration - if isinstance(first_item_str, dict) and 'error' in first_item_str: + if isinstance(first_item_str, dict) and 'error' in first_item_str and first_item_str.get('error') != {"message": "","type": "","param": "","code": None}: # 如果第一个 yield 的项是错误信息，抛出 HTTPException status_code = first_item_str.get('status_code', 500) detail = first_item_str.get('details', f"{first_item_str}") From 0c42621160971dc27444bd07352effb06d15fe48 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 20 Dec 2024 05:05:02 +0000 Subject: [PATCH 390/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.149?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7547e448..8e82720b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.148 +0.0.149 From 387d954493401b31e47a867b52c87c72cd1bb464 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 20 Dec 2024 19:55:26 +0800 Subject: [PATCH 391/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20there=20is=20no=20spacing=20between=20the=20thinking?= =?UTF-8?q?=20text=20and=20the=20actual=20answer=20in=20gemini-2.0-flash-t?= =?UTF-8?q?hinking-exp-1219.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/response.py b/response.py index fc2fe17b..f3984c9b 100644 --- a/response.py +++ b/response.py @@ -30,20 +30,26 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): revicing_function_call = False function_full_response = "{" need_function_call = False + line_index = 0 + last_text_line = 0 async for chunk in response.aiter_text(): buffer += chunk while "\n" in buffer: line, buffer = buffer.split("\n", 1) + line_index += 1 # print(line) if line and '\"text\": \"' in line: try: json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) + if last_text_line == line_index - 3: + content = "\n\n" + content.lstrip() sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") + last_text_line = line_index if line and ('\"functionCall\": {' in line or revicing_function_call): revicing_function_call = True From a7549cf53d2d638f751f6f4fd8e837252cfda2e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 20 Dec 2024 11:55:49 +0000 Subject: [PATCH 392/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.150?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8e82720b..85a5e001 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.149 +0.0.150 From b0d3e77b36a8cf2bc53d5fea204dfcef701a94ef Mon Sep 17 00:00:00 2001 From: yym68686 Date: Mon, 23 Dec 2024 23:03:41 +0800 Subject: [PATCH 393/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20model=5Ftimeout=20value=20causes=20an=20error?= =?UTF-8?q?=20when=20it=20is=20of=20int=20type.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 4e15c3c8..f7f01867 100644 --- a/main.py +++ b/main.py @@ -730,10 +730,13 @@ async def ensure_config(request: Request, call_next): # 存储超时配置 app.state.timeouts = {} if app.state.config and 'preferences' in app.state.config: - for model_name, timeout_value in app.state.config['preferences'].get('model_timeout', {}).items(): - app.state.timeouts[model_name] = timeout_value - if "default" not in app.state.config['preferences'].get('model_timeout', {}): - app.state.timeouts["default"] = DEFAULT_TIMEOUT + if isinstance(app.state.config['preferences'].get('model_timeout'), int): + app.state.timeouts["default"] = app.state.config['preferences'].get('model_timeout') + else: + for model_name, timeout_value in app.state.config['preferences'].get('model_timeout', {"default": DEFAULT_TIMEOUT}).items(): + app.state.timeouts[model_name] = timeout_value + if "default" not in app.state.config['preferences'].get('model_timeout', {}): + app.state.timeouts["default"] = DEFAULT_TIMEOUT app.state.provider_timeouts = defaultdict(lambda: defaultdict(lambda: DEFAULT_TIMEOUT)) for provider in app.state.config["providers"]: @@ -1358,7 +1361,7 @@ def generate_api_key(): # Define the character set (only alphanumeric) chars = string.ascii_letters + string.digits # Generate a random string of 36 characters - random_string = ''.join(secrets.choice(chars) for _ in range(36)) + random_string = ''.join(secrets.choice(chars) for _ in range(48)) api_key = "sk-" + random_string return JSONResponse(content={"api_key": api_key}) From 0d743fb477f8e3e2ff97aa5ac1ae293d704ebade Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 23 Dec 2024 15:04:08 +0000 Subject: [PATCH 394/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.151?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 85a5e001..be8c9e18 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.150 +0.0.151 From 82580dfdf3894e5de8b7a3d6d9c3386f24c12f8a Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 25 Dec 2024 23:35:29 +0800 Subject: [PATCH 395/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Add=20feature:?= =?UTF-8?q?=20Support=20gemini-2.0-flash-thinking=20model=20to=20add=20ref?= =?UTF-8?q?erences=20in=20the=20thinking=20part.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/response.py b/response.py index f3984c9b..0fc4ee04 100644 --- a/response.py +++ b/response.py @@ -32,6 +32,10 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): need_function_call = False line_index = 0 last_text_line = 0 + if "thinking" in model: + is_thinking = True + else: + is_thinking = False async for chunk in response.aiter_text(): buffer += chunk while "\n" in buffer: @@ -43,8 +47,13 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) + if last_text_line == 0 and is_thinking: + content = "> " + content.lstrip() + if is_thinking: + content = content.replace("\n", "\n> ") if last_text_line == line_index - 3: - content = "\n\n" + content.lstrip() + is_thinking = False + content = "\n\n\n" + content.lstrip() sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: From a0e48a2a101617e4ef388e8cc38aca415e57cd6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 25 Dec 2024 15:35:52 +0000 Subject: [PATCH 396/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.152?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index be8c9e18..7117656a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.151 +0.0.152 From d39a1fdb880dde5531e50cd4c3fc904886c0645f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 5 Jan 2025 13:31:52 +0800 Subject: [PATCH 397/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20Gemini=20API=20where=20the=20response=5Fformat=20?= =?UTF-8?q?field=20is=20placed=20in=20the=20request=20body.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✨ Feature: Add feature: support displaying timeout duration in logs --- main.py | 3 ++- request.py | 3 ++- response.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index f7f01867..95d47020 100644 --- a/main.py +++ b/main.py @@ -1192,7 +1192,8 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques # 根据异常类型设置状态码和错误消息 if isinstance(e, httpx.ReadTimeout): status_code = 504 # Gateway Timeout - error_message = "Request timed out" + timeout_value = e.request.extensions.get('timeout', {}).get('read', -1) + error_message = f"Request timed out after {timeout_value} seconds" elif isinstance(e, httpx.ConnectError): status_code = 503 # Service Unavailable error_message = "Unable to connect to service" diff --git a/request.py b/request.py index aefbef91..bdea4757 100644 --- a/request.py +++ b/request.py @@ -256,7 +256,8 @@ async def get_gemini_payload(request, engine, provider): 'user', 'include_usage', 'logprobs', - 'top_logprobs' + 'top_logprobs', + 'response_format' ] for field, value in request.model_dump(exclude_unset=True).items(): diff --git a/response.py b/response.py index 0fc4ee04..f8b0281d 100644 --- a/response.py +++ b/response.py @@ -47,6 +47,7 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) + # content = content.replace("\n", "\n\n") if last_text_line == 0 and is_thinking: content = "> " + content.lstrip() if is_thinking: From aca4e1781f7d0fa57b49a6544ad89baf1cb121f0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 5 Jan 2025 05:32:12 +0000 Subject: [PATCH 398/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=200.?= =?UTF-8?q?0.153?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7117656a..ea348486 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.152 +0.0.153 From 9dc798a5dd79ab2869eebdbfa60ce267a797ac95 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 5 Jan 2025 16:05:43 +0800 Subject: [PATCH 399/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Remove=20invalid?= =?UTF-8?q?=20base=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- request.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index ea348486..3eefcb9d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.153 +1.0.0 diff --git a/request.py b/request.py index bdea4757..b7e515f4 100644 --- a/request.py +++ b/request.py @@ -666,10 +666,7 @@ async def get_gpt_payload(request, engine, provider): model_dict = get_model_dict(provider) model = model_dict[request.model] if provider.get("api"): - if provider['base_url'] == "https://api-ext.felo.ai/one-ai/completions" or provider['base_url'] == "https://api-ext.felo.ai/trail/v1/chat/completions": - headers['Authorization'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" - else: - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" elif provider['provider'].startswith("sk-"): headers['Authorization'] = f"Bearer {provider['provider']}" From db864077b539da03bf236133a28683afe7c49dd0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 5 Jan 2025 08:06:13 +0000 Subject: [PATCH 400/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3eefcb9d..7dea76ed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.0.1 From 62c2ecd52980f1a169528b0c8f48f97bac217f66 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 9 Jan 2025 01:46:22 +0800 Subject: [PATCH 401/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20of=20duplicate=20answers=20in=20Gemini=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/response.py b/response.py index f8b0281d..f4ee5d8f 100644 --- a/response.py +++ b/response.py @@ -32,15 +32,20 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): need_function_call = False line_index = 0 last_text_line = 0 + is_finish = False if "thinking" in model: is_thinking = True else: is_thinking = False async for chunk in response.aiter_text(): buffer += chunk + while "\n" in buffer: line, buffer = buffer.split("\n", 1) line_index += 1 + if line and '\"finishReason\": \"' in line: + is_finish = True + break # print(line) if line and '\"text\": \"' in line: try: @@ -70,6 +75,9 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response += line + if is_finish: + break + if need_function_call: function_call = json.loads(function_full_response) function_call_name = function_call["functionCall"]["name"] From 4505decd3835dd4793a741067554b06322b8fd9f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Jan 2025 17:46:44 +0000 Subject: [PATCH 402/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7dea76ed..6d7de6e6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.0.2 From de53499e4a68f58c2f9f842fe0758685f5287357 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 9 Jan 2025 01:57:27 +0800 Subject: [PATCH 403/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20o1-mini=20and=20o1-preview=20cannot=20use=20system?= =?UTF-8?q?=20messages.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/request.py b/request.py index b7e515f4..5fbd7c69 100644 --- a/request.py +++ b/request.py @@ -710,6 +710,10 @@ async def get_gpt_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) + if "o1-mini" in model or "o1-preview" in model and len(messages) > 1 and messages[0]["role"] == "system": + system_msg = messages.pop(0) + messages[0]["content"] = system_msg["content"] + messages[0]["content"] + payload = { "model": model, "messages": messages, From de6b5402054be06ca81a8187b0205a9d6334699b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Jan 2025 17:57:47 +0000 Subject: [PATCH 404/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6d7de6e6..21e8796a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.2 +1.0.3 From f39ed57ced77b575a65a211f85d13151e29fdcb9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 9 Jan 2025 16:55:56 +0800 Subject: [PATCH 405/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20that=20prevents=20the=20vertex=20API=20gemini=20experimental?= =?UTF-8?q?=20model=20from=20being=20used=20in=20multiple=20regions.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 9 +++++++-- utils.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/request.py b/request.py index 5fbd7c69..aa806a16 100644 --- a/request.py +++ b/request.py @@ -8,7 +8,7 @@ import io from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, BaseAPI, get_model_dict, provider_api_circular_list, safe_get +from utils import c35s, c3s, c3o, c3h, gemini1, gemini2, BaseAPI, get_model_dict, provider_api_circular_list, safe_get import imghdr @@ -372,7 +372,12 @@ async def get_vertex_gemini_payload(request, engine, provider): gemini_stream = "streamGenerateContent" model_dict = get_model_dict(provider) model = model_dict[request.model] - location = gem + + if "gemini-2.0" in model or "gemini-exp" in model: + location = gemini2 + else: + location = gemini1 + url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) messages = [] diff --git a/utils.py b/utils.py index baca0a36..2b4af3df 100644 --- a/utils.py +++ b/utils.py @@ -600,6 +600,8 @@ def get_all_models(config): return all_models # 【GCP-Vertex AI 目前有這些區域可用】 https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude?hl=zh_cn +# https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations?hl=zh-cn#available-regions + # c3.5s # us-east5 # europe-west1 @@ -623,7 +625,8 @@ def get_all_models(config): c3s = ThreadSafeCircularList(["us-east5", "us-central1", "asia-southeast1"]) c3o = ThreadSafeCircularList(["us-east5"]) c3h = ThreadSafeCircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) -gem = ThreadSafeCircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) +gemini1 = ThreadSafeCircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) +gemini2 = ThreadSafeCircularList(["us-central1"]) class BaseAPI: def __init__( From 964eda0b7da089c640416919644f05877af09585 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jan 2025 08:56:27 +0000 Subject: [PATCH 406/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 21e8796a..ee90284c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.3 +1.0.4 From 7b5bbbe5629124fbe73f33ab586cceac781c63f9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 9 Jan 2025 23:56:36 +0800 Subject: [PATCH 407/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20o1=20model=20message=20length=20calculation.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index aa806a16..8952e61e 100644 --- a/request.py +++ b/request.py @@ -715,7 +715,7 @@ async def get_gpt_payload(request, engine, provider): else: messages.append({"role": msg.role, "content": content}) - if "o1-mini" in model or "o1-preview" in model and len(messages) > 1 and messages[0]["role"] == "system": + if ("o1-mini" in model or "o1-preview" in model) and len(messages) > 1 and messages[0]["role"] == "system": system_msg = messages.pop(0) messages[0]["content"] = system_msg["content"] + messages[0]["content"] From 6ccab9a08d7afca6b5ddd5937af992589081fe9f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jan 2025 15:57:01 +0000 Subject: [PATCH 408/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee90284c..90a27f9c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.5 From 04ac8887d1ca7bb82d46009b95b4d606b3918d8c Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 10 Jan 2025 16:04:06 +0800 Subject: [PATCH 409/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Add=20custom=20p?= =?UTF-8?q?ort=20environment=20variable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 95d47020..f2471d21 100644 --- a/main.py +++ b/main.py @@ -2152,10 +2152,12 @@ async def delete_row(row_id: str): if __name__ == '__main__': import uvicorn + import os + PORT = os.getenv("PORT", 8000) uvicorn.run( "__main__:app", host="0.0.0.0", - port=8000, + port=PORT, reload=True, reload_dirs=["./"], reload_includes=["*.py", "api.yaml"], From 432b1098a8dad1a9c67f97e15b11ad004a95b2e4 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 10 Jan 2025 16:05:49 +0800 Subject: [PATCH 410/476] =?UTF-8?q?=F0=9F=92=BB=20Code:=20Add=20custom=20p?= =?UTF-8?q?ort=20environment=20variable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index f2471d21..66ae25f6 100644 --- a/main.py +++ b/main.py @@ -2153,7 +2153,7 @@ async def delete_row(row_id: str): if __name__ == '__main__': import uvicorn import os - PORT = os.getenv("PORT", 8000) + PORT = int(os.getenv("PORT", "8000")) uvicorn.run( "__main__:app", host="0.0.0.0", From a3fc41eed2a125eb0521f18e003c6204f90fbcd9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 10 Jan 2025 08:06:38 +0000 Subject: [PATCH 411/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 90a27f9c..af0b7ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.5 +1.0.6 From f216a0c96ef12dd91b354bcc717e53794689f251 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 10 Jan 2025 16:09:37 +0800 Subject: [PATCH 412/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9c9048b..6c48b272 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ providers: - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, can be any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required, automatically uses base_url and api to get all available models through the /v1/models endpoint. - # Multiple providers can be configured here, each provider can configure multiple API Keys, and each API Key can configure multiple models. + # Multiple providers can be configured here, each provider can configure multiple API Keys, and each provider can configure multiple models. api_keys: - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key, user request uni-api requires API key, required # This API Key can use all models, that is, it can use all models in all channels set under providers, without needing to add available channels one by one. diff --git a/README_CN.md b/README_CN.md index 37a4612e..e380abaf 100644 --- a/README_CN.md +++ b/README_CN.md @@ -51,7 +51,7 @@ providers: - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter，随便取名字，必填 base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填，自动使用 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 - # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个 API Key 可以配置多个模型。 + # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个提供商可以配置多个模型。 api_keys: - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key，用户请求 uni-api 需要 API key，必填 # 该 API Key 可以使用所有模型，即可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 From f451cff93af64bc6b1bda4550da91922a5a88477 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 12 Jan 2025 01:47:55 +0800 Subject: [PATCH 413/476] =?UTF-8?q?=E2=9C=A8=20Feature:=20Completely=20shu?= =?UTF-8?q?t=20down=20Gemini=20content=20review.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/request.py b/request.py index 8952e61e..bb9a97a5 100644 --- a/request.py +++ b/request.py @@ -217,19 +217,19 @@ async def get_gemini_payload(request, engine, provider): "safetySettings": [ { "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE" + "threshold": "OFF" }, { "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE" + "threshold": "OFF" }, { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE" + "threshold": "OFF" }, { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE" + "threshold": "OFF" } ] } From d8ceff2889c6df4971ff06ed74c13195e9ea6192 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 11 Jan 2025 17:48:19 +0000 Subject: [PATCH 414/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index af0b7ddb..238d6e88 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.6 +1.0.7 From de66b721f057d21b8f256970d9adbd356b9d13e6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 12 Jan 2025 04:38:12 +0800 Subject: [PATCH 415/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20the=20security=20settings=20cannot=20be=20closed=20i?= =?UTF-8?q?n=20gemini-2.0-flash-thinking-exp-1219.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/request.py b/request.py index bb9a97a5..26e13750 100644 --- a/request.py +++ b/request.py @@ -211,25 +211,29 @@ async def get_gemini_payload(request, engine, provider): content[0]["text"] = re.sub(r"_+", "_", content[0]["text"]) systemInstruction = {"parts": content} + if "gemini-2.0-flash-exp" in model or "gemini-1.5" in model: + safety_settings = "OFF" + else: + safety_settings = "BLOCK_NONE" payload = { "contents": messages or [{"role": "user", "parts": [{"text": "No messages"}]}], "safetySettings": [ { "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "OFF" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "OFF" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "OFF" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "OFF" + "threshold": safety_settings } ] } From acf1a002a48055f198ec80d6ef5367c334459bd7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 11 Jan 2025 20:38:32 +0000 Subject: [PATCH 416/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 238d6e88..b0f3d96f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.7 +1.0.8 From cedf2811516e04c260c9ecc9e3aee2d73076b2fc Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 12 Jan 2025 16:11:18 +0800 Subject: [PATCH 417/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- README_CN.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6c48b272..7d8e5abc 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ ssh login to the serv00 server, execute the following command: git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git cd uni-api python -m venv uni-api -tmux new -s uni-api +tmux new -A -s uni-api source uni-api/bin/activate export CFLAGS="-I/usr/local/include" export CXXFLAGS="-I/usr/local/include" @@ -258,7 +258,7 @@ cpuset -l 0 pip install -r -vv requirements.txt ctrl+b d to exit tmux, wait a few hours for the installation to complete, and after the installation is complete, execute the following command: ```bash -tmux attach -t uni-api +tmux new -A -s uni-api source uni-api/bin/activate export CONFIG_URL=http://file_url/api.yaml export DISABLE_DATABASE=true diff --git a/README_CN.md b/README_CN.md index e380abaf..fb50f554 100644 --- a/README_CN.md +++ b/README_CN.md @@ -242,7 +242,7 @@ ssh 登陆到 serv00 服务器，执行下面的命令： git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git cd uni-api python -m venv uni-api -tmux new -s uni-api +tmux new -A -s uni-api source uni-api/bin/activate export CFLAGS="-I/usr/local/include" export CXXFLAGS="-I/usr/local/include" @@ -258,7 +258,7 @@ cpuset -l 0 pip install -r -vv requirements.txt ctrl+b d 退出 tmux 等待几个小时安装完成，安装完成后执行下面的命令： ```bash -tmux attach -t uni-api +tmux new -A -s uni-api source uni-api/bin/activate export CONFIG_URL=http://file_url/api.yaml export DISABLE_DATABASE=true From 2332b596a24925c4c420126561560b96e71e1362 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 12 Jan 2025 20:09:05 +0800 Subject: [PATCH 418/476] =?UTF-8?q?=F0=9F=93=96=20Docs:=20Update=20documen?= =?UTF-8?q?tation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7d8e5abc..af94e69c 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ api_keys: - gpt-4o # Usable model name, can use all gpt-4o models provided by providers - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers - gemini/* # Usable model name, can only use all models provided by providers named gemini, where gemini is the provider name, * represents all models - role: admin + role: admin # Set the alias of the API key, optional. The request log will display the alias of the API key. If role is admin, only this API key can request the v1/stats,/v1/generate-api-key endpoints. If all API keys do not have role set to admin, the first API key is set as admin and has permission to request the v1/stats,/v1/generate-api-key endpoints. - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: diff --git a/README_CN.md b/README_CN.md index fb50f554..297575b2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -146,7 +146,7 @@ api_keys: - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 - role: admin + role: admin # 设置 API key 的别名，选填。请求日志会显示该 API key 的别名。如果 role 为 admin，则仅有此 API key 可以请求 v1/stats,/v1/generate-api-key 端点。如果所有 API key 都没有设置 role 为 admin，则默认第一个 API key 为 admin 拥有请求 v1/stats,/v1/generate-api-key 端点的权限。 - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: From 994a6e2b39450e621333bd46d2c34fa1f3a95919 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 14 Jan 2025 18:28:42 +0800 Subject: [PATCH 419/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20where=20Vertex=20Gemini=20cannot=20use=20search.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request.py b/request.py index 26e13750..f7b7a80a 100644 --- a/request.py +++ b/request.py @@ -508,11 +508,11 @@ async def get_vertex_gemini_payload(request, engine, provider): if request.model.endswith("-search"): if "tools" not in payload: payload["tools"] = [{ - "googleSearchRetrieval": {} + "googleSearch": {} }] else: payload["tools"].append({ - "googleSearchRetrieval": {} + "googleSearch": {} }) return url, headers, payload From 50b91639f56d6d22e6cafcc00bdcf6587ce490f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 14 Jan 2025 10:29:09 +0000 Subject: [PATCH 420/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b0f3d96f..66c4c226 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.8 +1.0.9 From ae5eefec27dd12931b9312465f48cc4fd0282c8e Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 15 Jan 2025 15:25:36 +0800 Subject: [PATCH 421/476] =?UTF-8?q?=E2=9C=A8=20Update=20Function=20model:?= =?UTF-8?q?=20Set=20default=20value=20for=20description=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index 7b58c65c..a5edfdd9 100644 --- a/models.py +++ b/models.py @@ -10,7 +10,7 @@ class FunctionParameter(BaseModel): class Function(BaseModel): name: str - description: str + description: str = Field(default=None) parameters: Optional[FunctionParameter] = Field(default=None, exclude=None) class Tool(BaseModel): From 104191e9653a3b83622d321d3687519857c92177 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jan 2025 07:25:55 +0000 Subject: [PATCH 422/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?0.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 66c4c226..7ee7020b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.9 +1.0.10 From 91e63defafbe3860ee6221bb00d8ebcca8cc0718 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 16:03:31 +0800 Subject: [PATCH 423/476] =?UTF-8?q?=F0=9F=93=A6=20Bump=20version=20to=201.?= =?UTF-8?q?1.0=20and=20set=20default=20value=20for=20'required'=20field=20?= =?UTF-8?q?in=20FunctionParameter=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7ee7020b..9084fa2f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.10 +1.1.0 diff --git a/models.py b/models.py index a5edfdd9..f274b02f 100644 --- a/models.py +++ b/models.py @@ -6,7 +6,7 @@ class FunctionParameter(BaseModel): type: str properties: Dict[str, Dict[str, Any]] - required: List[str] + required: List[str] = None class Function(BaseModel): name: str From 51466aba33350a2ad1de6d79a5ee1997836495c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 08:03:59 +0000 Subject: [PATCH 424/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9084fa2f..524cb552 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.1.1 From 70ed3a40715d1921c051a622d7a7be3586c9b3a8 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 17:05:05 +0800 Subject: [PATCH 425/476] =?UTF-8?q?=E2=9C=A8=20Enhance=20Tool=20model:=20A?= =?UTF-8?q?dd=20model=5Fdump=20method=20to=20conditionally=20exclude=20'pa?= =?UTF-8?q?rameters'=20field=20and=20include=20example=20usage=20in=20main?= =?UTF-8?q?=20block?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/models.py b/models.py index f274b02f..ff0bb603 100644 --- a/models.py +++ b/models.py @@ -17,6 +17,16 @@ class Tool(BaseModel): type: str function: Function + def model_dump(self, **kwargs): + data = super().model_dump(**kwargs) + function_data = data['function'] + if 'parameters' in function_data and ( + function_data['parameters'] is None or + not function_data['parameters'].get('properties') + ): + function_data.pop('parameters', None) + return data + class FunctionCall(BaseModel): name: str arguments: str @@ -174,3 +184,19 @@ def set_request_type(cls, values): else: raise ValueError("无法确定请求类型") return values + +if __name__ == "__main__": + tool = Tool( + type="function", + function=Function( + name="clock-time____getCurrentTime____standalone", + description="获取当前时间", + parameters=FunctionParameter( + type="object", + properties={} # 空字典 + ) + ) + ) + + # parameters 字段将被自动排除 + print(tool.model_dump(exclude_unset=True)) \ No newline at end of file From 483c528baff1c2237c722b9ae65cdea8d0660396 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 09:05:27 +0000 Subject: [PATCH 426/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 524cb552..45a1b3f4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 +1.1.2 From b9b5443554ee94f9e01d9709fc737521d7c06a54 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 17:08:45 +0800 Subject: [PATCH 427/476] =?UTF-8?q?=E2=9C=A8=20Update=20search=20tool=20in?= =?UTF-8?q?tegration=20in=20get=5Fvertex=5Fgemini=5Fpayload:=20streamline?= =?UTF-8?q?=20tool=20assignment=20based=20on=20model=20type=20for=20improv?= =?UTF-8?q?ed=20functionality.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/request.py b/request.py index f7b7a80a..d742292d 100644 --- a/request.py +++ b/request.py @@ -376,11 +376,14 @@ async def get_vertex_gemini_payload(request, engine, provider): gemini_stream = "streamGenerateContent" model_dict = get_model_dict(provider) model = model_dict[request.model] + search_tool = None if "gemini-2.0" in model or "gemini-exp" in model: location = gemini2 + search_tool = {"googleSearch": {}} else: location = gemini1 + search_tool = {"googleSearchRetrieval": {}} url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) @@ -507,13 +510,9 @@ async def get_vertex_gemini_payload(request, engine, provider): if request.model.endswith("-search"): if "tools" not in payload: - payload["tools"] = [{ - "googleSearch": {} - }] + payload["tools"] = [search_tool] else: - payload["tools"].append({ - "googleSearch": {} - }) + payload["tools"].append(search_tool) return url, headers, payload From 6ddb56f1c7cf594ff225f2c90abc451c8f66c05d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 09:09:05 +0000 Subject: [PATCH 428/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 45a1b3f4..781dcb07 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.2 +1.1.3 From fee59cdfa220d47819c9f2fb00453ea00c219b1f Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 17:21:03 +0800 Subject: [PATCH 429/476] =?UTF-8?q?=E2=9C=A8=20Add=20JSON=20parsing=20meth?= =?UTF-8?q?od=20to=20Tool=20model=20and=20update=20example=20usage=20in=20?= =?UTF-8?q?main=20block;=20modify=20function=20declaration=20handling=20in?= =?UTF-8?q?=20get=5Fvertex=5Fgemini=5Fpayload=20for=20improved=20data=20ex?= =?UTF-8?q?traction.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 33 ++++++++++++++++++++++----------- request.py | 2 +- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/models.py b/models.py index ff0bb603..62df3ade 100644 --- a/models.py +++ b/models.py @@ -17,6 +17,11 @@ class Tool(BaseModel): type: str function: Function + @classmethod + def parse_raw(cls, json_str: str) -> 'Tool': + """从JSON字符串解析Tool对象""" + return cls.model_validate_json(json_str) + def model_dump(self, **kwargs): data = super().model_dump(**kwargs) function_data = data['function'] @@ -186,17 +191,23 @@ def set_request_type(cls, values): return values if __name__ == "__main__": - tool = Tool( - type="function", - function=Function( - name="clock-time____getCurrentTime____standalone", - description="获取当前时间", - parameters=FunctionParameter( - type="object", - properties={} # 空字典 - ) - ) - ) + # 示例JSON字符串 + json_str = ''' + { + "type": "function", + "function": { + "name": "clock-time____getCurrentTime____standalone", + "description": "获取当前时间", + "parameters": { + "type": "object", + "properties": {} + } + } + } + ''' + + # 解析JSON字符串为Tool对象 + tool = Tool.parse_raw(json_str) # parameters 字段将被自动排除 print(tool.model_dump(exclude_unset=True)) \ No newline at end of file diff --git a/request.py b/request.py index d742292d..6f72eb35 100644 --- a/request.py +++ b/request.py @@ -497,7 +497,7 @@ async def get_vertex_gemini_payload(request, engine, provider): if field == "tools": payload.update({ "tools": [{ - "function_declarations": [tool["function"] for tool in value] + "function_declarations": [tool.model_dump(exclude_unset=True)["function"] for tool in value] }], "tool_config": { "function_calling_config": { From 02c48a1aaa2743f70f4193b4a38318aba346bd31 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 09:21:22 +0000 Subject: [PATCH 430/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 781dcb07..65087b4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.3 +1.1.4 From 5e0dec92984678db3b851068ea1e8f5c708b0738 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 17:49:16 +0800 Subject: [PATCH 431/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20model=5Fdump=20m?= =?UTF-8?q?ethod=20in=20Tool=20and=20RequestModel=20classes:=20remove=20re?= =?UTF-8?q?dundant=20parameters=20handling=20in=20Tool,=20and=20enhance=20?= =?UTF-8?q?RequestModel=20to=20conditionally=20exclude=20empty=20parameter?= =?UTF-8?q?s=20in=20tools.=20Update=20function=20declaration=20handling=20?= =?UTF-8?q?in=20get=5Fvertex=5Fgemini=5Fpayload=20for=20improved=20payload?= =?UTF-8?q?=20structure.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models.py | 27 +++++++++----- request.py | 5 +-- test/test_request.py | 87 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 test/test_request.py diff --git a/models.py b/models.py index 62df3ade..751e4747 100644 --- a/models.py +++ b/models.py @@ -22,16 +22,6 @@ def parse_raw(cls, json_str: str) -> 'Tool': """从JSON字符串解析Tool对象""" return cls.model_validate_json(json_str) - def model_dump(self, **kwargs): - data = super().model_dump(**kwargs) - function_data = data['function'] - if 'parameters' in function_data and ( - function_data['parameters'] is None or - not function_data['parameters'].get('properties') - ): - function_data.pop('parameters', None) - return data - class FunctionCall(BaseModel): name: str arguments: str @@ -119,6 +109,23 @@ def get_last_text_message(self) -> Optional[str]: return item.text return "" + def model_dump(self, **kwargs): + data = super().model_dump(**kwargs) + + # 检查并处理 tools 字段 + if 'tools' in data and data['tools']: + for tool in data['tools']: + if 'function' in tool: + function_data = tool['function'] + # 如果 parameters 为空或没有 properties，则移除 + if 'parameters' in function_data and ( + function_data['parameters'] is None or + not function_data['parameters'].get('properties') + ): + function_data.pop('parameters', None) + + return data + class ImageGenerationRequest(BaseRequest): prompt: str model: Optional[str] = "dall-e-3" diff --git a/request.py b/request.py index 6f72eb35..34c2d322 100644 --- a/request.py +++ b/request.py @@ -8,7 +8,7 @@ import io from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gemini1, gemini2, BaseAPI, get_model_dict, provider_api_circular_list, safe_get +from utils import c35s, c3s, c3o, c3h, gemini1, gemini2, BaseAPI, get_model_dict, provider_api_circular_list, safe_get, ThreadSafeCircularList import imghdr @@ -491,13 +491,12 @@ async def get_vertex_gemini_payload(request, engine, provider): 'logprobs', 'top_logprobs' ] - for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: if field == "tools": payload.update({ "tools": [{ - "function_declarations": [tool.model_dump(exclude_unset=True)["function"] for tool in value] + "function_declarations": [tool["function"] for tool in value] }], "tool_config": { "function_calling_config": { diff --git a/test/test_request.py b/test/test_request.py new file mode 100644 index 00000000..a366acbb --- /dev/null +++ b/test/test_request.py @@ -0,0 +1,87 @@ +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from models import RequestModel +from request import get_payload +import json + +async def test_gemini_payload(): + # 构造测试请求 + request_data = { + "model": "gemini-1.5-pro-latest", + "messages": [ + { + "role": "system", + "content": "\n\nDisplay a clock to show current time\n获取当前时间\n\n" + }, + { + "role": "user", + "content": "几点了？" + } + ], + "stream": True, + "temperature": 1.3, + "top_p": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "user": "d5a7516e-e919-45f0-81a1-42bad3da6125", + "tools": [ + { + "type": "function", + "function": { + "name": "clock-time____getCurrentTime____standalone", + "description": "获取当前时间", + "parameters": { + "type": "object", + "properties": {} + } + } + } + ] + } + + request = RequestModel(**request_data) + provider = { + "provider": "gemini", + "base_url": "https://generativelanguage.googleapis.com", + "api": "your-api-key", # 测试时替换为实际的 API key + "model": ["gemini-1.5-pro-latest"], + "project_id": "your-project-id" + } + + url, headers, payload = await get_payload(request, "vertex-gemini", provider) + # url, headers, payload = await get_payload(request, "gemini", provider) + + print("payload", json.dumps(payload, indent=4, ensure_ascii=False)) + + # 验证生成的 payload 结构 + assert "contents" in payload + assert "tools" in payload + assert len(payload["tools"]) == 1 + assert "function_declarations" in payload["tools"][0] + + # 验证工具配置 + assert payload["tools"][0]["function_declarations"][0]["name"] == "clock-time____getCurrentTime____standalone" + assert payload["tools"][0]["function_declarations"][0]["description"] == "获取当前时间" + + # 验证消息内容 + assert len(payload["contents"]) == 2 + assert payload["contents"][0]["role"] == "system" + assert payload["contents"][1]["role"] == "user" + + # 验证其他参数 + assert payload["temperature"] == 1.3 + assert payload["top_p"] == 1.0 + + # 验证安全设置 + assert "safetySettings" in payload + assert len(payload["safetySettings"]) == 4 + + # 验证工具配置 + assert "tool_config" in payload + assert payload["tool_config"]["function_calling_config"]["mode"] == "AUTO" + +if __name__ == "__main__": + import asyncio + asyncio.run(test_gemini_payload()) \ No newline at end of file From 5f495f1b03d8496535ea929c5577df9aef6b73b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 09:49:41 +0000 Subject: [PATCH 432/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 65087b4f..e25d8d9f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.4 +1.1.5 From f9fe744232c454d396034436536cc5f04c9a7472 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 18:51:15 +0800 Subject: [PATCH 433/476] =?UTF-8?q?=E2=9C=A8=20Update=20endpoint=20handlin?= =?UTF-8?q?g=20in=20process=5Frequest=20function:=20allow=20'stable-diffus?= =?UTF-8?q?ion'=20model=20to=20trigger=20'dalle'=20engine=20alongside=20ex?= =?UTF-8?q?isting=20'/v1/images/generations'=20endpoint=20check=20for=20im?= =?UTF-8?q?proved=20model=20compatibility.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 66ae25f6..bd8adca2 100644 --- a/main.py +++ b/main.py @@ -859,7 +859,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if "gemini" in original_model and engine == "vertex": engine = "vertex-gemini" - if endpoint == "/v1/images/generations": + if endpoint == "/v1/images/generations" or "stable-diffusion" in original_model: engine = "dalle" request.stream = False From eed99a3ae46d777294fa8ed749d6071141c4be29 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 10:51:34 +0000 Subject: [PATCH 434/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e25d8d9f..0664a8fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.5 +1.1.6 From 680e8e454c7b0f80c989cd9783fc2e3c1c0b2e2b Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 16 Jan 2025 19:08:37 +0800 Subject: [PATCH 435/476] =?UTF-8?q?=E2=9C=A8=20Update=20engine=20assignmen?= =?UTF-8?q?t=20logic=20in=20process=5Frequest=20function:=20streamline=20e?= =?UTF-8?q?ngine=20selection=20by=20allowing=20provider=20engine=20to=20ta?= =?UTF-8?q?ke=20precedence,=20improving=20flexibility=20in=20handling=20di?= =?UTF-8?q?fferent=20models.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index bd8adca2..00837787 100644 --- a/main.py +++ b/main.py @@ -859,6 +859,9 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if "gemini" in original_model and engine == "vertex": engine = "vertex-gemini" + if provider.get("engine"): + engine = provider["engine"] + if endpoint == "/v1/images/generations" or "stable-diffusion" in original_model: engine = "dalle" request.stream = False @@ -878,9 +881,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A engine = "tts" request.stream = False - if provider.get("engine"): - engine = provider["engine"] - channel_id = f"{provider['provider']}" if engine != "moderation": logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine} role: {role}") From d24f6a9824c3f6747621f8dc713f543a2f176365 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jan 2025 11:08:58 +0000 Subject: [PATCH 436/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0664a8fd..2bf1ca5f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.6 +1.1.7 From 07b827d1e3efbbe4baa3d09c5aad57a123fb4747 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 19 Jan 2025 19:24:45 +0800 Subject: [PATCH 437/476] =?UTF-8?q?=E2=9C=A8=20Enhance=20model=20selection?= =?UTF-8?q?=20logic=20in=20process=5Frequest=20function:=20include=20'deep?= =?UTF-8?q?seek'=20in=20the=20engine=20exclusion=20criteria=20for=20improv?= =?UTF-8?q?ed=20model=20handling.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 00837787..36cd8bce 100644 --- a/main.py +++ b/main.py @@ -845,6 +845,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A if "claude" not in original_model \ and "gpt" not in original_model \ + and "deepseek" not in original_model \ and "o1" not in original_model \ and "gemini" not in original_model \ and "learnlm" not in original_model \ From 346eb0e38f211d204791f5ca7eb28a26704ba5ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Jan 2025 11:25:05 +0000 Subject: [PATCH 438/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2bf1ca5f..18efdb9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.7 +1.1.8 From 74a0abd67a15db631234cbb10622f63653b295cf Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 19 Jan 2025 22:10:52 +0800 Subject: [PATCH 439/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20get=5Fgemini=5Fp?= =?UTF-8?q?ayload=20function:=20streamline=20generation=20configuration=20?= =?UTF-8?q?handling=20by=20consolidating=20temperature,=20max=5Ftokens,=20?= =?UTF-8?q?and=20top=5Fp=20into=20a=20generation=5Fconfig=20dictionary.=20?= =?UTF-8?q?Ensure=20default=20maxOutputTokens=20is=20set=20to=208192=20if?= =?UTF-8?q?=20not=20provided,=20enhancing=20payload=20structure=20for=20im?= =?UTF-8?q?proved=20model=20compatibility.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/request.py b/request.py index 34c2d322..2396a4e1 100644 --- a/request.py +++ b/request.py @@ -237,6 +237,7 @@ async def get_gemini_payload(request, engine, provider): } ] } + if systemInstruction: if api_version == "v1beta": payload["systemInstruction"] = systemInstruction @@ -251,9 +252,6 @@ async def get_gemini_payload(request, engine, provider): 'messages', 'stream', 'tool_choice', - 'temperature', - 'top_p', - 'max_tokens', 'presence_penalty', 'frequency_penalty', 'n', @@ -263,6 +261,7 @@ async def get_gemini_payload(request, engine, provider): 'top_logprobs', 'response_format' ] + generation_config = None for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: @@ -293,9 +292,20 @@ async def get_gemini_payload(request, engine, provider): } } }) + elif field == "temperature": + generation_config["temperature"] = value + elif field == "max_tokens": + generation_config["maxOutputTokens"] = value + elif field == "top_p": + generation_config["topP"] = value else: payload[field] = value + if generation_config: + payload["generationConfig"] = generation_config + if "maxOutputTokens" not in generation_config: + payload["generationConfig"]["maxOutputTokens"] = 8192 + if request.model.endswith("-search"): if "tools" not in payload: payload["tools"] = [{ From b2ef86f39815f6831929337e5297e69ac51d8a7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Jan 2025 14:11:11 +0000 Subject: [PATCH 440/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 18efdb9a..512a1faa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.8 +1.1.9 From ac3a65cf30bd8ed1aaea9620b87b1d195d41c1be Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 19 Jan 2025 22:30:56 +0800 Subject: [PATCH 441/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20get=5Fvertex=5Fg?= =?UTF-8?q?emini=5Fpayload=20function:=20remove=20direct=20generation=20co?= =?UTF-8?q?nfiguration=20parameters=20and=20consolidate=20them=20into=20a?= =?UTF-8?q?=20generation=5Fconfig=20dictionary.=20Ensure=20default=20max?= =?UTF-8?q?=5Foutput=5Ftokens=20is=20set=20to=208192=20if=20not=20specifie?= =?UTF-8?q?d,=20improving=20payload=20structure=20and=20model=20compatibil?= =?UTF-8?q?ity.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/request.py b/request.py index 2396a4e1..5fb6120a 100644 --- a/request.py +++ b/request.py @@ -475,12 +475,6 @@ async def get_vertex_gemini_payload(request, engine, provider): # "threshold": "BLOCK_NONE" # } # ] - "generationConfig": { - "temperature": 0.5, - "max_output_tokens": 8192, - "top_k": 40, - "top_p": 0.95 - }, } if systemInstruction: payload["system_instruction"] = systemInstruction @@ -490,9 +484,6 @@ async def get_vertex_gemini_payload(request, engine, provider): 'messages', 'stream', 'tool_choice', - 'temperature', - 'top_p', - 'max_tokens', 'presence_penalty', 'frequency_penalty', 'n', @@ -501,6 +492,8 @@ async def get_vertex_gemini_payload(request, engine, provider): 'logprobs', 'top_logprobs' ] + generation_config = None + for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: if field == "tools": @@ -514,9 +507,20 @@ async def get_vertex_gemini_payload(request, engine, provider): } } }) + elif field == "temperature": + generation_config["temperature"] = value + elif field == "max_tokens": + generation_config["max_output_tokens"] = value + elif field == "top_p": + generation_config["top_p"] = value else: payload[field] = value + if generation_config: + payload["generationConfig"] = generation_config + if "max_output_tokens" not in generation_config: + payload["generationConfig"]["max_output_tokens"] = 8192 + if request.model.endswith("-search"): if "tools" not in payload: payload["tools"] = [search_tool] From d086aafdddb3a807ec1b1697375aa83fc759eddf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Jan 2025 14:31:29 +0000 Subject: [PATCH 442/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?1.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 512a1faa..5ed5faa5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.9 +1.1.10 From 70ae8d316204da30090f3de2a56897c4600bde27 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 19 Jan 2025 22:49:55 +0800 Subject: [PATCH 443/476] =?UTF-8?q?=E2=9C=A8=20Update=20generation=20confi?= =?UTF-8?q?guration=20handling=20in=20get=5Fgemini=5Fpayload=20and=20get?= =?UTF-8?q?=5Fvertex=5Fgemini=5Fpayload=20functions:=20replace=20None=20wi?= =?UTF-8?q?th=20an=20empty=20dictionary=20for=20generation=5Fconfig=20to?= =?UTF-8?q?=20improve=20payload=20structure.=20=F0=9F=93=96=20Bump=20versi?= =?UTF-8?q?on=20to=201.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- request.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 5ed5faa5..26aaba0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.10 +1.2.0 diff --git a/request.py b/request.py index 5fb6120a..752acca0 100644 --- a/request.py +++ b/request.py @@ -261,7 +261,7 @@ async def get_gemini_payload(request, engine, provider): 'top_logprobs', 'response_format' ] - generation_config = None + generation_config = {} for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: @@ -492,7 +492,7 @@ async def get_vertex_gemini_payload(request, engine, provider): 'logprobs', 'top_logprobs' ] - generation_config = None + generation_config = {} for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: From 206bb464569edf8bca7e7ef485081c934dc1c1ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Jan 2025 14:50:20 +0000 Subject: [PATCH 444/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 26aaba0e..6085e946 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.2.1 From 3450f22578da460c46eaf4b49b8c52d35e7fc9e3 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 22 Jan 2025 16:11:03 +0800 Subject: [PATCH 445/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20fetch=5Fgemini?= =?UTF-8?q?=5Fresponse=5Fstream=20function:=20comment=20out=20unused=20var?= =?UTF-8?q?iables=20and=20logic=20related=20to=20line=20indexing=20and=20t?= =?UTF-8?q?hinking=20state,=20simplifying=20the=20code=20structure=20and?= =?UTF-8?q?=20improving=20readability.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- response.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/response.py b/response.py index f4ee5d8f..7652944d 100644 --- a/response.py +++ b/response.py @@ -30,19 +30,19 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): revicing_function_call = False function_full_response = "{" need_function_call = False - line_index = 0 - last_text_line = 0 is_finish = False - if "thinking" in model: - is_thinking = True - else: - is_thinking = False + # line_index = 0 + # last_text_line = 0 + # if "thinking" in model: + # is_thinking = True + # else: + # is_thinking = False async for chunk in response.aiter_text(): buffer += chunk while "\n" in buffer: line, buffer = buffer.split("\n", 1) - line_index += 1 + # line_index += 1 if line and '\"finishReason\": \"' in line: is_finish = True break @@ -53,18 +53,18 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): content = json_data.get('text', '') content = "\n".join(content.split("\\n")) # content = content.replace("\n", "\n\n") - if last_text_line == 0 and is_thinking: - content = "> " + content.lstrip() - if is_thinking: - content = content.replace("\n", "\n> ") - if last_text_line == line_index - 3: - is_thinking = False - content = "\n\n\n" + content.lstrip() + # if last_text_line == 0 and is_thinking: + # content = "> " + content.lstrip() + # if is_thinking: + # content = content.replace("\n", "\n> ") + # if last_text_line == line_index - 3: + # is_thinking = False + # content = "\n\n\n" + content.lstrip() sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") - last_text_line = line_index + # last_text_line = line_index if line and ('\"functionCall\": {' in line or revicing_function_call): revicing_function_call = True From 02c1c3a2c2f13ee59ecaca9a9a63ede2f28b5109 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Jan 2025 08:11:30 +0000 Subject: [PATCH 446/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6085e946..23aa8390 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.1 +1.2.2 From 8d73a5b7905244de20a84feb4035c385ef7bfce6 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 22 Jan 2025 17:29:16 +0800 Subject: [PATCH 447/476] =?UTF-8?q?=E2=9C=A8=20Normalize=20original=5Fmode?= =?UTF-8?q?l=20input=20in=20get=5Ftimeout=5Fvalue=20function:=20convert=20?= =?UTF-8?q?original=5Fmodel=20to=20lowercase=20to=20ensure=20consistent=20?= =?UTF-8?q?timeout=20value=20retrieval=20from=20provider=5Ftimeouts.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 36cd8bce..0cf0cc7f 100644 --- a/main.py +++ b/main.py @@ -805,6 +805,7 @@ async def ensure_config(request: Request, call_next): def get_timeout_value(provider_timeouts, original_model): timeout_value = None + original_model = original_model.lower() if original_model in provider_timeouts: timeout_value = provider_timeouts[original_model] else: From e76805787b552aecf255e9bd48682e179c94d18d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Jan 2025 09:29:36 +0000 Subject: [PATCH 448/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 23aa8390..0495c4a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.2 +1.2.3 From 34f1e5e7c0c41449424cdbc4ef7ec5610616e889 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Thu, 23 Jan 2025 16:01:12 +0800 Subject: [PATCH 449/476] =?UTF-8?q?=E2=9C=A8=20Update=20model=20ownership?= =?UTF-8?q?=20in=20post=5Fall=5Fmodels=20function:=20change=20owned=5Fby?= =?UTF-8?q?=20field=20to=20"uni-api"=20for=20improved=20clarity=20and=20co?= =?UTF-8?q?nsistency=20in=20model=20information.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 2b4af3df..ef16fd5e 100644 --- a/utils.py +++ b/utils.py @@ -574,7 +574,7 @@ def post_all_models(api_index, config, api_list, models_list): "id": model, "object": "model", "created": 1720524448858, - "owned_by": model + "owned_by": "uni-api" } all_models.append(model_info) From 8cfc2fb16a6f7847f0f086c107c3c032b2e2b8e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 23 Jan 2025 08:01:32 +0000 Subject: [PATCH 450/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0495c4a8..e8ea05db 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.3 +1.2.4 From 282778e6ca18edd3593f4d4bfaf168123ef62402 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 24 Jan 2025 04:56:07 +0800 Subject: [PATCH 451/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20image=20encoding?= =?UTF-8?q?=20in=20request.py:=20replace=20imghdr=20with=20a=20custom=20ge?= =?UTF-8?q?t=5Fimage=5Fformat=20function=20for=20improved=20image=20format?= =?UTF-8?q?=20detection=20and=20error=20handling,=20ensuring=20better=20su?= =?UTF-8?q?pport=20for=20various=20image=20types.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/request.py b/request.py index 752acca0..db1be13c 100644 --- a/request.py +++ b/request.py @@ -10,20 +10,27 @@ from models import RequestModel from utils import c35s, c3s, c3o, c3h, gemini1, gemini2, BaseAPI, get_model_dict, provider_api_circular_list, safe_get, ThreadSafeCircularList -import imghdr +def get_image_format(file_content): + try: + img = Image.open(io.BytesIO(file_content)) + return img.format.lower() + except: + return None def encode_image(image_path): with open(image_path, "rb") as image_file: file_content = image_file.read() - file_type = imghdr.what(None, file_content) + img_format = get_image_format(file_content) + if not img_format: + raise ValueError("无法识别的图片格式") base64_encoded = base64.b64encode(file_content).decode('utf-8') - if file_type == 'png': + if img_format == 'png': return f"data:image/png;base64,{base64_encoded}" - elif file_type in ['jpeg', 'jpg']: + elif img_format in ['jpg', 'jpeg']: return f"data:image/jpeg;base64,{base64_encoded}" else: - raise ValueError(f"不支持的图片格式: {file_type}") + raise ValueError(f"不支持的图片格式: {img_format}") async def get_doc_from_url(url): filename = urllib.parse.unquote(url.split("/")[-1]) From 68e757cf578fa4b9d27222f452c49f6875a09abe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 23 Jan 2025 20:56:50 +0000 Subject: [PATCH 452/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e8ea05db..c813fe11 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.4 +1.2.5 From 81a66fde5e4a330dc81c244b6454c0d939040757 Mon Sep 17 00:00:00 2001 From: Jingqi Date: Fri, 24 Jan 2025 10:46:44 +0800 Subject: [PATCH 453/476] Add Azure support for embeddings endpoint 1.Add function param to build_azure_endpoint 2.Handle Azure-specific headers for embeddings --- request.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/request.py b/request.py index db1be13c..7d791be4 100644 --- a/request.py +++ b/request.py @@ -788,12 +788,12 @@ async def get_gpt_payload(request, engine, provider): return url, headers, payload -def build_azure_endpoint(base_url, deployment_id, api_version="2024-10-21"): +def build_azure_endpoint(base_url, deployment_id, function="chat/completions", api_version="2024-10-21"): # 移除base_url末尾的斜杠(如果有) base_url = base_url.rstrip('/') # 构建路径 - path = f"/openai/deployments/{deployment_id}/chat/completions" + path = f"/openai/deployments/{deployment_id}/{function}" # 使用urljoin拼接base_url和path full_url = urllib.parse.urljoin(base_url, path) @@ -1291,10 +1291,18 @@ async def get_embedding_payload(request, engine, provider): headers = { "Content-Type": "application/json", } - if provider.get("api"): - headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" url = provider['base_url'] - url = BaseAPI(url).embeddings + is_azure = "openai.azure.com" in url + if provider.get("api"): + if is_azure: + headers['api-key'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" + else: + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + if is_azure: + url = build_azure_endpoint(url, model, function="embeddings") + else: + url = BaseAPI(url).embeddings payload = { "input": request.input, @@ -1366,4 +1374,4 @@ async def get_payload(request: RequestModel, engine, provider): elif engine == "embedding": return await get_embedding_payload(request, engine, provider) else: - raise ValueError("Unknown payload") \ No newline at end of file + raise ValueError("Unknown payload") From 67de1dc15a3f787328bc5b6e4d01edbeb4bcbf1a Mon Sep 17 00:00:00 2001 From: Jingqi Date: Fri, 24 Jan 2025 14:08:54 +0800 Subject: [PATCH 454/476] Update request.py --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index 7d791be4..6df46992 100644 --- a/request.py +++ b/request.py @@ -1292,7 +1292,7 @@ async def get_embedding_payload(request, engine, provider): "Content-Type": "application/json", } url = provider['base_url'] - is_azure = "openai.azure.com" in url + is_azure = url.endswith(".azure.com") if provider.get("api"): if is_azure: headers['api-key'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" From 0bf651afaf320c554cbd1dc8184b0dfe551c6325 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 26 Jan 2025 15:56:57 +0800 Subject: [PATCH 455/476] =?UTF-8?q?=E2=9C=A8=20Enhance=20error=20handling?= =?UTF-8?q?=20in=20utils.py:=20add=20support=20for=20parsing=20nested=20er?= =?UTF-8?q?ror=20responses=20and=20improve=20network=20error=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index ef16fd5e..9a902944 100644 --- a/utils.py +++ b/utils.py @@ -466,6 +466,12 @@ async def error_handling_wrapper(generator, channel_id, engine, stream, error_tr detail = first_item_str.get('details', f"{first_item_str}") raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) + if isinstance(first_item_str, dict) and safe_get(first_item_str, "choices", 0, "error", default=None): + # 如果第一个 yield 的项是错误信息，抛出 HTTPException + status_code = safe_get(first_item_str, "choices", 0, "error", "code", default=500) + detail = safe_get(first_item_str, "choices", 0, "error", "message", default=f"{first_item_str}") + raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) + if isinstance(first_item_str, dict) and engine not in ["tts", "embedding", "dalle", "moderation", "whisper"] and stream == False: if any(x in str(first_item_str) for x in error_triggers): logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) @@ -488,8 +494,8 @@ async def new_generator(): return except (httpx.ReadError, httpx.RemoteProtocolError) as e: # 只记录真正的网络错误 - logger.error(f"provider: {channel_id:<11} Network error in new_generator: {e}") - raise + # logger.error(f"provider: {channel_id:<11} Network error in new_generator: {e}") + raise HTTPException(status_code=502, detail=f"Network error in new_generator: {e}") return new_generator(), first_response_time From e07de17c717949250bfe721577d2291e1b57e0de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 26 Jan 2025 07:57:17 +0000 Subject: [PATCH 456/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c813fe11..3c43790f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.5 +1.2.6 From e101a6b5b7707917284519f082219e6bf0df6c0d Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sat, 1 Feb 2025 13:36:41 +0800 Subject: [PATCH 457/476] =?UTF-8?q?=E2=9C=A8=20Add=20reasoning=20effort=20?= =?UTF-8?q?configuration=20for=20O3-mini=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/request.py b/request.py index db1be13c..bde3c968 100644 --- a/request.py +++ b/request.py @@ -767,6 +767,14 @@ async def get_gpt_payload(request, engine, provider): # request.stream = False payload.pop("stream_options", None) + if "o3-mini" in model: + if request.model.endswith("high"): + payload["reasoning_effort"] = "high" + elif request.model.endswith("low"): + payload["reasoning_effort"] = "low" + else: + payload["reasoning_effort"] = "medium" + if request.model.endswith("-search") and "gemini" in request.model: if "tools" not in payload: payload["tools"] = [{ From 1b165d58bba02a49eedc5c05f56f76fa74a9c926 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 1 Feb 2025 05:37:02 +0000 Subject: [PATCH 458/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3c43790f..c04c650a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.6 +1.2.7 From 93e2902803c457ccf5ca3a653d10c7d3de3de2c7 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 2 Feb 2025 02:26:23 +0800 Subject: [PATCH 459/476] =?UTF-8?q?=F0=9F=94=A7=20Refactor=20Gemini=20payl?= =?UTF-8?q?oad=20generation:=20improve=20tool=20processing=20and=20add=20c?= =?UTF-8?q?onditional=20tool=20inclusion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/request.py b/request.py index bde3c968..9c887eee 100644 --- a/request.py +++ b/request.py @@ -287,18 +287,20 @@ async def get_gemini_payload(request, engine, provider): prop_value["description"] = f"{description}\nDefault: {default_value}" # 删除 default 字段 del prop_value["default"] - processed_tools.append({"function": function_def}) - - payload.update({ - "tools": [{ - "function_declarations": [tool["function"] for tool in processed_tools] - }], - "tool_config": { - "function_calling_config": { - "mode": "AUTO" + if function_def["name"] != "googleSearch" and function_def["name"] != "googleSearch": + processed_tools.append({"function": function_def}) + + if processed_tools: + payload.update({ + "tools": [{ + "function_declarations": [tool["function"] for tool in processed_tools] + }], + "tool_config": { + "function_calling_config": { + "mode": "AUTO" + } } - } - }) + }) elif field == "temperature": generation_config["temperature"] = value elif field == "max_tokens": From c3e66c24aa937f1914d2faf6c5dd1982af1e8a51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 1 Feb 2025 18:26:44 +0000 Subject: [PATCH 460/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c04c650a..db6fb4a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.7 +1.2.8 From 1e2d9d0e52e220226a5158666c8958d88d17c697 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 2 Feb 2025 02:51:01 +0800 Subject: [PATCH 461/476] =?UTF-8?q?=E2=9C=A8=20Enhance=20API=20request=20h?= =?UTF-8?q?andling:=20add=20O3=20model=20support=20and=20custom=20API=20ke?= =?UTF-8?q?y=20authorization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 1 + request.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/main.py b/main.py index 0cf0cc7f..7530badb 100644 --- a/main.py +++ b/main.py @@ -848,6 +848,7 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A and "gpt" not in original_model \ and "deepseek" not in original_model \ and "o1" not in original_model \ + and "o3" not in original_model \ and "gemini" not in original_model \ and "learnlm" not in original_model \ and "grok" not in original_model \ diff --git a/request.py b/request.py index 9c887eee..bbfd00ef 100644 --- a/request.py +++ b/request.py @@ -895,6 +895,9 @@ async def get_openrouter_payload(request, engine, provider): if provider.get("api"): headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + elif provider['provider'].startswith("sk-"): + headers['Authorization'] = f"Bearer {provider['provider']}" + url = provider['base_url'] messages = [] From bf21c5d335af5ce6d4f62dcf63691c39e61eb110 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 1 Feb 2025 18:51:29 +0000 Subject: [PATCH 462/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index db6fb4a9..9d4f8239 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.8 +1.2.9 From c098e7361f67c00f1983dea7469c9e8fd0f2fb46 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Sun, 2 Feb 2025 21:40:35 +0800 Subject: [PATCH 463/476] =?UTF-8?q?=F0=9F=90=9B=20Bug:=20Fix=20the=20bug?= =?UTF-8?q?=20in=20the=20o3=20request=20parameter=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request.py b/request.py index bbfd00ef..583cc9e7 100644 --- a/request.py +++ b/request.py @@ -756,7 +756,7 @@ async def get_gpt_payload(request, engine, provider): for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: - if field == "max_tokens" and "o1" in model: + if field == "max_tokens" and ("o1" in model or "o3" in model): payload["max_completion_tokens"] = value else: payload[field] = value From 546214ca103970c4d056e52e320c915b7d34d33a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 2 Feb 2025 13:40:59 +0000 Subject: [PATCH 464/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9d4f8239..963ed7cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.9 +1.2.10 From b31be3c4df55a46a6200be29751d655885797cbb Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 4 Feb 2025 03:42:01 +0800 Subject: [PATCH 465/476] =?UTF-8?q?=F0=9F=90=9B=20Fix:=20Remove=20temperat?= =?UTF-8?q?ure=20parameter=20from=20GPT=20payload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/request.py b/request.py index 583cc9e7..1fa4e479 100644 --- a/request.py +++ b/request.py @@ -776,6 +776,8 @@ async def get_gpt_payload(request, engine, provider): payload["reasoning_effort"] = "low" else: payload["reasoning_effort"] = "medium" + if "temperature" in payload: + payload.pop("temperature") if request.model.endswith("-search") and "gemini" in request.model: if "tools" not in payload: From 90a877e328e691508da1ea73e2f3f45782cae18a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 3 Feb 2025 19:42:25 +0000 Subject: [PATCH 466/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 963ed7cf..c1147005 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.10 +1.2.11 From 63cb7002cc9dd1c620a5ee1a50c8c63736c65794 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Tue, 4 Feb 2025 21:23:07 +0800 Subject: [PATCH 467/476] =?UTF-8?q?=F0=9F=90=9B=20Fix:=20Remove=20temperat?= =?UTF-8?q?ure=20for=20O3-mini=20and=20O1=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/request.py b/request.py index 1fa4e479..86edfdb4 100644 --- a/request.py +++ b/request.py @@ -776,6 +776,8 @@ async def get_gpt_payload(request, engine, provider): payload["reasoning_effort"] = "low" else: payload["reasoning_effort"] = "medium" + + if "o3-mini" in model or "o1" in model: if "temperature" in payload: payload.pop("temperature") From 8dbb944bf15f769a76de4a2b9dcf5e8e63d8bf84 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 4 Feb 2025 13:23:27 +0000 Subject: [PATCH 468/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c1147005..f2ae0b4a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.11 +1.2.12 From 1764a880cf935d58f3398f884d73f6fa674d6167 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 7 Feb 2025 03:17:54 +0800 Subject: [PATCH 469/476] =?UTF-8?q?=F0=9F=90=9B=20Fix:=20Prevent=20auto-re?= =?UTF-8?q?try=20for=20413=20Payload=20Too=20Large=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 7530badb..292996c1 100644 --- a/main.py +++ b/main.py @@ -1237,7 +1237,7 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques if is_debug: import traceback traceback.print_exc() - if auto_retry: + if auto_retry and status_code != 413: continue else: return JSONResponse( From f7e8c6d6fc9198475143fbf041fbb3b23fc0dbd5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 6 Feb 2025 19:18:18 +0000 Subject: [PATCH 470/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f2ae0b4a..0b1f1edf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.12 +1.2.13 From 365309ac840612b077f5603a91a9e86bb5ee32b9 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 7 Feb 2025 17:37:29 +0800 Subject: [PATCH 471/476] =?UTF-8?q?=E2=9C=A8=20Enhance=20timeout=20calcula?= =?UTF-8?q?tion=20for=20multi-provider=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 292996c1..b663e115 100644 --- a/main.py +++ b/main.py @@ -820,7 +820,7 @@ def get_timeout_value(provider_timeouts, original_model): return timeout_value # 在 process_request 函数中更新成功和失败计数 -async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, role=None): +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, role=None, num_matching_providers=1): url = provider['base_url'] parsed_url = urlparse(url) # print("parsed_url", parsed_url) @@ -905,7 +905,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A timeout_value = get_timeout_value(app.state.provider_timeouts["global_time_out"], original_model) if timeout_value is None: timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) - # print("timeout_value", timeout_value) + timeout_value = timeout_value * num_matching_providers + # print("timeout_value", channel_id, timeout_value) proxy = safe_get(provider, "preferences", "proxy", default=None) # print("proxy", proxy) @@ -1187,8 +1188,17 @@ async def request_model(self, request: Union[RequestModel, ImageGenerationReques current_index = (start_index + index) % num_matching_providers index += 1 provider = matching_providers[current_index] + + if provider['provider'].startswith("sk-") and provider['provider'] in app.state.api_list: + local_provider_api_index = app.state.api_list.index(provider['provider']) + local_provider_scheduling_algorithm = safe_get(config, 'api_keys', local_provider_api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") + local_provider_matching_providers = await get_right_order_providers(request_model, config, local_provider_api_index, local_provider_scheduling_algorithm) + local_provider_num_matching_providers = len(local_provider_matching_providers) + else: + local_provider_num_matching_providers = 1 + try: - response = await process_request(request, provider, endpoint, role) + response = await process_request(request, provider, endpoint, role, local_provider_num_matching_providers) return response except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: From 56f45be6f1b0ad2456b95b37f2093c8f0583f5a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 7 Feb 2025 09:37:56 +0000 Subject: [PATCH 472/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.14?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0b1f1edf..fd9d1a5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.13 +1.2.14 From 04170c71d47e97cac7be682a431e3084669c2dc5 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 7 Feb 2025 19:32:38 +0800 Subject: [PATCH 473/476] =?UTF-8?q?=E2=9C=A8=20Refactor=20engine=20detecti?= =?UTF-8?q?on=20logic=20into=20a=20separate=20utility=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 64 +++--------------------------- utils.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 107 insertions(+), 73 deletions(-) diff --git a/main.py b/main.py index b663e115..b782ab48 100644 --- a/main.py +++ b/main.py @@ -20,6 +20,7 @@ from response import fetch_response, fetch_response_stream from utils import ( safe_get, + get_engine, load_config, save_api_yaml, get_model_dict, @@ -821,68 +822,13 @@ def get_timeout_value(provider_timeouts, original_model): # 在 process_request 函数中更新成功和失败计数 async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, role=None, num_matching_providers=1): - url = provider['base_url'] - parsed_url = urlparse(url) - # print("parsed_url", parsed_url) - engine = None - if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): - engine = "gemini" - elif parsed_url.netloc == 'aiplatform.googleapis.com': - engine = "vertex" - elif parsed_url.netloc.rstrip('/').endswith('openai.azure.com'): - engine = "azure" - elif parsed_url.netloc == 'api.cloudflare.com': - engine = "cloudflare" - elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): - engine = "claude" - elif parsed_url.netloc == 'api.cohere.com': - engine = "cohere" - request.stream = True - else: - engine = "gpt" - model_dict = get_model_dict(provider) original_model = model_dict[request.model] - if "claude" not in original_model \ - and "gpt" not in original_model \ - and "deepseek" not in original_model \ - and "o1" not in original_model \ - and "o3" not in original_model \ - and "gemini" not in original_model \ - and "learnlm" not in original_model \ - and "grok" not in original_model \ - and parsed_url.netloc != 'api.cloudflare.com' \ - and parsed_url.netloc != 'api.cohere.com': - engine = "openrouter" - - if "claude" in original_model and engine == "vertex": - engine = "vertex-claude" - - if "gemini" in original_model and engine == "vertex": - engine = "vertex-gemini" - - if provider.get("engine"): - engine = provider["engine"] - - if endpoint == "/v1/images/generations" or "stable-diffusion" in original_model: - engine = "dalle" - request.stream = False - - if endpoint == "/v1/audio/transcriptions": - engine = "whisper" - request.stream = False - - if endpoint == "/v1/moderations": - engine = "moderation" - request.stream = False - - if endpoint == "/v1/embeddings": - engine = "embedding" - - if endpoint == "/v1/audio/speech": - engine = "tts" - request.stream = False + engine, stream_mode = get_engine(provider, endpoint, original_model) + + if stream_mode != None: + request.stream = stream_mode channel_id = f"{provider['provider']}" if engine != "moderation": diff --git a/utils.py b/utils.py index 9a902944..1199dc56 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,7 @@ import json from fastapi import HTTPException import httpx +from urllib.parse import urlparse from log_config import logger @@ -204,20 +205,43 @@ def get_model_dict(provider): model_dict.update({new: old for old, new in model.items()}) return model_dict -def update_initial_model(api_url, api): +def update_initial_model(provider): try: - endpoint = BaseAPI(api_url=api_url) - endpoint_models_url = endpoint.v1_models - if isinstance(api, list): - api = api[0] - headers = {"Authorization": f"Bearer {api}"} - response = httpx.get( - endpoint_models_url, - headers=headers, - ) - models = response.json() - if models.get("error"): - raise Exception({"error": models.get("error"), "endpoint": endpoint_models_url, "api": api}) + engine, stream_mode = get_engine(provider, endpoint=None, original_model="") + # print("engine", engine, provider) + api_url = provider['base_url'] + api = provider['api'] + + if engine == "gemini": + url = "https://generativelanguage.googleapis.com/v1beta/models" + params = {"key": api} + + with httpx.Client() as client: + response = client.get(url, params=params) + + original_models = response.json() + if original_models.get("error"): + raise Exception({"error": original_models.get("error"), "endpoint": url, "api": api}) + + models = {"data": []} + for model in original_models["models"]: + models["data"].append({ + "id": model["name"].split("models/")[-1], + }) + else: + endpoint = BaseAPI(api_url=api_url) + endpoint_models_url = endpoint.v1_models + if isinstance(api, list): + api = api[0] + headers = {"Authorization": f"Bearer {api}"} + response = httpx.get( + endpoint_models_url, + headers=headers, + ) + models = response.json() + if models.get("error"): + raise Exception({"error": models.get("error"), "endpoint": endpoint_models_url, "api": api}) + # print(models) models_list = models["data"] models_id = [model["id"] for model in models_list] @@ -283,7 +307,7 @@ def update_config(config_data, use_config_url=False): ] if not provider.get("model"): - model_list = update_initial_model(provider['base_url'], provider['api']) + model_list = update_initial_model(provider) if model_list: provider["model"] = model_list if not use_config_url: @@ -745,3 +769,67 @@ async def generate_no_stream_response(timestamp, model, content=None, tools_id=N json_data = json.dumps(sample_data, ensure_ascii=False) return json_data + +def get_engine(provider, endpoint=None, original_model=""): + parsed_url = urlparse(provider['base_url']) + # print("parsed_url", parsed_url) + engine = None + stream = None + if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): + engine = "gemini" + elif parsed_url.netloc == 'aiplatform.googleapis.com': + engine = "vertex" + elif parsed_url.netloc.rstrip('/').endswith('openai.azure.com'): + engine = "azure" + elif parsed_url.netloc == 'api.cloudflare.com': + engine = "cloudflare" + elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): + engine = "claude" + elif parsed_url.netloc == 'api.cohere.com': + engine = "cohere" + stream = True + else: + engine = "gpt" + + if original_model \ + and "claude" not in original_model \ + and "gpt" not in original_model \ + and "deepseek" not in original_model \ + and "o1" not in original_model \ + and "o3" not in original_model \ + and "gemini" not in original_model \ + and "learnlm" not in original_model \ + and "grok" not in original_model \ + and parsed_url.netloc != 'api.cloudflare.com' \ + and parsed_url.netloc != 'api.cohere.com': + engine = "openrouter" + + if "claude" in original_model and engine == "vertex": + engine = "vertex-claude" + + if "gemini" in original_model and engine == "vertex": + engine = "vertex-gemini" + + if provider.get("engine"): + engine = provider["engine"] + + if endpoint == "/v1/images/generations" or "stable-diffusion" in original_model: + engine = "dalle" + stream = False + + if endpoint == "/v1/audio/transcriptions": + engine = "whisper" + stream = False + + if endpoint == "/v1/moderations": + engine = "moderation" + stream = False + + if endpoint == "/v1/embeddings": + engine = "embedding" + + if endpoint == "/v1/audio/speech": + engine = "tts" + stream = False + + return engine, stream \ No newline at end of file From b62393e3290e3b4282b16261cba2e76c40dc9c8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 7 Feb 2025 11:32:58 +0000 Subject: [PATCH 474/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?2.15?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd9d1a5a..1fc5b820 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.14 +1.2.15 From 931b39fe286c0689dd45dc2a5a06246539d38430 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Fri, 7 Feb 2025 19:50:35 +0800 Subject: [PATCH 475/476] =?UTF-8?q?=E2=9C=A8=20Add=20Claude=20engine=20sup?= =?UTF-8?q?port=20in=20response=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- response.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1fc5b820..f0bb29e7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.15 +1.3.0 diff --git a/response.py b/response.py index 7652944d..379741af 100644 --- a/response.py +++ b/response.py @@ -353,6 +353,20 @@ async def fetch_response(client, url, headers, payload, engine, model): timestamp = int(datetime.timestamp(datetime.now())) yield await generate_no_stream_response(timestamp, model, content=content, tools_id=None, function_call_name=None, function_call_content=None, role=role, total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=candidates_tokens) + elif engine == "claude": + response_json = response.json() + + content = safe_get(response_json, "content", 0, "text") + + prompt_tokens = safe_get(response_json, "usage", "input_tokens") + output_tokens = safe_get(response_json, "usage", "output_tokens") + total_tokens = prompt_tokens + output_tokens + + role = safe_get(response_json, "role") + + timestamp = int(datetime.timestamp(datetime.now())) + yield await generate_no_stream_response(timestamp, model, content=content, tools_id=None, function_call_name=None, function_call_content=None, role=role, total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=output_tokens) + elif engine == "azure": response_json = response.json() # 删除 content_filter_results From 5b8cea5bfb9a34f12047a82d8eb6505a040b509b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 7 Feb 2025 11:50:56 +0000 Subject: [PATCH 476/476] =?UTF-8?q?=F0=9F=93=96=20Bump=20version=20to=201.?= =?UTF-8?q?3.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f0bb29e7..3a3cd8cc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1