diff --git a/.dockerignore b/.dockerignore index b5f14095..23478050 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ api.yaml test -json_str \ No newline at end of file +json_str +*.jpg +*.json +*.png +*.db \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30aac54d..b2df4b11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,7 @@ on: branches: - main paths: + - VERSION - main.py - utils.py - models.py @@ -15,6 +16,7 @@ on: - requirements.txt - docker-compose.yml - .github/workflows/main.yml + workflow_dispatch: jobs: build-and-push: @@ -36,6 +38,40 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.PACK_TOKEN }} + + - name: Get current version + id: get_version + run: | + VERSION=$(cat VERSION || echo "0.0.0") + echo "Current version: $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Bump version + id: bump_version + run: | + IFS='.' read -ra VERSION_PARTS <<< "${{ steps.get_version.outputs.version }}" + PATCH=$((VERSION_PARTS[2] + 1)) + NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH" + echo $NEW_VERSION > VERSION + echo "New version: $NEW_VERSION" + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + + - name: Commit version bump + env: + GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git add VERSION + git commit -m "📖 Bump version to ${{ steps.bump_version.outputs.new_version }}" + git push + - name: Build and push Docker image uses: docker/build-push-action@v2.7.0 with: @@ -43,4 +79,8 @@ jobs: file: Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: yym68686/uni-api:latest \ No newline at end of file + tags: | + yym68686/uni-api:latest + yym68686/uni-api:${{ steps.bump_version.outputs.new_version }} + ghcr.io/${{ github.repository }}:latest + ghcr.io/${{ github.repository }}:${{ steps.bump_version.outputs.new_version }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..8f005ca2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,69 @@ +name: Build and Release PEX + +on: + push: + tags: + - 'v*' + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + platform: linux + arch: x86_64 + - os: macos-latest + platform: macos + arch: arm64 + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10.12' + + - name: Install pex + run: pip install pex + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Get Version + id: get_version + run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV + + - name: Build Linux PEX + if: matrix.platform == 'linux' + run: | + pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex + + - name: Build MacOS PEX + if: matrix.platform == 'macos' + run: | + pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex + + - name: Create Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + name: Release ${{ env.VERSION }} + draft: false + prerelease: false + files: | + uni-api-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.VERSION }}.pex diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 00000000..ff6aa8d7 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,37 @@ +name: Sync Fork + +on: + schedule: + - cron: '0 0 * * *' # 每天凌晨执行 + # - cron: '0 */12 * * *' # 每12小时执行一次 + workflow_dispatch: # 支持手动触发 + +jobs: + sync: + runs-on: ubuntu-latest + if: github.repository != 'yym68686/uni-api' + + steps: + - name: Checkout target repo + uses: actions/checkout@v4.2.1 + with: + fetch-depth: 0 # 获取所有历史记录，以确保正确同步 + token: ${{ secrets.PAT }} # 使用PAT替代GITHUB_TOKEN + + - name: Sync Fork + uses: aormsby/Fork-Sync-With-Upstream-action@v3.4.1 + with: + target_repo_token: ${{ secrets.PAT }} + upstream_sync_repo: yym68686/uni-api + upstream_sync_branch: main + target_sync_branch: main + upstream_pull_args: --allow-unrelated-histories --no-edit --strategy-option theirs + test_mode: false + + - name: Check for new commits + if: steps.sync.outputs.has_new_commits == 'true' + run: echo "新的提交已同步。" + + - name: No new commits + if: steps.sync.outputs.has_new_commits == 'false' + run: echo "没有新的提交需要同步。" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 83edc3dd..5adf2c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,16 @@ api.json -api.yaml +*.yaml .env __pycache__ .vscode node_modules .wrangler -.pytest_cache \ No newline at end of file +.pytest_cache +*.jpg +*.json +!vercel.json +# *.png +*.db +.aider* +.idea +docker-compose-test.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 95fc0423..9c0c76c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,5 @@ EXPOSE 8000 WORKDIR /home COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages COPY . /home -ENTRYPOINT ["python", "-u", "/home/main.py"] \ No newline at end of file +ENV WATCHFILES_FORCE_POLLING=true +ENTRYPOINT ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--reload", "--reload-include", "*.yaml"] \ No newline at end of file diff --git a/README.md b/README.md index e419fcd6..af94e69c 100644 --- a/README.md +++ b/README.md @@ -9,94 +9,291 @@

+[English](./README.md) | [Chinese](./README_CN.md) ## Introduction -这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 +For personal use, one/new-api is too complex with many commercial features that individuals don't need. If you don't want a complicated frontend interface and prefer support for more models, you can try uni-api. This is a project that unifies the management of large language model APIs, allowing you to call multiple backend services through a single unified API interface, converting them all to OpenAI format, and supporting load balancing. Currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Azure, xai, Cohere, Groq, Cloudflare, OpenRouter, and more. -## Features +## ✨ Features -- 统一管理多个后端服务 -- 支持负载均衡 -- 支持 OpenAI, Anthropic, Gemini, Vertex 函数调用 -- 支持多个模型 -- 支持多个 API Key -- 支持 Vertex 区域负载均衡，支持 Vertex 高并发 +- No front-end, pure configuration file to configure API channels. You can run your own API station just by writing a file, and the documentation has a detailed configuration guide, beginner-friendly. +- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. +- Simultaneously supports Anthropic, Gemini, Vertex AI, Azure, xai, Cohere, Groq, Cloudflare. Vertex simultaneously supports Claude and Gemini API. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure, xai native tool use function calls. +- Support OpenAI, Anthropic, Gemini, Vertex, Azure, xai native image recognition API. +- Support four types of load balancing. + 1. Supports channel-level weighted load balancing, allowing requests to be distributed according to different channel weights. It is not enabled by default and requires configuring channel weights. + 2. Support Vertex regional load balancing and high concurrency, which can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. It is not enabled by default and requires configuring `SCHEDULING_ALGORITHM` as `round_robin`. + 4. Support automatic API key-level round-robin load balancing for multiple API Keys in a single channel. +- Support automatic retry, when an API channel response fails, automatically retry the next API channel. +- Support channel cooling: When an API channel response fails, the channel will automatically be excluded and cooled for a period of time, and requests to the channel will be stopped. After the cooling period ends, the model will automatically be restored until it fails again, at which point it will be cooled again. +- Support fine-grained model timeout settings, allowing different timeout durations for each model. +- Support fine-grained permission control. Support using wildcards to set specific models available for API key channels. +- Support rate limiting, you can set the maximum number of requests per minute as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. +- Supports multiple standard OpenAI format interfaces: `/v1/chat/completions`, `/v1/images/generations`, `/v1/audio/transcriptions`, `/v1/moderations`, `/v1/models`. +- Support OpenAI moderation moral review, which can conduct moral reviews of user messages. If inappropriate messages are found, an error message will be returned. This reduces the risk of the backend API being banned by providers. -## Configuration +## Usage method -使用api.yaml配置文件，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是 api.yaml 配置文件的示例： +To start uni-api, a configuration file must be used. There are two ways to start with a configuration file: + +1. The first method is to use the `CONFIG_URL` environment variable to fill in the configuration file URL, which will be automatically downloaded when uni-api starts. +2. The second method is to mount a configuration file named `api.yaml` into the container. + +### Method 1: Mount the `api.yaml` configuration file to start uni-api + +You must fill in the configuration file in advance to start `uni-api`, and you must use a configuration file named `api.yaml` to start `uni-api`, you can configure multiple models, each model can configure multiple backend services, and support load balancing. Below is an example of the minimum `api.yaml` configuration file that can be run: ```yaml providers: - - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter、deepbricks，随便取名字，必填 - base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 - api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 - model: # 至少填一个模型 - - gpt-4o # 可以使用的模型名称，必填 - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, can be any name, required + base_url: https://api.your.com/v1/chat/completions # Backend service API address, required + api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required, automatically uses base_url and api to get all available models through the /v1/models endpoint. + # Multiple providers can be configured here, each provider can configure multiple API Keys, and each provider can configure multiple models. +api_keys: + - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key, user request uni-api requires API key, required + # This API Key can use all models, that is, it can use all models in all channels set under providers, without needing to add available channels one by one. +``` + +Detailed advanced configuration of `api.yaml`: + +```yaml +providers: + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, can be any name, required + base_url: https://api.your.com/v1/chat/completions # Backend service API address, required + api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required + model: # Optional, if model is not configured, all available models will be automatically obtained through base_url and api via the /v1/models endpoint. + - gpt-4o # Usable model name, required + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional + - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: sk-ant-api03-bNnAOJyA-xQw_twAA + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + - sk-ant-api03-bNnAOJyA-xQw_twAA + - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 - tools: true # 是否支持工具，如生成代码、生成文档等，默认是 true，选填 + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simple name to replace the original complex name, optional + tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini - base_url: https://generativelanguage.googleapis.com/v1beta # base_url 支持 v1beta/v1, 仅供 Gemini 模型使用，必填 - api: AIzaSyAN2k6IRdgw + base_url: https://generativelanguage.googleapis.com/v1beta # base_url supports v1beta/v1, only for Gemini model use, required + api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + - AIzaSyAN2k6IRdgw123 + - AIzaSyAN2k6IRdgw456 + - AIzaSyAN2k6IRdgw789 model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # 重命名后，原来的模型名字 gemini-1.5-flash-exp-0827 无法使用，如果要使用原来的名字，可以在 model 中添加原来的名字，只要加上下面一行就可以使用原来的名字了 - - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the line below to use the original name + - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true + preferences: + api_key_rate_limit: 15/min # Each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day + # api_key_rate_limit: # You can set different frequency limits for each model + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day + # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default + api_key_cooldown_period: 60 # Each API Key will be cooled down for 60 seconds after encountering a 429 error. Optional, the default is 0 seconds. When set to 0, the cooling mechanism is not enabled. When there are multiple API keys, the cooling mechanism will take effect. + api_key_schedule_algorithm: round_robin # Set the request order of multiple API Keys, optional. The default is round_robin, and the optional values are: round_robin, random, fixed_priority. It will take effect when there are multiple API keys. round_robin is polling load balancing, and random is random load balancing. fixed_priority is fixed priority scheduling, always use the first available API key. + model_timeout: # Model timeout, in seconds, default 100 seconds, optional + gemini-1.5-pro: 10 # Model gemini-1.5-pro timeout is 10 seconds + gemini-1.5-flash: 10 # Model gemini-1.5-flash timeout is 10 seconds + default: 10 # Model does not have a timeout set, use the default timeout of 10 seconds, when requesting a model not in model_timeout, the timeout is also 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, the default timeout is 100 seconds + proxy: socks5://[username]:[password]@[ip]:[port] # Proxy address, optional. Supports socks5 and http proxies, default is not used. - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个JSON格式的字符串，包含服务账号的私钥信息。获取方式： 在Google Cloud Console中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在Google Cloud Console的"IAM与管理"部分查看服务账号详情获得。 + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key for Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating a service account, or you can view the service account details in the "IAM and Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash + - gemini-1.5-pro: gemini-1.5-pro-search # Only supports using the gemini-1.5-pro-search model to request uni-api when using the Vertex Gemini API, to automatically use the Google official search tool. - claude-3-5-sonnet@20240620: claude-3-5-sonnet - claude-3-opus@20240229: claude-3-opus - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes, otherwise yaml syntax error, llama-3.1-8b is the renamed name, you can use a simple name to replace the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes, otherwise yaml syntax error + + - provider: azure + base_url: https://your-endpoint.openai.azure.com + api: your-api-key + model: + - gpt-4o - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA model: - causallm-35b-beta2ep-q6k: causallm-35b + - anthropic/claude-3-5-sonnet tools: false - engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 + engine: openrouter # Force the use of a specific message format, currently supports gpt, claude, gemini, openrouter native format, optional api_keys: - - api: sk-KjjI60Yf0JFWtfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 - model: # 该 API Key 可以使用的模型，必填 - - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 - - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 - - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 - role: admin - - - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key, required for users to use this service + model: # Models that can be used by this API Key, required. Default channel-level polling load balancing is enabled, and each request model is requested in sequence according to the model configuration. It is not related to the original channel order in providers. Therefore, you can set different request sequences for each API key. + - gpt-4o # Usable model name, can use all gpt-4o models provided by providers + - claude-3-5-sonnet # Usable model name, can use all claude-3-5-sonnet models provided by providers + - gemini/* # Usable model name, can only use all models provided by providers named gemini, where gemini is the provider name, * represents all models + role: admin # Set the alias of the API key, optional. The request log will display the alias of the API key. If role is admin, only this API key can request the v1/stats,/v1/generate-api-key endpoints. If all API keys do not have role set to admin, the first API key is set as admin and has permission to request the v1/stats,/v1/generate-api-key endpoints. + + - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。 + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. This syntax will not match the model named anthropic/claude-3-5-sonnet provided by other-provider. + - # By adding angle brackets on both sides of the model name, it will not search for the claude-3-5-sonnet model under the channel named anthropic, but will take the entire anthropic/claude-3-5-sonnet as the model name. This syntax can match the model named anthropic/claude-3-5-sonnet provided by other-provider. But it will not match the claude-3-5-sonnet model under anthropic. + - openai-test/text-moderation-latest # When message moderation is enabled, the text-moderation-latest model under the channel named openai-test can be used for moderation. + - sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo/* # Support using other API keys as channels preferences: - USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true - AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true + SCHEDULING_ALGORITHM: fixed_priority # When SCHEDULING_ALGORITHM is fixed_priority, use fixed priority scheduling, always execute the channel of the first model with a request. Default is enabled, SCHEDULING_ALGORITHM default value is fixed_priority. SCHEDULING_ALGORITHM optional values are: fixed_priority, round_robin, weighted_round_robin, lottery, random. + # When SCHEDULING_ALGORITHM is random, use random polling load balancing, randomly request the channel of the model with a request. + # When SCHEDULING_ALGORITHM is round_robin, use polling load balancing, request the channel of the model used by the user in order. + AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true for automatic retry, false for no automatic retry, default is true. Also supports setting a number, indicating the number of retries. + rate_limit: 15/min # Supports rate limiting, each API Key can request up to 15 times per minute, optional. The default is 999999/min. Supports multiple frequency constraints: 15/min,10/day + # rate_limit: # You can set different frequency limits for each model + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day + # default: 4/min # If the model does not set the frequency limit, use the frequency limit of default + ENABLE_MODERATION: true # Whether to enable message moderation, true for enable, false for disable, default is false, when enabled, it will moderate the user's message, if inappropriate messages are found, an error message will be returned. + + # Channel-level weighted load balancing configuration example + - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + model: + - gcp1/*: 5 # The number after the colon is the weight, weight only supports positive integers. + - gcp2/*: 3 # The size of the number represents the weight, the larger the number, the greater the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 10 requests will have 5 requests for the gcp1/* model, 2 requests for the gcp2/* model, and 3 requests for the gcp3/* model. + + preferences: + SCHEDULING_ALGORITHM: weighted_round_robin # Only when SCHEDULING_ALGORITHM is weighted_round_robin and the above channel has weights, it will request according to the weighted order. Use weighted polling load balancing, request the channel of the model with a request according to the weight order. When SCHEDULING_ALGORITHM is lottery, use lottery polling load balancing, request the channel of the model with a request according to the weight randomly. Channels without weights automatically fall back to round_robin polling load balancing. + AUTO_RETRY: true + +preferences: # Global configuration + model_timeout: # Model timeout, in seconds, default 100 seconds, optional + gpt-4o: 10 # Model gpt-4o timeout is 10 seconds, gpt-4o is the model name, when requesting models like gpt-4o-2024-08-06, the timeout is also 10 seconds + claude-3-5-sonnet: 10 # Model claude-3-5-sonnet timeout is 10 seconds, when requesting models like claude-3-5-sonnet-20240620, the timeout is also 10 seconds + default: 10 # Model does not have a timeout set, use the default timeout of 10 seconds, when requesting a model not in model_timeout, the default timeout is 10 seconds, if default is not set, uni-api will use the default timeout set by the environment variable TIMEOUT, the default timeout is 100 seconds + o1-mini: 30 # Model o1-mini timeout is 30 seconds, when requesting models starting with o1-mini, the timeout is 30 seconds + o1-preview: 100 # Model o1-preview timeout is 100 seconds, when requesting models starting with o1-preview, the timeout is 100 seconds + cooldown_period: 300 # Channel cooldown time, in seconds, default 300 seconds, optional. When a model request fails, the channel will be automatically excluded and cooled down for a period of time, and will not request the channel again. After the cooldown time ends, the model will be automatically restored until the request fails again, and it will be cooled down again. When cooldown_period is set to 0, the cooling mechanism is not enabled. + error_triggers: # Error triggers, when the message returned by the model contains any of the strings in the error_triggers, the channel will return an error. Optional + - The bot's usage is covered by the developer + - process this request due to overload or policy +``` + +Mount the configuration file and start the uni-api docker container: + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +``` + +### Method two: Start uni-api using the `CONFIG_URL` environment variable + +After writing the configuration file according to method one, upload it to the cloud disk, get the file's direct link, and then use the `CONFIG_URL` environment variable to start the uni-api docker container: + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-e CONFIG_URL=http://file_url/api.yaml \ +yym68686/uni-api:latest +``` + +## Environment variable + +- CONFIG_URL: The download address of the configuration file, which can be a local file or a remote file, optional +- TIMEOUT: Request timeout, default is 100 seconds. The timeout can control the time needed to switch to the next channel when one channel does not respond. Optional +- DISABLE_DATABASE: Whether to disable the database, default is false, optional + +## Vercel remote deployment + +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) + +After clicking the one-click deploy button above, set the environment variable `CONFIG_URL` to the direct link of the configuration file, `DISABLE_DATABASE` to true, and then click Create to create the project. After deployment, you need to manually set the Function Max Duration to 60 seconds in the Vercel project panel under Settings -> Functions, and then click the Deployments menu and click Redeploy to redeploy, which will set the timeout to 60 seconds. If you do not redeploy, the default timeout will remain at the original 10 seconds. Note that you should not delete the Vercel project and recreate it; instead, click redeploy in the Deployments menu within the currently deployed Vercel project to make the Function Max Duration modification take effect. + +## Ubuntu deployment + +In the warehouse Releases, find the latest version of the corresponding binary file, for example, a file named uni-api-linux-x86_64-0.0.99.pex. Download the binary file on the server and run it: + +```bash +wget https://github.com/yym68686/uni-api/releases/download/v0.0.99/uni-api-linux-x86_64-0.0.99.pex +chmod +x uni-api-linux-x86_64-0.0.99.pex +./uni-api-linux-x86_64-0.0.99.pex +``` + +## Serv00 Remote Deployment (FreeBSD 14.0) + +First, log in to the panel, in Additional services click on the tab Run your own applications to enable the option to run your own programs, then go to the panel Port reservation to randomly open a port. + +If you don't have your own domain name, go to the panel WWW websites and delete the default domain name provided. Then create a new domain with the Domain being the one you just deleted. After clicking Advanced settings, set the Website type to Proxy domain, and the Proxy port should point to the port you just opened. Do not select Use HTTPS. + +ssh login to the serv00 server, execute the following command: + +```bash +git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git +cd uni-api +python -m venv uni-api +tmux new -A -s uni-api +source uni-api/bin/activate +export CFLAGS="-I/usr/local/include" +export CXXFLAGS="-I/usr/local/include" +export CC=gcc +export CXX=g++ +export MAX_CONCURRENCY=1 +export CPUCOUNT=1 +export MAKEFLAGS="-j1" +CMAKE_BUILD_PARALLEL_LEVEL=1 cpuset -l 0 pip install -vv -r requirements.txt +cpuset -l 0 pip install -r -vv requirements.txt +``` + +ctrl+b d to exit tmux, wait a few hours for the installation to complete, and after the installation is complete, execute the following command: + +```bash +tmux new -A -s uni-api +source uni-api/bin/activate +export CONFIG_URL=http://file_url/api.yaml +export DISABLE_DATABASE=true +# Modify the port, xxx is the port, modify it yourself, corresponding to the port opened in the panel Port reservation +sed -i '' 's/port=8000/port=xxx/' main.py +sed -i '' 's/reload=True/reload=False/' main.py +python main.py +``` + +Use ctrl+b d to exit tmux, allowing the program to run in the background. At this point, you can use uni-api in other chat clients. curl test script: + +```bash +curl -X POST https://xxx.serv00.net/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-xxx' \ +-d '{"model": "gpt-4o","messages": [{"role": "user","content": "Hello"}]}' ``` -## Docker Local Deployment +Reference document: + +https://docs.serv00.com/Python/ + +https://linux.do/t/topic/201181 + +https://linux.do/t/topic/218738 + +## Docker local deployment Start the container ```bash docker run --user root -p 8001:8000 --name uni-api -dit \ --v ./api.yaml:/home/api.yaml \ +-e CONFIG_URL=http://file_url/api.yaml \ # If the local configuration file has already been mounted, there is no need to set CONFIG_URL +-v ./api.yaml:/home/api.yaml \ # If CONFIG_URL is already set, there is no need to mount the configuration file +-v ./uniapi_db:/home/data \ # If you do not want to save statistical data, there is no need to mount this folder yym68686/uni-api:latest ``` @@ -108,13 +305,16 @@ services: container_name: uni-api image: yym68686/uni-api:latest environment: - - CONFIG_URL=http://file_url/api.yaml + - CONFIG_URL=http://file_url/api.yaml # If a local configuration file is already mounted, there is no need to set CONFIG_URL ports: - 8001:8000 volumes: - - ./api.yaml:/home/api.yaml + - ./api.yaml:/home/api.yaml # If CONFIG_URL is already set, there is no need to mount the configuration file + - ./uniapi_db:/home/data # If you do not want to save statistical data, there is no need to mount this folder ``` +CONFIG_URL is the URL of the remote configuration file that can be automatically downloaded. For example, if you are not comfortable modifying the configuration file on a certain platform, you can upload the configuration file to a hosting service and provide a direct link to uni-api to download, which is the CONFIG_URL. If you are using a local mounted configuration file, there is no need to set CONFIG_URL. CONFIG_URL is used when it is not convenient to mount the configuration file. + Run Docker Compose container in the background ```bash @@ -139,6 +339,7 @@ docker rm -f uni-api docker run --user root -p 8001:8000 -dit --name uni-api \ -e CONFIG_URL=http://file_url/api.yaml \ -v ./api.yaml:/home/api.yaml \ +-v ./uniapi_db:/home/data \ yym68686/uni-api:latest docker logs -f uni-api ``` @@ -152,7 +353,123 @@ curl -X POST http://127.0.0.1:8000/v1/chat/completions \ -d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' ``` -## Star History +pex linux packaging: + +```bash +VERSION=$(cat VERSION) +pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex +``` + +macOS packaging: + +```bash +VERSION=$(cat VERSION) +pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex +``` + +## Sponsors + +We thank the following sponsors for their support: + +- @PowerHunter: ¥2000 +- @ioi：¥50 + +## How to sponsor us + +If you would like to support our project, you can sponsor us in the following ways: + +1. [PayPal](https://www.paypal.me/yym68686) + +2. [USDT-TRC20](https://pb.yym68686.top/~USDT-TRC20), USDT-TRC20 wallet address: `TLFbqSv5pDu5he43mVmK1dNx7yBMFeN7d8` + +3. [WeChat](https://pb.yym68686.top/~wechat) + +4. [Alipay](https://pb.yym68686.top/~alipay) + +Thank you for your support! + +## FAQ + +- Why does the error `Error processing request or performing moral check: 404: No matching model found` always appear? + +Setting ENABLE_MODERATION to false will fix this issue. When ENABLE_MODERATION is true, the API must be able to use the text-moderation-latest model, and if you have not provided text-moderation-latest in the provider model settings, an error will occur indicating that the model cannot be found. + +- How to prioritize requests for a specific channel, how to set the priority of a channel? + +Directly set the channel order in the api_keys. No other settings are required. Sample configuration file: + +```yaml +providers: + - provider: ai1 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + + - provider: ai2 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + +api_keys: + - api: sk-1234 + model: + - ai2/* + - ai1/* +``` + +In this way, request ai2 first, and if it fails, request ai1. + +- What is the behavior behind various scheduling algorithms? For example, fixed_priority, weighted_round_robin, lottery, random, round_robin? + +All scheduling algorithms need to be enabled by setting api_keys.(api).preferences.SCHEDULING_ALGORITHM in the configuration file to any of the values: fixed_priority, weighted_round_robin, lottery, random, round_robin. + +1. fixed_priority: Fixed priority scheduling. All requests are always executed by the channel of the model that first has a user request. In case of an error, it will switch to the next channel. This is the default scheduling algorithm. + +2. weighted_round_robin: Weighted round-robin load balancing, requests channels with the user's requested model according to the weight order set in the configuration file api_keys.(api).model. + +3. lottery: Draw round-robin load balancing, randomly request the channel of the model with user requests according to the weight set in the configuration file api_keys.(api).model. + +4. round_robin: Round-robin load balancing, requests the channel that owns the model requested by the user according to the configuration order in the configuration file api_keys.(api).model. You can check the previous question on how to set the priority of channels. + +- How should the base_url be filled in correctly? + +Except for some special channels shown in the advanced configuration, all OpenAI format providers need to fill in the base_url completely, which means the base_url must end with /v1/chat/completions. If you are using GitHub models, the base_url should be filled in as https://models.inference.ai.azure.com/chat/completions, not Azure's URL. + +- How does the model timeout time work? What is the priority of the channel-level timeout setting and the global model timeout setting? + +The channel-level timeout setting has higher priority than the global model timeout setting. The priority order is: channel-level model timeout setting > channel-level default timeout setting > global model timeout setting > global default timeout setting > environment variable TIMEOUT. + +By adjusting the model timeout time, you can avoid the error of some channels timing out. If you encounter the error `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}`, please try to increase the model timeout time. + +- How does api_key_rate_limit work? How do I set the same rate limit for multiple models? + +If you want to set the same frequency limit for the four models gemini-1.5-pro-latest, gemini-1.5-pro, gemini-1.5-pro-001, gemini-1.5-pro-002 simultaneously, you can set it like this: + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min +``` + +This will match all models containing the gemini-1.5-pro string. The frequency limit for these four models, gemini-1.5-pro-latest, gemini-1.5-pro, gemini-1.5-pro-001, gemini-1.5-pro-002, will all be set to 1000/min. The logic for configuring the api_key_rate_limit field is as follows, here is a sample configuration file: + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min + gemini-1.5-pro-002: 500/min +``` + +At this time, if there is a request using the model gemini-1.5-pro-002. + +First, the uni-api will attempt to precisely match the model in the api_key_rate_limit. If the rate limit for gemini-1.5-pro-002 is set, then the rate limit for gemini-1.5-pro-002 is 500/min. If the requested model at this time is not gemini-1.5-pro-002, but gemini-1.5-pro-latest, since the api_key_rate_limit does not have a rate limit set for gemini-1.5-pro-latest, it will look for any model with the same prefix as gemini-1.5-pro-latest that has been set, thus the rate limit for gemini-1.5-pro-latest will be set to 1000/min. + +## ⭐ Star History Star History Chart diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 00000000..297575b2 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,476 @@ +# uni-api + +

+ + + + + docker pull + +

+ +[英文](./README.md) | [中文](./README_CN.md) + +## 介绍 + +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，又想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型 API 的项目，可以通过一个统一的API 接口调用多种不同提供商的服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、Azure、xai、Cohere、Groq、Cloudflare、OpenRouter 等。 + +## ✨ 特性 + +- 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 同时支持 Anthropic、Gemini、Vertex AI、Azure、xai、Cohere、Groq、Cloudflare。Vertex 同时支持 Claude 和 Gemini API。 +- 支持 OpenAI、 Anthropic、Gemini、Vertex、Azure、xai 原生 tool use 函数调用。 +- 支持 OpenAI、Anthropic、Gemini、Vertex、Azure、xai 原生识图 API。 +- 支持四种负载均衡。 + 1. 支持渠道级加权负载均衡，可以根据不同的渠道权重分配请求。默认不开启，需要配置渠道权重。 + 2. 支持 Vertex 区域级负载均衡，支持 Vertex 高并发，最高可将 Gemini，Claude 并发提高 （API数量 * 区域数量） 倍。自动开启不需要额外配置。 + 3. 除了 Vertex 区域级负载均衡，所有 API 均支持渠道级顺序负载均衡，提高沉浸式翻译体验。默认不开启，需要配置 `SCHEDULING_ALGORITHM` 为 `round_robin`。 + 4. 支持单个渠道多个 API Key 自动开启 API key 级别的轮训负载均衡。 +- 支持自动重试，当一个 API 渠道响应失败时，自动重试下一个 API 渠道。 +- 支持渠道冷却，当一个 API 渠道响应失败时，会自动将该渠道排除冷却一段时间，不再请求该渠道，冷却时间结束后，会自动将该模型恢复，直到再次请求失败，会重新冷却。 +- 支持细粒度的模型超时时间设置，可以为每个模型设置不同的超时时间。 +- 支持细粒度的权限控制。支持使用通配符设置 API key 可用渠道的特定模型。 +- 支持限流，可以设置每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认60/min。 +- 支持多个标准 OpenAI 格式的接口：`/v1/chat/completions`，`/v1/images/generations`，`/v1/audio/transcriptions`，`/v1/moderations`，`/v1/models`。 +- 支持 OpenAI moderation 道德审查，可以对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。降低后台 API 被提供商封禁的风险。 + +## 使用方法 + +启动 uni-api 必须使用配置文件，有两种方式可以启动配置文件： + +1. 第一种是使用 `CONFIG_URL` 环境变量填写配置文件 URL，uni-api启动时会自动下载。 +2. 第二种就是挂载名为 `api.yaml` 的配置文件到容器内。 + +### 方法一：挂载 `api.yaml` 配置文件启动 uni-api + +必须事先填写完成配置文件才能启动 `uni-api`，必须使用名为 `api.yaml` 的配置文件才能启动 `uni-api`，可以配置多个模型，每个模型可以配置多个后端服务，支持负载均衡。下面是最小可运行的 `api.yaml` 配置文件的示例： + +```yaml +providers: + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter，随便取名字，必填 + base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 + api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填，自动使用 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 + # 这里可以配置多个提供商，每个提供商可以配置多个 API Key，每个提供商可以配置多个模型。 +api_keys: + - api: sk-Pkj60Yf8JFWxfgRmXQFWyGtWUddGZnmi3KlvowmRWpWpQxx # API Key，用户请求 uni-api 需要 API key，必填 + # 该 API Key 可以使用所有模型，即可以使用 providers 下面设置的所有渠道里面的所有模型，不需要一个个添加可用渠道。 +``` + +`api.yaml` 详细的高级配置： + +```yaml +providers: + - provider: provider_name # 服务提供商名称, 如 openai、anthropic、gemini、openrouter，随便取名字，必填 + base_url: https://api.your.com/v1/chat/completions # 后端服务的API地址，必填 + api: sk-YgS6GTi0b4bEabc4C # 提供商的API Key，必填 + model: # 选填，如果不配置 model，会自动通过 base_url 和 api 通过 /v1/models 端点获取可用的所有模型。 + - gpt-4o # 可以使用的模型名称，必填 + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - dall-e-3 + + - provider: anthropic + base_url: https://api.anthropic.com/v1/messages + api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + - sk-ant-api03-bNnAOJyA-xQw_twAA + - sk-ant-api02-bNnxxxx + model: + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # 重命名模型，claude-3-5-sonnet-20240620 是服务商的模型名称，claude-3-5-sonnet 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + tools: true # 是否支持工具，如生成代码、生成文档等，默认是 true，选填 + + - provider: gemini + base_url: https://generativelanguage.googleapis.com/v1beta # base_url 支持 v1beta/v1, 仅供 Gemini 模型使用，必填 + api: # 支持多个 API Key，多个 key 自动开启轮训负载均衡，至少一个 key，必填 + - AIzaSyAN2k6IRdgw123 + - AIzaSyAN2k6IRdgw456 + - AIzaSyAN2k6IRdgw789 + model: + - gemini-1.5-pro + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # 重命名后，原来的模型名字 gemini-1.5-flash-exp-0827 无法使用，如果要使用原来的名字，可以在 model 中添加原来的名字，只要加上下面一行就可以使用原来的名字了 + - gemini-1.5-flash-exp-0827 # 加上这一行，gemini-1.5-flash-exp-0827 和 gemini-1.5-flash 都可以被请求 + tools: true + preferences: + api_key_rate_limit: 15/min # 每个 API Key 每分钟最多请求次数，选填。默认为 999999/min。支持多个频率约束条件：15/min,10/day + # api_key_rate_limit: # 可以为每个模型设置不同的频率限制 + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day + # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 + api_key_cooldown_period: 60 # 每个 API Key 遭遇 429 错误后的冷却时间，单位为秒，选填。默认为 0 秒, 当设置为 0 秒时，不启用冷却机制。当存在多个 API key 时才会生效。 + api_key_schedule_algorithm: round_robin # 设置多个 API Key 的请求顺序，选填。默认为 round_robin，可选值有：round_robin，random，fixed_priority。当存在多个 API key 时才会生效。round_robin 是轮询负载均衡，random 是随机负载均衡，fixed_priority 是固定优先级调度，永远使用第一个可用的 API key。 + model_timeout: # 模型超时时间，单位为秒，默认 100 秒，选填 + gemini-1.5-pro: 10 # 模型 gemini-1.5-pro 的超时时间为 10 秒 + gemini-1.5-flash: 10 # 模型 gemini-1.5-flash 的超时时间为 10 秒 + default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用全局配置的模型超时时间。 + proxy: socks5://[用户名]:[密码]@[IP地址]:[端口] # 代理地址，选填。支持 socks5 和 http 代理，默认不使用代理。 + + - provider: vertex + project_id: gen-lang-client-xxxxxxxxxxxxxx # 描述： 您的Google Cloud项目ID。格式： 字符串，通常由小写字母、数字和连字符组成。获取方式： 在Google Cloud Console的项目选择器中可以找到您的项目ID。 + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # 描述： Google Cloud Vertex AI服务账号的私钥。格式： 一个 JSON 格式的字符串，包含服务账号的私钥信息。获取方式： 在 Google Cloud Console 中创建服务账号，生成JSON格式的密钥文件，然后将其内容设置为此环境变量的值。 + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # 描述： Google Cloud Vertex AI 服务账号的电子邮件地址。格式： 通常是形如 "service-account-name@project-id.iam.gserviceaccount.com" 的字符串。获取方式： 在创建服务账号时生成，也可以在 Google Cloud Console 的"IAM与管理"部分查看服务账号详情获得。 + model: + - gemini-1.5-pro + - gemini-1.5-flash + - gemini-1.5-pro: gemini-1.5-pro-search # 仅支持在 vertex Gemini API 中，以 -search 后缀重命名模型后，使用 gemini-1.5-pro-search 模型请求 uni-api 时，表示 gemini-1.5-pro 模型自动使用 Google 官方搜索工具。 + - claude-3-5-sonnet@20240620: claude-3-5-sonnet + - claude-3-opus@20240229: claude-3-opus + - claude-3-sonnet@20240229: claude-3-sonnet + - claude-3-haiku@20240307: claude-3-haiku + tools: true + notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key，必填 + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID，必填 + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # 重命名模型，@cf/meta/llama-3.1-8b-instruct 是服务商的原始的模型名称，必须使用引号包裹模型名，否则yaml语法错误，llama-3.1-8b 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - '@cf/meta/llama-3.1-8b-instruct' # 必须使用引号包裹模型名，否则yaml语法错误 + + - provider: azure + base_url: https://your-endpoint.openai.azure.com + api: your-api-key + model: + - gpt-4o + + - provider: other-provider + base_url: https://api.xxx.com/v1/messages + api: sk-bNnAOJyA-xQw_twAA + model: + - causallm-35b-beta2ep-q6k: causallm-35b + - anthropic/claude-3-5-sonnet + tools: false + engine: openrouter # 强制使用某个消息格式，目前支持 gpt，claude，gemini，openrouter 原生格式，选填 + +api_keys: + - api: sk-KjjI60Yf0JFWxfgRmXqFWyGtWUd9GZnmi3KlvowmRWpWpQRo # API Key，用户使用本服务需要 API key，必填 + model: # 该 API Key 可以使用的模型，必填。默认开启渠道级轮询负载均衡，每次请求模型按照 model 配置的顺序依次请求。与 providers 里面原始的渠道顺序无关。因此你可以设置每个 API key 请求顺序不一样。 + - gpt-4o # 可以使用的模型名称，可以使用所有提供商提供的 gpt-4o 模型 + - claude-3-5-sonnet # 可以使用的模型名称，可以使用所有提供商提供的 claude-3-5-sonnet 模型 + - gemini/* # 可以使用的模型名称，仅可以使用名为 gemini 提供商提供的所有模型，其中 gemini 是 provider 名称，* 代表所有模型 + role: admin # 设置 API key 的别名，选填。请求日志会显示该 API key 的别名。如果 role 为 admin，则仅有此 API key 可以请求 v1/stats,/v1/generate-api-key 端点。如果所有 API key 都没有设置 role 为 admin，则默认第一个 API key 为 admin 拥有请求 v1/stats,/v1/generate-api-key 端点的权限。 + + - api: sk-pkhf60Yf0JGyJxgRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy + model: + - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。这种写法不会匹配到other-provider提供的名为anthropic/claude-3-5-sonnet的模型。 + - # 通过在模型名两侧加上尖括号，这样就不会去名为anthropic的渠道下去寻找claude-3-5-sonnet模型，而是将整个 anthropic/claude-3-5-sonnet 作为模型名称。这种写法可以匹配到other-provider提供的名为 anthropic/claude-3-5-sonnet 的模型。但不会匹配到anthropic下面的claude-3-5-sonnet模型。 + - openai-test/text-moderation-latest # 当开启消息道德审查后，可以使用名为 openai-test 渠道下的 text-moderation-latest 模型进行道德审查。 + - sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo/* # 支持将其他 api key 当作渠道 + preferences: + SCHEDULING_ALGORITHM: fixed_priority # 当 SCHEDULING_ALGORITHM 为 fixed_priority 时，使用固定优先级调度，永远执行第一个拥有请求的模型的渠道。默认开启，SCHEDULING_ALGORITHM 缺省值为 fixed_priority。SCHEDULING_ALGORITHM 可选值有：fixed_priority，round_robin，weighted_round_robin, lottery, random。 + # 当 SCHEDULING_ALGORITHM 为 random 时，使用随机轮训负载均衡，随机请求拥有请求的模型的渠道。 + # 当 SCHEDULING_ALGORITHM 为 round_robin 时，使用轮训负载均衡，按照顺序请求用户使用的模型的渠道。 + AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true。也可以设置为数字，表示重试次数。 + rate_limit: 15/min # 支持限流，每分钟最多请求次数，可以设置为整数，如 2/min，2 次每分钟、5/hour，5 次每小时、10/day，10 次每天，10/month，10 次每月，10/year，10 次每年。默认999999/min，选填。支持多个频率约束条件：15/min,10/day + # rate_limit: # 可以为每个模型设置不同的频率限制 + # gemini-1.5-flash: 15/min,1500/day + # gemini-1.5-pro: 2/min,50/day + # default: 4/min # 如果模型没有设置频率限制，使用 default 的频率限制 + ENABLE_MODERATION: true # 是否开启消息道德审查，true 为开启，false 为不开启，默认为 false，当开启后，会对用户的消息进行道德审查，如果发现不当的消息，会返回错误信息。 + + # 渠道级加权负载均衡配置示例 + - api: sk-KjjI60Yd0JFWtxxxxxxxxxxxxxxwmRWpWpQRo + model: + - gcp1/*: 5 # 冒号后面就是权重，权重仅支持正整数。 + - gcp2/*: 3 # 数字的大小代表权重，数字越大，请求的概率越大。 + - gcp3/*: 2 # 在该示例中，所有渠道加起来一共有 10 个权重，及 10 个请求里面有 5 个请求会请求 gcp1/* 模型，2 个请求会请求 gcp2/* 模型，3 个请求会请求 gcp3/* 模型。 + + preferences: + SCHEDULING_ALGORITHM: weighted_round_robin # 仅当 SCHEDULING_ALGORITHM 为 weighted_round_robin 并且上面的渠道如果有权重，会按照加权后的顺序请求。使用加权轮训负载均衡，按照权重顺序请求拥有请求的模型的渠道。当 SCHEDULING_ALGORITHM 为 lottery 时，使用抽奖轮训负载均衡，按照权重随机请求拥有请求的模型的渠道。没设置权重的渠道自动回退到 round_robin 轮训负载均衡。 + AUTO_RETRY: true + +preferences: # 全局配置 + model_timeout: # 模型超时时间，单位为秒，默认 100 秒，选填 + gpt-4o: 10 # 模型 gpt-4o 的超时时间为 10 秒,gpt-4o 是模型名称，当请求 gpt-4o-2024-08-06 等模型时，超时时间也是 10 秒 + claude-3-5-sonnet: 10 # 模型 claude-3-5-sonnet 的超时时间为 10 秒，当请求 claude-3-5-sonnet-20240620 等模型时，超时时间也是 10 秒 + default: 10 # 模型没有设置超时时间，使用默认的超时时间 10 秒，当请求的不在 model_timeout 里面的模型时，超时时间默认是 10 秒，不设置 default，uni-api 会使用 环境变量 TIMEOUT 设置的默认超时时间，默认超时时间是 100 秒 + o1-mini: 30 # 模型 o1-mini 的超时时间为 30 秒，当请求名字是 o1-mini 开头的模型时，超时时间是 30 秒 + o1-preview: 100 # 模型 o1-preview 的超时时间为 100 秒，当请求名字是 o1-preview 开头的模型时，超时时间是 100 秒 + cooldown_period: 300 # 渠道冷却时间，单位为秒，默认 300 秒，选填。当模型请求失败时，会自动将该渠道排除冷却一段时间，不再请求该渠道，冷却时间结束后，会自动将该模型恢复，直到再次请求失败，会重新冷却。当 cooldown_period 设置为 0 时，不启用冷却机制。 + error_triggers: # 错误触发器，当模型返回的消息包含错误触发器中的任意一个字符串时，该渠道会自动返回报错。选填 + - The bot's usage is covered by the developer + - process this request due to overload or policy +``` + +挂载配置文件并启动 uni-api docker 容器： + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-v ./api.yaml:/home/api.yaml \ +yym68686/uni-api:latest +``` + +### 方法二：使用 `CONFIG_URL` 环境变量启动 uni-api + +按照方法一写完配置文件后，上传到云端硬盘，获取文件的直链，然后使用 `CONFIG_URL` 环境变量启动 uni-api docker 容器： + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-e CONFIG_URL=http://file_url/api.yaml \ +yym68686/uni-api:latest +``` + +## 环境变量 + +- CONFIG_URL: 配置文件的下载地址，可以是本地文件，也可以是远程文件，选填 +- TIMEOUT: 请求超时时间，默认为 100 秒，超时时间可以控制当一个渠道没有响应时，切换下一个渠道需要的时间。选填 +- DISABLE_DATABASE: 是否禁用数据库，默认为 false，选填 + +## Vercel 部署 + +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fyym68686%2Funi-api%2Ftree%2Fmain&env=CONFIG_URL,DISABLE_DATABASE&project-name=uni-api-vercel&repository-name=uni-api-vercel) + +点击上面的一键部署按钮后，设置环境变量 `CONFIG_URL` 为配置文件的直链， `DISABLE_DATABASE` 为 true，然后点击 Create 创建项目。部署完之后需要手动在 vercel 项目面板的 Settings -> Funcitons -> Function Max Duration 设置为 60 秒，然后点击 Deployments 菜单点击 Redeploy 重新部署，即可将超时时间设置为 60 秒，如果不重新部署，默认超时时间将是原来的 10 秒。注意不是删掉 vercel 项目重建，而是在当前部署好的 vercel 项目里面的 Deployments 菜单里面点 redeploy，这样才能让 Function Max Duration 的修改生效。 + +## Ubuntu 部署 + +在仓库 Releases 找到对应的二进制文件最新版本，例如名为 uni-api-linux-x86_64-0.0.99.pex 的文件。在服务器下载二进制文件并运行： + +```bash +wget https://github.com/yym68686/uni-api/releases/download/v0.0.99/uni-api-linux-x86_64-0.0.99.pex +chmod +x uni-api-linux-x86_64-0.0.99.pex +./uni-api-linux-x86_64-0.0.99.pex +``` + +## serv00 远程部署（FreeBSD 14.0） + +首先登录面板，Additional services 里面点击选项卡 Run your own applications 开启允许运行自己的程序，然后到面板 Port reservation 去随便开一个端口。 + +如果没有自己的域名，去面板 WWW websites 删掉默认给的域名，再新建一个域名 Domain 为刚才删掉的域名，点击 Advanced settings 后设置 Website type 为 Proxy 域名，Proxy port 指向你刚才开的端口，不要选中 Use HTTPS。 + +ssh 登陆到 serv00 服务器，执行下面的命令： + +```bash +git clone --depth 1 -b main --quiet https://github.com/yym68686/uni-api.git +cd uni-api +python -m venv uni-api +tmux new -A -s uni-api +source uni-api/bin/activate +export CFLAGS="-I/usr/local/include" +export CXXFLAGS="-I/usr/local/include" +export CC=gcc +export CXX=g++ +export MAX_CONCURRENCY=1 +export CPUCOUNT=1 +export MAKEFLAGS="-j1" +CMAKE_BUILD_PARALLEL_LEVEL=1 cpuset -l 0 pip install -vv -r requirements.txt +cpuset -l 0 pip install -r -vv requirements.txt +``` + +ctrl+b d 退出 tmux 等待几个小时安装完成，安装完成后执行下面的命令： + +```bash +tmux new -A -s uni-api +source uni-api/bin/activate +export CONFIG_URL=http://file_url/api.yaml +export DISABLE_DATABASE=true +# 修改端口，xxx 为端口，自行修改，对应刚刚在面板 Port reservation 开的端口 +sed -i '' 's/port=8000/port=xxx/' main.py +sed -i '' 's/reload=True/reload=False/' main.py +python main.py +``` + +使用 ctrl+b d 退出 tmux，即可让程序后台运行。此时就可以在其他聊天客户端使用 uni-api 了。curl 测试脚本： + +```bash +curl -X POST https://xxx.serv00.net/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-xxx' \ +-d '{"model": "gpt-4o","messages": [{"role": "user","content": "你好"}]}' +``` + +参考文档： + +https://docs.serv00.com/Python/ + +https://linux.do/t/topic/201181 + +https://linux.do/t/topic/218738 + +## Docker 本地部署 + +Start the container + +```bash +docker run --user root -p 8001:8000 --name uni-api -dit \ +-e CONFIG_URL=http://file_url/api.yaml \ # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL +-v ./api.yaml:/home/api.yaml \ # 如果已经设置 CONFIG_URL，不需要挂载配置文件 +-v ./uniapi_db:/home/data \ # 如果不想保存统计数据，不需要挂载该文件夹 +yym68686/uni-api:latest +``` + +Or if you want to use Docker Compose, here is a docker-compose.yml example: + +```yaml +services: + uni-api: + container_name: uni-api + image: yym68686/uni-api:latest + environment: + - CONFIG_URL=http://file_url/api.yaml # 如果已经挂载了本地配置文件，不需要设置 CONFIG_URL + ports: + - 8001:8000 + volumes: + - ./api.yaml:/home/api.yaml # 如果已经设置 CONFIG_URL，不需要挂载配置文件 + - ./uniapi_db:/home/data # 如果不想保存统计数据，不需要挂载该文件夹 +``` + +CONFIG_URL 就是可以自动下载远程的配置文件。比如你在某个平台不方便修改配置文件，可以把配置文件传到某个托管服务，可以提供直链给 uni-api 下载，CONFIG_URL 就是这个直链。如果使用本地挂载的配置文件，不需要设置 CONFIG_URL。CONFIG_URL 是在不方便挂载配置文件的情况下使用。 + +Run Docker Compose container in the background + +```bash +docker-compose pull +docker-compose up -d +``` + +Docker build + +```bash +docker build --no-cache -t uni-api:latest -f Dockerfile --platform linux/amd64 . +docker tag uni-api:latest yym68686/uni-api:latest +docker push yym68686/uni-api:latest +``` + +One-Click Restart Docker Image + +```bash +set -eu +docker pull yym68686/uni-api:latest +docker rm -f uni-api +docker run --user root -p 8001:8000 -dit --name uni-api \ +-e CONFIG_URL=http://file_url/api.yaml \ +-v ./api.yaml:/home/api.yaml \ +-v ./uniapi_db:/home/data \ +yym68686/uni-api:latest +docker logs -f uni-api +``` + +RESTful curl test + +```bash +curl -X POST http://127.0.0.1:8000/v1/chat/completions \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer ${API}" \ +-d '{"model": "gpt-4o","messages": [{"role": "user", "content": "Hello"}],"stream": true}' +``` + +pex linux 打包： + +```bash +VERSION=$(cat VERSION) +pex -D . -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + --platform linux_x86_64-cp-3.10.12-cp310 \ + --interpreter-constraint '==3.10.*' \ + --no-strip-pex-env \ + -o uni-api-linux-x86_64-${VERSION}.pex +``` + +macos 打包： + +```bash +VERSION=$(cat VERSION) +pex -r requirements.txt \ + -c uvicorn \ + --inject-args 'main:app --host 0.0.0.0 --port 8000' \ + -o uni-api-macos-arm64-${VERSION}.pex +``` + +## 赞助商 + +我们感谢以下赞助商的支持： + +- @PowerHunter：¥2000 +- @ioi：¥50 + +## 如何赞助我们 + +如果您想支持我们的项目，您可以通过以下方式赞助我们： + +1. [PayPal](https://www.paypal.me/yym68686) + +2. [USDT-TRC20](https://pb.yym68686.top/~USDT-TRC20)，USDT-TRC20 钱包地址：`TLFbqSv5pDu5he43mVmK1dNx7yBMFeN7d8` + +3. [微信](https://pb.yym68686.top/~wechat) + +4. [支付宝](https://pb.yym68686.top/~alipay) + +感谢您的支持！ + +## 常见问题 + +- 为什么总是出现 `Error processing request or performing moral check: 404: No matching model found` 错误？ + +将 ENABLE_MODERATION 设置为 false 将修复这个问题。当 ENABLE_MODERATION 为 true 时，API 必须能够使用 text-moderation-latest 模型，如果你没有在提供商模型设置里面提供 text-moderation-latest，将会报错找不到模型。 + +- 怎么优先请求某个渠道，怎么设置渠道的优先级？ + +直接在api_keys里面通过设置渠道顺序即可。不需要做其他设置，示例配置文件： + +```yaml +providers: + - provider: ai1 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + + - provider: ai2 + base_url: https://xxx/v1/chat/completions + api: sk-xxx + +api_keys: + - api: sk-1234 + model: + - ai2/* + - ai1/* +``` + +这样设置则先请求 ai2，失败后请求 ai1。 + +- 各种调度算法背后的行为是怎样的？比如 fixed_priority，weighted_round_robin，lottery，random，round_robin？ + +所有调度算法需要通过在配置文件的 api_keys.(api).preferences.SCHEDULING_ALGORITHM 设置为 fixed_priority，weighted_round_robin，lottery，random，round_robin 中的任意值来开启。 + +1. fixed_priority：固定优先级调度。所有请求永远执行第一个拥有用户请求的模型的渠道。报错时，会切换下一个渠道。这是默认的调度算法。 + +2. weighted_round_robin：加权轮训负载均衡，按照配置文件 api_keys.(api).model 设定的权重顺序请求拥有用户请求的模型的渠道。 + +3. lottery：抽奖轮训负载均衡，按照配置文件 api_keys.(api).model 设置的权重随机请求拥有用户请求的模型的渠道。 + +4. round_robin：轮训负载均衡，按照配置文件 api_keys.(api).model 的配置顺序请求拥有用户请求的模型的渠道。可以查看上一个问题，如何设置渠道的优先级。 + +- 应该怎么正确填写 base_url？ + +除了高级配置里面所展示的一些特殊的渠道，所有 OpenAI 格式的提供商需要把 base_url 填完整，也就是说 base_url 必须以 /v1/chat/completions 结尾。如果你使用的 GitHub models，base_url 应该填写为 https://models.inference.ai.azure.com/chat/completions，而不是 Azure 的 URL。 + +- 模型超时时间是如何确认的？渠道级别的超时设置和全局模型超时设置的优先级是什么？ + +渠道级别的超时设置优先级高于全局模型超时设置。优先级顺序：渠道级别模型超时设置 > 渠道级别默认超时设置 > 全局模型超时设置 > 全局默认超时设置 > 环境变量 TIMEOUT。 + +通过调整模型超时时间，可以避免出现某些渠道请求超时报错的情况。如果你遇到 `{'error': '500', 'details': 'fetch_response_stream Read Response Timeout'}` 错误，请尝试增加模型超时时间。 + +- api_key_rate_limit 是怎么工作的？我如何给多个模型设置相同的频率限制？ + +如果你想同时给 gemini-1.5-pro-latest，gemini-1.5-pro，gemini-1.5-pro-001，gemini-1.5-pro-002 这四个模型设置相同的频率限制，可以这样设置： + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min +``` + +这会匹配所有含有 gemini-1.5-pro 字符串的模型。gemini-1.5-pro-latest，gemini-1.5-pro，gemini-1.5-pro-001，gemini-1.5-pro-002 这四个模型频率限制都会设置为 1000/min。api_key_rate_limit 字段配置的逻辑如下，这是一个示例配置文件： + +```yaml +api_key_rate_limit: + gemini-1.5-pro: 1000/min + gemini-1.5-pro-002: 500/min +``` + +此时如果有一个使用模型 gemini-1.5-pro-002 的请求。 + +首先，uni-api 会尝试精确匹配 api_key_rate_limit 的模型。如果刚好设置了 gemini-1.5-pro-002 的频率限制，则 gemini-1.5-pro-002 的频率限制则为 500/min，如果此时请求的模型不是 gemini-1.5-pro-002，而是 gemini-1.5-pro-latest，由于 api_key_rate_limit 没有设置 gemini-1.5-pro-latest 的频率限制，因此会寻找有没有前缀和 gemini-1.5-pro-latest 相同的模型被设置了，因此 gemini-1.5-pro-latest 的频率限制会被设置为 1000/min。 + +## ⭐ Star 历史 + + + Star History Chart + \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..3a3cd8cc --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.3.1 diff --git a/components/provider_table.py b/components/provider_table.py new file mode 100644 index 00000000..5e933c0e --- /dev/null +++ b/components/provider_table.py @@ -0,0 +1,235 @@ +from xue import Div, Table, Thead, Tbody, Tr, Th, Td, Button, Input, Script, Head, Style, Span +from xue.components.checkbox import checkbox +from xue.components.dropdown import dropdown_menu, dropdown_menu_content +from xue.components.button import button +from xue.components.input import input + +Head.add_default_children([ + Style(""" + .data-table-container { + width: 100%; + overflow-x: auto; + border: 1px solid #e2e8f0; + border-radius: 0.5rem; + overflow-x: visible !important; + } + .data-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + } + .data-table th, .data-table td { + padding: 0.75rem 1rem; + text-align: left; + border-bottom: 1px solid #e2e8f0; + } + .data-table th { + font-weight: 500; + font-size: 0.875rem; + color: #4b5563; + height: 2.5rem; + transition: background-color 0.2s; + } + .data-table thead tr:hover th, + .data-table tbody tr:hover { + background-color: #f8fafc; + } + .data-table tbody tr:last-child td { + border-bottom: none; + } + .sortable-header { + cursor: pointer; + user-select: none; + display: inline-flex; + align-items: center; + padding: 0.25rem 0.5rem; + border-radius: 0.25rem; + transition: background-color 0.2s; + } + .sortable-header:hover { + background-color: #e5e7eb; + } + .sort-icon { + display: inline-block; + width: 1rem; + height: 1rem; + margin-left: 0.25rem; + transition: transform 0.2s; + opacity: 0; + } + .sortable-header:hover .sort-icon, + .sort-asc .sort-icon, + .sort-desc .sort-icon { + opacity: 1; + } + .sort-asc .sort-icon { + transform: rotate(180deg); + } + .table-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + } + .table-footer { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + .pagination { + display: flex; + gap: 0.5rem; + } + @media (prefers-color-scheme: dark) { + .data-table-container { + border-color: #4b5563; + } + .data-table th, .data-table td { + border-color: #4b5563; + } + .data-table th { + color: #d1d5db; + } + .data-table thead tr:hover th, + .data-table tbody tr:hover { + background-color: #1f2937; + } + .sortable-header:hover { + background-color: #374151; + } + } + """, id="data-table-style"), + Script(""" + function toggleAllRows(checked) { + const checkboxes = document.querySelectorAll('.row-checkbox'); + checkboxes.forEach(cb => cb.checked = checked); + updateSelectedCount(); + } + + function updateSelectedCount() { + const selectedCount = document.querySelectorAll('.row-checkbox:checked').length; + const totalCount = document.querySelectorAll('.row-checkbox').length; + document.getElementById('selected-count').textContent = `${selectedCount} of ${totalCount} row(s) selected.`; + } + + function sortTable(columnIndex, accessor) { + const table = document.querySelector('.data-table'); + const header = table.querySelector(`th[data-accessor="${accessor}"]`); + const isAscending = !header.classList.contains('sort-asc'); + + // Update sort direction + table.querySelectorAll('th').forEach(th => th.classList.remove('sort-asc', 'sort-desc')); + header.classList.add(isAscending ? 'sort-asc' : 'sort-desc'); + + // Sort the table + const rows = Array.from(table.querySelectorAll('tbody tr')); + rows.sort((a, b) => { + const aValue = a.querySelector(`td[data-accessor="${accessor}"]`).textContent; + const bValue = b.querySelector(`td[data-accessor="${accessor}"]`).textContent; + return isAscending ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue); + }); + + // Update the table + const tbody = table.querySelector('tbody'); + rows.forEach(row => tbody.appendChild(row)); + } + + document.addEventListener('change', function(event) { + if (event.target.classList.contains('row-checkbox')) { + updateSelectedCount(); + } + }); + """, id="data-table-script"), +]) + +def data_table(columns, data, id, with_filter=True, row_ids=None): + if row_ids is None: + row_ids = range(len(data)) + + tbody_content = Tbody( + *[Tr( + Td(checkbox(f"row-{i}", "", class_="row-checkbox")), + *[Td(row[col['value']], data_accessor=col['value']) for col in columns], + Td(row_actions_menu(row_id)), + id=f"row-{row_id}" + ) for i, (row, row_id) in enumerate(zip(data, row_ids))] + ) + + return Div( + Div( + input(type="text", placeholder="Filter...", id=f"{id}-filter", class_="mr-auto"), + Div( + button( + "Add Provider", + variant="secondary", + hx_get="/add-provider-sheet", + hx_target="#sheet-container", + hx_swap="innerHTML", + class_="h-[2.625rem]" + ), + dropdown_menu("Columns"), + ), + class_="table-header flex items-center" + ) if with_filter else None, + Div( + Div( + Table( + Thead( + Tr( + Th(checkbox("select-all", "", onclick="toggleAllRows(this.checked)")), + *[Th( + Div( + col['label'], + Span("▼", class_="sort-icon"), + class_="sortable-header" if col.get('sortable', False) else "", + onclick=f"sortTable({i}, '{col['value']}')" if col.get('sortable', False) else None + ), + data_accessor=col['value'] + ) for i, col in enumerate(columns)], + Th("Actions") # 新增的操作列 + ) + ), + tbody_content, + class_="data-table" + ), + class_="data-table-container" + ), + Div( + Div(id="selected-count", class_="text-sm text-gray-500"), + Div( + button("Previous", variant="outline", class_="mr-2"), + button("Next", variant="outline"), + class_="pagination" + ), + class_="table-footer" + ), + id=id + ), + ) + +def get_column_visibility_menu(id, columns): + return dropdown_menu_content(id, [ + {"label": col['label'], "value": col['value']} + for col in columns if col.get('can_hide', True) + ]) + +def row_actions_menu(row_id): + return dropdown_menu("⋮", id=f"row-actions-menu-{row_id}", hx_get=f"/dropdown-menu/dropdown-menu-⋮/{row_id}") + +def get_row_actions_menu(row_id): + return dropdown_menu_content(f"row-actions-{row_id}", [ + {"label": "Edit", "icon": "pencil"}, + {"label": "Duplicate", "icon": "copy"}, + {"label": "Delete", "icon": "trash"}, + "separator", + {"label": "More...", "icon": "more-horizontal"}, + ]) + +def render_row(row_data, row_id, columns): + return Tr( + Td(checkbox(f"row-{row_id}", "", class_="row-checkbox")), + *[Td(row_data[col['value']], data_accessor=col['value']) for col in columns], + Td(row_actions_menu(row_id)), + id=f"row-{row_id}" + ).render() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 644fe2b3..a49a80d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,4 +7,5 @@ services: ports: - 8001:8000 volumes: - - ./api.yaml:/home/api.yaml \ No newline at end of file + - ./api.yaml:/home/api.yaml + - ./uniapi_db:/home/data \ No newline at end of file diff --git a/json_str/Vertex/text.json b/json_str/Vertex/text.json deleted file mode 100644 index 33c4e402..00000000 --- a/json_str/Vertex/text.json +++ /dev/null @@ -1,30 +0,0 @@ -"contents": [ - { - "role": string, - "parts": [ - { - // Union field data can be only one of the following: - "text": string, - "inlineData": { - "mimeType": string, - "data": string - }, - "fileData": { - "mimeType": string, - "fileUri": string - }, - // End of list of possible types for union field data. - "videoMetadata": { - "startOffset": { - "seconds": integer, - "nanos": integer - }, - "endOffset": { - "seconds": integer, - "nanos": integer - } - } - } - ] - } - ], \ No newline at end of file diff --git a/json_str/claude/request.json b/json_str/claude/request.json deleted file mode 100644 index dd99c88e..00000000 --- a/json_str/claude/request.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "hi" - } - ] - } - ], - "temperature": 0.5, - "top_p": 0.7, - "max_tokens": 4096, - "stream": true, - "system": "You are Claude, a large language model trained by Anthropic. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Respond conversationally in English.", - "tools": [ - { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - }, - { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "input_schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - }, - { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - ], - "tool_choice": { - "type": "auto" - } -} \ No newline at end of file diff --git a/json_str/claude/tool_use.json b/json_str/claude/tool_use.json deleted file mode 100644 index aecfc326..00000000 --- a/json_str/claude/tool_use.json +++ /dev/null @@ -1,47 +0,0 @@ -data: {"type":"message_start","message":{"id":"msg_01Jp7JVrr2MFfTzUBL9hrgoH","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":558,"output_tokens":1}} } -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } -data: {"type": "ping"} -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" apolog"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ize, but I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" need to"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" respon"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d in"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" English as that"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" the language I've"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" been instruct"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ed to use."} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Let"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" me"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" r"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ephrase your"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" request"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" an"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d procee"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d with searching"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for today"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news."} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n\nTo"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" search for today"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news, I"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" use the Google search"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" function"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":". Here"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" how"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I'll do that"} } -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":"} } -data: {"type":"content_block_stop","index":0 } -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01M17un8HfqkS3uDKBPuBr35","name":"get_search_results","input":{}} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"promp"} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"t\""} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":": \"toda"} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"y's "} } -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"top news\"}"} } -data: {"type":"content_block_stop","index":1 } -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":124} } -data: {"type":"message_stop" } \ No newline at end of file diff --git a/json_str/claude/tools.json b/json_str/claude/tools.json deleted file mode 100644 index 1864f113..00000000 --- a/json_str/claude/tools.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "搜索今天的新闻" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "toolu_01RofFmKHUKsEaZvqESG5Hwz", - "name": "get_search_results", - "input": { - "prompt": "latest news today" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01RofFmKHUKsEaZvqESG5Hwz", - "content": "latest news today" - } - ] - } - ], - "temperature": 0.5, - "top_p": 0.7, - "max_tokens": 4096, - "stream": true, - "system": "You are Claude, a large language model trained by Anthropic. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Respond conversationally in English.", - "tools": [ - { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - }, - { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "input_schema": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - }, - { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "input_schema": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - ], - "tool_choice": { - "type": "auto" - } -} \ No newline at end of file diff --git a/json_str/gemini/request.json b/json_str/gemini/request.json deleted file mode 100644 index 715a4e3c..00000000 --- a/json_str/gemini/request.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "hi" - } - ] - }, - { - "role": "model", - "parts": [ - { - "text": "Hi! \n\nHow are you today? What can I do for you? \n" - } - ] - }, - { - "role": "user", - "parts": [ - { - "text": "怎么解决" - }, - { - "inlineData": { - "mimeType": "image/jpeg", - "data": "/9j/***" - } - } - ] - } - ], - "safetySettings": [ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE" - } - ] -} \ No newline at end of file diff --git a/json_str/gpt/mess_sse.json b/json_str/gpt/mess_sse.json deleted file mode 100644 index a782b58e..00000000 --- a/json_str/gpt/mess_sse.json +++ /dev/null @@ -1,12 +0,0 @@ -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} -data: {"id":"chatcmpl-9j98vC0GPtmMAdOsSgh1TGhFQAsZC","object":"chat.completion.chunk","created":1720546933,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_ce0793330f","choices":[],"usage":{"prompt_tokens":178,"completion_tokens":10,"total_tokens":188}} \ No newline at end of file diff --git a/json_str/gpt/tool_use.json b/json_str/gpt/tool_use.json deleted file mode 100644 index 25445f93..00000000 --- a/json_str/gpt/tool_use.json +++ /dev/null @@ -1,8 +0,0 @@ -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_hbFDbIHYbimw1J0v9d1qvpgl","type":"function","function":{"name":"get_search_results","arguments":""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"prompt"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"today"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"'s"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" news"}}]},"logprobs":null,"finish_reason":null}]} -data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]} \ No newline at end of file diff --git a/json_str/gpt/tools.json b/json_str/gpt/tools.json deleted file mode 100644 index 1b5aa963..00000000 --- a/json_str/gpt/tools.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "model": "gpt-4o", - "messages": [ - { - "role": "system", - "content": "You are ChatGPT, a large language model trained by OpenAI. Respond conversationally in English. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Knowledge cutoff: 2023-12. Current date: [ 2024-07-09 ]" - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "搜索今天的新闻" - } - ] - }, - { - "role": "function", - "name": "get_search_results", - "content": "latest news today" - } - ], - "max_tokens": 4096, - "stream": true, - "temperature": 0.5, - "top_p": 1.0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "n": 1, - "user": "function", - "tools": [ - { - "type": "function", - "function": { - "name": "get_search_results", - "description": "Search Google to enhance knowledge.", - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "The prompt to search." - } - }, - "required": [ - "prompt" - ] - } - } - }, - { - "type": "function", - "function": { - "name": "get_url_content", - "description": "Get the webpage content of a URL", - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "the URL to request" - } - }, - "required": [ - "url" - ] - } - } - }, - { - "type": "function", - "function": { - "name": "download_read_arxiv_pdf", - "description": "Get the content of the paper corresponding to the arXiv ID", - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "string", - "description": "the arXiv ID of the paper" - } - }, - "required": [ - "prompt" - ] - } - } - } - ], - "tool_choice": "auto" -} \ No newline at end of file diff --git a/log_config.py b/log_config.py index 8bf3c4e3..9a1aa037 100644 --- a/log_config.py +++ b/log_config.py @@ -2,4 +2,5 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") logger = logging.getLogger("uni-api") -logging.getLogger("httpx").setLevel(logging.CRITICAL) \ No newline at end of file +logging.getLogger("httpx").setLevel(logging.CRITICAL) +logging.getLogger("watchfiles.main").setLevel(logging.CRITICAL) \ No newline at end of file diff --git a/main.py b/main.py index 49e6285f..b782ab48 100644 --- a/main.py +++ b/main.py @@ -1,75 +1,594 @@ from log_config import logger +import copy import httpx import secrets +from time import time from contextlib import asynccontextmanager +from starlette.middleware.base import BaseHTTPMiddleware from fastapi.middleware.cors import CORSMiddleware -from fastapi import FastAPI, HTTPException, Depends, Request -from fastapi.responses import StreamingResponse, JSONResponse +from fastapi import FastAPI, HTTPException, Depends, Request, APIRouter +from fastapi.responses import JSONResponse +from fastapi.responses import StreamingResponse as FastAPIStreamingResponse +from starlette.responses import StreamingResponse as StarletteStreamingResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi.exceptions import RequestValidationError -from models import RequestModel -from utils import error_handling_wrapper, get_all_models, post_all_models, load_config +from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, TextToSpeechRequest, UnifiedRequest, EmbeddingRequest from request import get_payload from response import fetch_response, fetch_response_stream +from utils import ( + safe_get, + get_engine, + load_config, + save_api_yaml, + get_model_dict, + post_all_models, + circular_list_encoder, + error_handling_wrapper, + rate_limiter, + provider_api_circular_list, + ThreadSafeCircularList, +) -from typing import List, Dict +from collections import defaultdict +from typing import List, Dict, Union from urllib.parse import urlparse +import os +import string +import json + +DEFAULT_TIMEOUT = int(os.getenv("TIMEOUT", 100)) +is_debug = bool(os.getenv("DEBUG", False)) +# is_debug = False + +from sqlalchemy import inspect, text +from sqlalchemy.sql import sqltypes + +# 添加新的环境变量检查 +DISABLE_DATABASE = os.getenv("DISABLE_DATABASE", "false").lower() == "true" +IS_VERCEL = os.path.dirname(os.path.abspath(__file__)).startswith('/var/task') +logger.info("IS_VERCEL: %s", IS_VERCEL) +logger.info("DISABLE_DATABASE: %s", DISABLE_DATABASE) + +# 读取VERSION文件内容 +try: + with open('VERSION', 'r') as f: + VERSION = f.read().strip() +except: + VERSION = 'unknown' +logger.info("VERSION: %s", VERSION) + +async def create_tables(): + if DISABLE_DATABASE: + return + async with db_engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + # 检查并添加缺失的列 + def check_and_add_columns(connection): + inspector = inspect(connection) + for table in [RequestStat, ChannelStat]: + table_name = table.__tablename__ + existing_columns = {col['name']: col['type'] for col in inspector.get_columns(table_name)} + + for column_name, column in table.__table__.columns.items(): + if column_name not in existing_columns: + col_type = _map_sa_type_to_sql_type(column.type) + default = _get_default_sql(column.default) + connection.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {col_type}{default}")) + + await conn.run_sync(check_and_add_columns) + +def _map_sa_type_to_sql_type(sa_type): + type_map = { + sqltypes.Integer: "INTEGER", + sqltypes.String: "TEXT", + sqltypes.Float: "REAL", + sqltypes.Boolean: "BOOLEAN", + sqltypes.DateTime: "DATETIME", + sqltypes.Text: "TEXT" + } + return type_map.get(type(sa_type), "TEXT") + +def _get_default_sql(default): + if default is None: + return "" + if isinstance(default.arg, bool): + return f" DEFAULT {str(default.arg).upper()}" + if isinstance(default.arg, (int, float)): + return f" DEFAULT {default.arg}" + if isinstance(default.arg, str): + return f" DEFAULT '{default.arg}'" + return "" + @asynccontextmanager async def lifespan(app: FastAPI): + # print("Main app routes:") + # for route in app.routes: + # print(f"Route: {route.path}, methods: {route.methods}") + + # print("\nFrontend router routes:") + # for route in frontend_router.routes: + # print(f"Route: {route.path}, methods: {route.methods}") + # 启动时的代码 - timeout = httpx.Timeout(connect=15.0, read=20.0, write=30.0, pool=30.0) - default_headers = { - "User-Agent": "curl/7.68.0", # 模拟 curl 的 User-Agent - "Accept": "*/*", # curl 的默认 Accept 头 - } - app.state.client = httpx.AsyncClient( - timeout=timeout, - headers=default_headers, - http2=True, # 禁用 HTTP/2 - verify=True, # 保持 SSL 验证（如需禁用，设为 False，但不建议） - follow_redirects=True, # 自动跟随重定向 - ) - # app.state.client = httpx.AsyncClient(timeout=timeout) - app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) + if not DISABLE_DATABASE: + await create_tables() + yield # 关闭时的代码 - await app.state.client.aclose() + # await app.state.client.aclose() + if hasattr(app.state, 'client_manager'): + await app.state.client_manager.close() + +app = FastAPI(lifespan=lifespan, debug=is_debug) + +def generate_markdown_docs(): + openapi_schema = app.openapi() + + markdown = f"# {openapi_schema['info']['title']}\n\n" + markdown += f"Version: {openapi_schema['info']['version']}\n\n" + markdown += f"{openapi_schema['info'].get('description', '')}\n\n" + + markdown += "## API Endpoints\n\n" + + paths = openapi_schema['paths'] + for path, path_info in paths.items(): + for method, operation in path_info.items(): + markdown += f"### {method.upper()} {path}\n\n" + markdown += f"{operation.get('summary', '')}\n\n" + markdown += f"{operation.get('description', '')}\n\n" + + if 'parameters' in operation: + markdown += "Parameters:\n" + for param in operation['parameters']: + markdown += f"- {param['name']} ({param['in']}): {param.get('description', '')}\n" + + markdown += "\n---\n\n" + + return markdown + +@app.get("/docs/markdown") +async def get_markdown_docs(): + markdown = generate_markdown_docs() + return Response( + content=markdown, + media_type="text/markdown" + ) + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 404: + logger.error(f"404 Error: {exc.detail}") + return JSONResponse( + status_code=exc.status_code, + content={"message": exc.detail}, + ) + +import uuid +import asyncio +import contextvars +request_info = contextvars.ContextVar('request_info', default={}) + +async def parse_request_body(request: Request): + if request.method == "POST" and "application/json" in request.headers.get("content-type", ""): + try: + return await request.json() + except json.JSONDecodeError: + return None + return None + +class ChannelManager: + def __init__(self, cooldown_period=300): + self._excluded_models = defaultdict(lambda: None) + self.cooldown_period = cooldown_period + + async def exclude_model(self, provider: str, model: str): + model_key = f"{provider}/{model}" + self._excluded_models[model_key] = datetime.now() + + async def is_model_excluded(self, provider: str, model: str) -> bool: + model_key = f"{provider}/{model}" + excluded_time = self._excluded_models[model_key] + if not excluded_time: + return False + + if datetime.now() - excluded_time > timedelta(seconds=self.cooldown_period): + del self._excluded_models[model_key] + return False + return True + + async def get_available_providers(self, providers: list) -> list: + """过滤出可用的providers，仅排除不可用的模型""" + available_providers = [] + for provider in providers: + provider_name = provider['provider'] + model_dict = provider['model'][0] # 获取唯一的模型字典 + # source_model = list(model_dict.keys())[0] # 源模型名称 + target_model = list(model_dict.values())[0] # 目标模型名称 + + # 检查该模型是否被排除 + if not await self.is_model_excluded(provider_name, target_model): + available_providers.append(provider) + + return available_providers + +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import declarative_base, sessionmaker +from sqlalchemy import Column, Integer, String, Float, DateTime, select, Boolean, Text +from sqlalchemy.sql import func + +# 定义数据库模型 +Base = declarative_base() + +class RequestStat(Base): + __tablename__ = 'request_stats' + id = Column(Integer, primary_key=True) + request_id = Column(String) + endpoint = Column(String) + client_ip = Column(String) + process_time = Column(Float) + first_response_time = Column(Float) + provider = Column(String) + model = Column(String) + # success = Column(Boolean, default=False) + api_key = Column(String) + is_flagged = Column(Boolean, default=False) + text = Column(Text) + prompt_tokens = Column(Integer, default=0) + completion_tokens = Column(Integer, default=0) + total_tokens = Column(Integer, default=0) + # cost = Column(Float, default=0) + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + +class ChannelStat(Base): + __tablename__ = 'channel_stats' + id = Column(Integer, primary_key=True) + request_id = Column(String) + provider = Column(String) + model = Column(String) + api_key = Column(String) + success = Column(Boolean, default=False) + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + + +if not DISABLE_DATABASE: + # 获取数据库路径 + db_path = os.getenv('DB_PATH', './data/stats.db') + + # 确保 data 目录存在 + data_dir = os.path.dirname(db_path) + os.makedirs(data_dir, exist_ok=True) + + # 创建异步引擎和会话 + # db_engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=False) + db_engine = create_async_engine('sqlite+aiosqlite:///' + db_path, echo=is_debug) + async_session = sessionmaker(db_engine, class_=AsyncSession, expire_on_commit=False) + +from starlette.types import Scope, Receive, Send +from starlette.responses import Response + +from asyncio import Semaphore + +# 创建一个信号量来控制数据库访问 +db_semaphore = Semaphore(1) # 限制同时只有1个写入操作 + +async def update_stats(current_info): + if DISABLE_DATABASE: + return + + try: + # 等待获取数据库访问权限 + async with db_semaphore: + async with async_session() as session: + async with session.begin(): + try: + columns = [column.key for column in RequestStat.__table__.columns] + filtered_info = {k: v for k, v in current_info.items() if k in columns} + new_request_stat = RequestStat(**filtered_info) + session.add(new_request_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating stats: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + except Exception as e: + logger.error(f"Error acquiring database lock: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + +async def update_channel_stats(request_id, provider, model, api_key, success): + if DISABLE_DATABASE: + return + + try: + async with db_semaphore: + async with async_session() as session: + async with session.begin(): + try: + channel_stat = ChannelStat( + request_id=request_id, + provider=provider, + model=model, + api_key=api_key, + success=success, + ) + session.add(channel_stat) + await session.commit() + except Exception as e: + await session.rollback() + logger.error(f"Error updating channel stats: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + except Exception as e: + logger.error(f"Error acquiring database lock: {str(e)}") + if is_debug: + import traceback + traceback.print_exc() + +class LoggingStreamingResponse(Response): + def __init__(self, content, status_code=200, headers=None, media_type=None, current_info=None): + super().__init__(content=None, status_code=status_code, headers=headers, media_type=media_type) + self.body_iterator = content + self._closed = False + self.current_info = current_info + + # Remove Content-Length header if it exists + if 'content-length' in self.headers: + del self.headers['content-length'] + # Set Transfer-Encoding to chunked + self.headers['transfer-encoding'] = 'chunked' + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await send({ + 'type': 'http.response.start', + 'status': self.status_code, + 'headers': self.raw_headers, + }) + + try: + async for chunk in self._logging_iterator(): + await send({ + 'type': 'http.response.body', + 'body': chunk, + 'more_body': True, + }) + finally: + await send({ + 'type': 'http.response.body', + 'body': b'', + 'more_body': False, + }) + if hasattr(self.body_iterator, 'aclose') and not self._closed: + await self.body_iterator.aclose() + self._closed = True + + process_time = time() - self.current_info["start_time"] + self.current_info["process_time"] = process_time + await update_stats(self.current_info) + + async def _logging_iterator(self): + try: + async for chunk in self.body_iterator: + if isinstance(chunk, str): + chunk = chunk.encode('utf-8') + if self.current_info.get("endpoint") == "/v1/audio/speech": + yield chunk + continue + line = chunk.decode('utf-8') + if is_debug: + logger.info(f"{line.encode('utf-8').decode('unicode_escape')}") + if line.startswith("data:"): + line = line.lstrip("data: ") + if not line.startswith("[DONE]") and not line.startswith("OK"): + try: + resp: dict = json.loads(line) + input_tokens = safe_get(resp, "message", "usage", "input_tokens", default=0) + input_tokens = safe_get(resp, "usage", "prompt_tokens", default=0) + output_tokens = safe_get(resp, "usage", "completion_tokens", default=0) + total_tokens = input_tokens + output_tokens + + self.current_info["prompt_tokens"] = input_tokens + self.current_info["completion_tokens"] = output_tokens + self.current_info["total_tokens"] = total_tokens + except Exception as e: + logger.error(f"Error parsing response: {str(e)}, line: {repr(line)}") + continue + yield chunk + except Exception as e: + raise + finally: + logger.debug("_logging_iterator finished") + + async def close(self): + if not self._closed: + self._closed = True + if hasattr(self.body_iterator, 'aclose'): + await self.body_iterator.aclose() + +class StatsMiddleware(BaseHTTPMiddleware): + def __init__(self, app): + super().__init__(app) + + async def dispatch(self, request: Request, call_next): + start_time = time() + + enable_moderation = False # 默认不开启道德审查 + + config = app.state.config + # 根据token决定是否启用道德审查 + if request.headers.get("x-api-key"): + token = request.headers.get("x-api-key") + elif request.headers.get("Authorization"): + api_split_list = request.headers.get("Authorization").split(" ") + if len(api_split_list) > 1: + token = api_split_list[1] + else: + return JSONResponse( + status_code=403, + content={"error": "Invalid or missing API Key"} + ) + else: + token = None + + api_index = None + if token: + try: + api_list = app.state.api_list + api_index = api_list.index(token) + except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + # token不在api_list中，使用默认值（不开启） + pass + + if api_index is not None: + enable_moderation = safe_get(config, 'api_keys', api_index, "preferences", "ENABLE_MODERATION", default=False) + else: + return JSONResponse( + status_code=403, + content={"error": "Invalid or missing API Key"} + ) + else: + # 如果token为None，检查全局设置 + enable_moderation = config.get('ENABLE_MODERATION', False) + + # 在 app.state 中存储此请求的信息 + request_id = str(uuid.uuid4()) + + # 初始化请求信息 + request_info_data = { + "request_id": request_id, + "start_time": start_time, + "endpoint": f"{request.method} {request.url.path}", + "client_ip": request.client.host, + "process_time": 0, + "first_response_time": -1, + "provider": None, + "model": None, + "success": False, + "api_key": token, + "is_flagged": False, + "text": None, + "prompt_tokens": 0, + "completion_tokens": 0, + # "cost": 0, + "total_tokens": 0 + } + + # 设置请求信息到上下文 + current_request_info = request_info.set(request_info_data) + current_info = request_info.get() + + parsed_body = await parse_request_body(request) + if parsed_body: + try: + request_model = UnifiedRequest.model_validate(parsed_body).data + if is_debug: + logger.info("request_model: %s", json.dumps(request_model.model_dump(exclude_unset=True), indent=2, ensure_ascii=False)) + model = request_model.model + current_info["model"] = model -app = FastAPI(lifespan=lifespan) + final_api_key = app.state.api_list[api_index] + try: + await app.state.user_api_keys_rate_limit[final_api_key].next(model) + except Exception as e: + return JSONResponse( + status_code=429, + content={"error": "Too many requests"} + ) -# from time import time -# from collections import defaultdict -# import asyncio + moderated_content = None + if request_model.request_type == "chat": + moderated_content = request_model.get_last_text_message() + elif request_model.request_type == "image": + moderated_content = request_model.prompt + elif request_model.request_type == "tts": + moderated_content = request_model.input + elif request_model.request_type == "moderation": + pass + elif request_model.request_type == "embedding": + if isinstance(request_model.input, list) and len(request_model.input) > 0 and isinstance(request_model.input[0], str): + moderated_content = "\n".join(request_model.input) + else: + moderated_content = request_model.input + else: + logger.error(f"Unknown request type: {request_model.request_type}") + + if moderated_content: + current_info["text"] = moderated_content + + if enable_moderation and moderated_content: + moderation_response = await self.moderate_content(moderated_content, api_index) + is_flagged = moderation_response.get('results', [{}])[0].get('flagged', False) + + if is_flagged: + logger.error(f"Content did not pass the moral check: %s", moderated_content) + process_time = time() - start_time + current_info["process_time"] = process_time + current_info["is_flagged"] = is_flagged + await update_stats(current_info) + return JSONResponse( + status_code=400, + content={"error": "Content did not pass the moral check, please modify and try again."} + ) + except RequestValidationError: + logger.error(f"Invalid request body: {parsed_body}") + pass + except Exception as e: + if is_debug: + import traceback + traceback.print_exc() -# class StatsMiddleware: -# def __init__(self): -# self.request_counts = defaultdict(int) -# self.request_times = defaultdict(float) -# self.ip_counts = defaultdict(lambda: defaultdict(int)) -# self.lock = asyncio.Lock() + logger.error(f"Error processing request or performing moral check: {str(e)}") -# async def __call__(self, request: Request, call_next): -# start_time = time() -# response = await call_next(request) -# process_time = time() - start_time + try: + response = await call_next(request) -# endpoint = f"{request.method} {request.url.path}" -# client_ip = request.client.host + if request.url.path.startswith("/v1") and not DISABLE_DATABASE: + if isinstance(response, (FastAPIStreamingResponse, StarletteStreamingResponse)) or type(response).__name__ == '_StreamingResponse': + response = LoggingStreamingResponse( + content=response.body_iterator, + status_code=response.status_code, + media_type=response.media_type, + headers=response.headers, + current_info=current_info, + ) + elif hasattr(response, 'json'): + logger.info(f"Response: {await response.json()}") + else: + logger.info(f"Response: type={type(response).__name__}, status_code={response.status_code}, headers={response.headers}") + + return response + finally: + # print("current_request_info", current_request_info) + request_info.reset(current_request_info) + + async def moderate_content(self, content, api_index): + moderation_request = ModerationRequest(input=content) -# async with self.lock: -# self.request_counts[endpoint] += 1 -# self.request_times[endpoint] += process_time -# self.ip_counts[endpoint][client_ip] += 1 + # 直接调用 moderations 函数 + response = await moderations(moderation_request, api_index) + + # 读取流式响应的内容 + moderation_result = b"" + async for chunk in response.body_iterator: + if isinstance(chunk, str): + moderation_result += chunk.encode('utf-8') + else: + moderation_result += chunk -# return response -# # 创建 StatsMiddleware 实例 -# stats_middleware = StatsMiddleware() + # 解码并解析 JSON + moderation_data = json.loads(moderation_result.decode('utf-8')) -# # 添加 StatsMiddleware -# app.add_middleware(StatsMiddleware) + return moderation_data # 配置 CORS 中间件 app.add_middleware( @@ -80,206 +599,1527 @@ async def lifespan(app: FastAPI): allow_headers=["*"], # 允许所有头部字段 ) -async def process_request(request: RequestModel, provider: Dict): - url = provider['base_url'] - parsed_url = urlparse(url) - # print(parsed_url) - engine = None - if parsed_url.netloc == 'generativelanguage.googleapis.com': - engine = "gemini" - elif parsed_url.netloc == 'aiplatform.googleapis.com': - engine = "vertex" - elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): - engine = "claude" - elif parsed_url.netloc == 'openrouter.ai': - engine = "openrouter" +app.add_middleware(StatsMiddleware) + +class ClientManager: + def __init__(self, pool_size=100): + self.pool_size = pool_size + self.clients = {} # {host_timeout_proxy: AsyncClient} + + async def init(self, default_config): + self.default_config = default_config + + @asynccontextmanager + async def get_client(self, timeout_value, base_url, proxy=None): + # 直接获取或创建客户端,不使用锁 + timeout_value = int(timeout_value) + + # 从base_url中提取主机名 + parsed_url = urlparse(base_url) + host = parsed_url.netloc + + # 创建唯一的客户端键 + client_key = f"{host}_{timeout_value}" + if proxy: + # 对代理URL进行规范化处理 + proxy_normalized = proxy.replace('socks5h://', 'socks5://') + client_key += f"_{proxy_normalized}" + + if client_key not in self.clients or IS_VERCEL: + timeout = httpx.Timeout( + connect=15.0, + read=timeout_value, + write=30.0, + pool=self.pool_size + ) + limits = httpx.Limits(max_connections=self.pool_size) + + client_config = { + **self.default_config, + "timeout": timeout, + "limits": limits + } + + if proxy: + # 解析代理URL + parsed = urlparse(proxy) + scheme = parsed.scheme.rstrip('h') + + if scheme == 'socks5': + try: + from httpx_socks import AsyncProxyTransport + proxy = proxy.replace('socks5h://', 'socks5://') + transport = AsyncProxyTransport.from_url(proxy) + client_config["transport"] = transport + # print("proxy", proxy) + except ImportError: + logger.error("httpx-socks package is required for SOCKS proxy support") + raise ImportError("Please install httpx-socks package for SOCKS proxy support: pip install httpx-socks") + else: + client_config["proxies"] = { + "http://": proxy, + "https://": proxy + } + + self.clients[client_key] = httpx.AsyncClient(**client_config) + + try: + yield self.clients[client_key] + except Exception as e: + if client_key in self.clients: + tmp_client = self.clients[client_key] + del self.clients[client_key] # 先删除引用 + await tmp_client.aclose() # 然后关闭客户端 + raise e + + async def close(self): + for client in self.clients.values(): + await client.aclose() + self.clients.clear() + +@app.middleware("http") +async def ensure_config(request: Request, call_next): + + if app and not hasattr(app.state, 'config'): + # logger.warning("Config not found, attempting to reload") + app.state.config, app.state.api_keys_db, app.state.api_list = await load_config(app) + + if app.state.api_list: + app.state.user_api_keys_rate_limit = defaultdict(ThreadSafeCircularList) + for api_index, api_key in enumerate(app.state.api_list): + app.state.user_api_keys_rate_limit[api_key] = ThreadSafeCircularList( + [api_key], + safe_get(app.state.config, 'api_keys', api_index, "preferences", "rate_limit", default={"default": "999999/min"}), + "round_robin" + ) + + for item in app.state.api_keys_db: + if item.get("role") == "admin": + app.state.admin_api_key = item.get("api") + if not hasattr(app.state, "admin_api_key"): + if len(app.state.api_keys_db) >= 1: + app.state.admin_api_key = app.state.api_keys_db[0].get("api") + else: + from utils import yaml_error_message + if yaml_error_message: + return JSONResponse( + status_code=500, + content={"error": yaml_error_message} + ) + else: + return JSONResponse( + status_code=500, + content={"error": "No admin API key found"} + ) + + if app and not hasattr(app.state, 'client_manager'): + + default_config = { + "headers": { + "User-Agent": "curl/7.68.0", + "Accept": "*/*", + }, + "http2": True, + "verify": True, + "follow_redirects": True + } + + # 初始化客户端管理器 + app.state.client_manager = ClientManager(pool_size=200) + await app.state.client_manager.init(default_config) + + # 存储超时配置 + app.state.timeouts = {} + if app.state.config and 'preferences' in app.state.config: + if isinstance(app.state.config['preferences'].get('model_timeout'), int): + app.state.timeouts["default"] = app.state.config['preferences'].get('model_timeout') + else: + for model_name, timeout_value in app.state.config['preferences'].get('model_timeout', {"default": DEFAULT_TIMEOUT}).items(): + app.state.timeouts[model_name] = timeout_value + if "default" not in app.state.config['preferences'].get('model_timeout', {}): + app.state.timeouts["default"] = DEFAULT_TIMEOUT + + app.state.provider_timeouts = defaultdict(lambda: defaultdict(lambda: DEFAULT_TIMEOUT)) + for provider in app.state.config["providers"]: + # print("provider", provider) + provider_timeout_settings = safe_get(provider, "preferences", "model_timeout", default={}) + # print("provider_timeout_settings", provider_timeout_settings) + if provider_timeout_settings: + for model_name, timeout_value in provider_timeout_settings.items(): + app.state.provider_timeouts[provider['provider']][model_name] = timeout_value + + app.state.provider_timeouts["global_time_out"] = app.state.timeouts + + # provider_timeouts_dict = { + # provider: dict(timeouts) + # for provider, timeouts in app.state.provider_timeouts.items() + # } + # print("app.state.provider_timeouts", provider_timeouts_dict) + # print("ai" in app.state.provider_timeouts) + + if app and not hasattr(app.state, "channel_manager"): + if app.state.config and 'preferences' in app.state.config: + COOLDOWN_PERIOD = app.state.config['preferences'].get('cooldown_period', 300) + else: + COOLDOWN_PERIOD = 300 + + app.state.channel_manager = ChannelManager(cooldown_period=COOLDOWN_PERIOD) + + if app and not hasattr(app.state, "error_triggers"): + if app.state.config and 'preferences' in app.state.config: + ERROR_TRIGGERS = app.state.config['preferences'].get('error_triggers', []) + else: + ERROR_TRIGGERS = [] + app.state.error_triggers = ERROR_TRIGGERS + + if app and app.state.api_keys_db and not hasattr(app.state, "models_list"): + app.state.models_list = {} + for item in app.state.api_keys_db: + api_key_model_list = item.get("model", []) + for provider_rule in api_key_model_list: + provider_name = provider_rule.split("/")[0] + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + models_list = [] + try: + # 构建请求头 + headers = { + "Authorization": f"Bearer {provider_name}" + } + # 发送GET请求获取模型列表 + base_url = "http://127.0.0.1:8000/v1/models" + async with app.state.client_manager.get_client(1, base_url) as client: + response = await client.get( + base_url, + headers=headers + ) + if response.status_code == 200: + models_data = response.json() + # 将获取到的模型添加到models_list + for model in models_data.get("data", []): + models_list.append(model["id"]) + except Exception as e: + if str(e): + logger.error(f"获取模型列表失败: {str(e)}") + app.state.models_list[provider_name] = models_list + + return await call_next(request) + +def get_timeout_value(provider_timeouts, original_model): + timeout_value = None + original_model = original_model.lower() + if original_model in provider_timeouts: + timeout_value = provider_timeouts[original_model] else: - engine = "gpt" + # 尝试模糊匹配模型 + for timeout_model in provider_timeouts: + if timeout_model != "default" and timeout_model in original_model: + timeout_value = provider_timeouts[timeout_model] + break + else: + # 如果模糊匹配失败，使用渠道的默认值 + timeout_value = provider_timeouts.get("default") + return timeout_value - if "claude" not in provider['model'][request.model] \ - and "gpt" not in provider['model'][request.model] \ - and "gemini" not in provider['model'][request.model]: - engine = "openrouter" +# 在 process_request 函数中更新成功和失败计数 +async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], provider: Dict, endpoint=None, role=None, num_matching_providers=1): + model_dict = get_model_dict(provider) + original_model = model_dict[request.model] - if provider.get("engine"): - engine = provider["engine"] + engine, stream_mode = get_engine(provider, endpoint, original_model) - logger.info(f"provider: {provider['provider']:<10} model: {request.model:<10} engine: {engine}") + if stream_mode != None: + request.stream = stream_mode + + channel_id = f"{provider['provider']}" + if engine != "moderation": + logger.info(f"provider: {channel_id:<11} model: {request.model:<22} engine: {engine} role: {role}") url, headers, payload = await get_payload(request, engine, provider) + if is_debug: + logger.info(url) + logger.info(json.dumps(headers, indent=4, ensure_ascii=False)) + if payload.get("file"): + pass + else: + logger.info(json.dumps(payload, indent=4, ensure_ascii=False)) + + current_info = request_info.get() + + provider_timeouts = safe_get(app.state.provider_timeouts, channel_id, default=app.state.provider_timeouts["global_time_out"]) + timeout_value = get_timeout_value(provider_timeouts, original_model) + if timeout_value is None: + timeout_value = get_timeout_value(app.state.provider_timeouts["global_time_out"], original_model) + if timeout_value is None: + timeout_value = app.state.timeouts.get("default", DEFAULT_TIMEOUT) + timeout_value = timeout_value * num_matching_providers + # print("timeout_value", channel_id, timeout_value) + + proxy = safe_get(provider, "preferences", "proxy", default=None) + # print("proxy", proxy) + + try: + async with app.state.client_manager.get_client(timeout_value, url, proxy) as client: + if request.stream: + generator = fetch_response_stream(client, url, headers, payload, engine, original_model) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream, app.state.error_triggers) + response = StarletteStreamingResponse(wrapped_generator, media_type="text/event-stream") + else: + generator = fetch_response(client, url, headers, payload, engine, original_model) + wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id, engine, request.stream, app.state.error_triggers) + + # 处理音频和其他二进制响应 + if endpoint == "/v1/audio/speech": + if isinstance(wrapped_generator, bytes): + response = Response(content=wrapped_generator, media_type="audio/mpeg") + else: + first_element = await anext(wrapped_generator) + first_element = first_element.lstrip("data: ") + first_element = json.loads(first_element) + response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json") + + # 更新成功计数和首次响应时间 + await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=True) + current_info["first_response_time"] = first_response_time + current_info["success"] = True + current_info["provider"] = channel_id + return response + + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: + await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=False) + raise e + +def weighted_round_robin(weights): + provider_names = list(weights.keys()) + current_weights = {name: 0 for name in provider_names} + num_selections = total_weight = sum(weights.values()) + weighted_provider_list = [] + + for _ in range(num_selections): + max_ratio = -1 + selected_letter = None + + for name in provider_names: + current_weights[name] += weights[name] + ratio = current_weights[name] / weights[name] + + if ratio > max_ratio: + max_ratio = ratio + selected_letter = name + + weighted_provider_list.append(selected_letter) + current_weights[selected_letter] -= total_weight + + return weighted_provider_list + +import random + +def lottery_scheduling(weights): + total_tickets = sum(weights.values()) + selections = [] + for _ in range(total_tickets): + ticket = random.randint(1, total_tickets) + cumulative = 0 + for provider, weight in weights.items(): + cumulative += weight + if ticket <= cumulative: + selections.append(provider) + break + return selections + +async def get_provider_rules(model_rule, config, request_model): + provider_rules = [] + if model_rule == "all": + # 如模型名为 all，则返回所有模型 + for provider in config["providers"]: + model_dict = get_model_dict(provider) + for model in model_dict.keys(): + provider_rules.append(provider["provider"] + "/" + model) + + elif "/" in model_rule: + if model_rule.startswith("<") and model_rule.endswith(">"): + model_rule = model_rule[1:-1] + # 处理带斜杠的模型名 + for provider in config['providers']: + model_dict = get_model_dict(provider) + if model_rule in model_dict.keys(): + provider_rules.append(provider['provider'] + "/" + model_rule) + else: + provider_name = model_rule.split("/")[0] + model_name_split = "/".join(model_rule.split("/")[1:]) + models_list = [] + + # api_keys 中 api 为 sk- 时，表示继承 api_keys，将 api_keys 中的 api key 当作 渠道 + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + if app.state.models_list.get(provider_name): + models_list = app.state.models_list[provider_name] + else: + models_list = [] + else: + for provider in config['providers']: + model_dict = get_model_dict(provider) + if provider['provider'] == provider_name: + models_list.extend(list(model_dict.keys())) + + # print("models_list", models_list) + # print("model_name", model_name) + # print("model_name_split", model_name_split) + # print("model", model) + + # api_keys 中 model 为 provider_name/* 时，表示所有模型都匹配 + if model_name_split == "*": + if request_model in models_list: + provider_rules.append(provider_name + "/" + request_model) + + # 如果请求模型名： gpt-4* ，则匹配所有以模型名开头且不以 * 结尾的模型 + for models_list_model in models_list: + if request_model.endswith("*") and models_list_model.startswith(request_model.rstrip("*")): + provider_rules.append(provider_name + "/" + models_list_model) + + # api_keys 中 model 为 provider_name/model_name 时，表示模型名完全匹配 + elif model_name_split == request_model \ + or (request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*"))): # api_keys 中 model 为 provider_name/model_name 时，请求模型名： model_name* + if model_name_split in models_list: + provider_rules.append(provider_name + "/" + model_name_split) - # request_info = { - # "url": url, - # "headers": headers, - # "payload": payload - # } - # import json - # logger.info(f"Request details: {json.dumps(request_info, indent=4, ensure_ascii=False)}") - - if request.stream: - model = provider['model'][request.model] - generator = fetch_response_stream(app.state.client, url, headers, payload, engine, model) - wrapped_generator = await error_handling_wrapper(generator, status_code=500) - return StreamingResponse(wrapped_generator, media_type="text/event-stream") else: - return await fetch_response(app.state.client, url, headers, payload) + for provider in config["providers"]: + model_dict = get_model_dict(provider) + if model_rule in model_dict.keys(): + provider_rules.append(provider["provider"] + "/" + model_rule) + + return provider_rules + +def get_provider_list(provider_rules, config, request_model): + provider_list = [] + # print("provider_rules", provider_rules) + for item in provider_rules: + provider_name = item.split("/")[0] + if provider_name.startswith("sk-") and provider_name in app.state.api_list: + provider_list.append({"provider": provider_name, "base_url": "http://127.0.0.1:8000/v1/chat/completions", "model": [{request_model: request_model}], "tools": True}) + else: + for provider in config['providers']: + model_dict = get_model_dict(provider) + model_name_split = "/".join(item.split("/")[1:]) + if "/" in item and provider['provider'] == provider_name and model_name_split in model_dict.keys(): + new_provider = copy.deepcopy(provider) + # old: new + # print("item", item) + # print("model_dict", model_dict) + # print("model_name_split", model_name_split) + # print("request_model", request_model) + new_provider["model"] = [{model_dict[model_name_split]: request_model}] + if request_model in model_dict.keys() and model_name_split == request_model: + provider_list.append(new_provider) + + elif request_model.endswith("*") and model_name_split.startswith(request_model.rstrip("*")): + provider_list.append(new_provider) + return provider_list + +async def get_matching_providers(request_model, config, api_index): + provider_rules = [] + + for model_rule in config['api_keys'][api_index]['model']: + provider_rules.extend(await get_provider_rules(model_rule, config, request_model)) + + provider_list = get_provider_list(provider_rules, config, request_model) + + # print("provider_list", provider_list) + return provider_list + +async def get_right_order_providers(request_model, config, api_index, scheduling_algorithm): + matching_providers = await get_matching_providers(request_model, config, api_index) + + if not matching_providers: + raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") + + num_matching_providers = len(matching_providers) + if app.state.channel_manager.cooldown_period > 0 and num_matching_providers > 1: + matching_providers = await app.state.channel_manager.get_available_providers(matching_providers) + if not matching_providers: + raise HTTPException(status_code=503, detail="No available providers at the moment") + + # 检查是否启用轮询 + if scheduling_algorithm == "random": + matching_providers = random.sample(matching_providers, num_matching_providers) + + weights = safe_get(config, 'api_keys', api_index, "weights") + + if weights: + intersection = None + all_providers = set(provider['provider'] + "/" + request_model for provider in matching_providers) + if all_providers: + weight_keys = set(weights.keys()) + provider_rules = [] + for model_rule in weight_keys: + provider_rules.extend(await get_provider_rules(model_rule, config, request_model)) + provider_list = get_provider_list(provider_rules, config, request_model) + weight_keys = set([provider['provider'] + "/" + request_model for provider in provider_list]) + # print("all_providers", all_providers) + # print("weights", weights) + # print("weight_keys", weight_keys) + + # 步骤 3: 计算交集 + intersection = all_providers.intersection(weight_keys) + # print("intersection", intersection) + if len(intersection) == 1: + intersection = None + + if intersection: + filtered_weights = {k.split("/")[0]: v for k, v in weights.items() if k.split("/")[0] + "/" + request_model in intersection} + # print("filtered_weights", filtered_weights) + + if scheduling_algorithm == "weighted_round_robin": + weighted_provider_name_list = weighted_round_robin(filtered_weights) + elif scheduling_algorithm == "lottery": + weighted_provider_name_list = lottery_scheduling(filtered_weights) + else: + weighted_provider_name_list = list(filtered_weights.keys()) + # print("weighted_provider_name_list", weighted_provider_name_list) + + new_matching_providers = [] + for provider_name in weighted_provider_name_list: + for provider in matching_providers: + if provider['provider'] == provider_name: + new_matching_providers.append(provider) + matching_providers = new_matching_providers + + if is_debug: + for provider in matching_providers: + logger.info("available provider: %s", json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) + + return matching_providers import asyncio class ModelRequestHandler: def __init__(self): - self.last_provider_index = -1 + self.last_provider_indices = defaultdict(lambda: -1) + self.locks = defaultdict(asyncio.Lock) - def get_matching_providers(self, model_name, token): + async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest], api_index: int = None, endpoint=None): config = app.state.config - # api_keys_db = app.state.api_keys_db - api_list = app.state.api_list + request_model = request.model + if not safe_get(config, 'api_keys', api_index, 'model'): + raise HTTPException(status_code=404, detail=f"No matching model found: {request_model}") - api_index = api_list.index(token) - provider_rules = [] + scheduling_algorithm = safe_get(config, 'api_keys', api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") - for model in config['api_keys'][api_index]['model']: - if "/" in model: - provider_name = model.split("/")[0] - model = model.split("/")[1] - models_list = [] - for provider in config['providers']: - if provider['provider'] == provider_name: - models_list.extend(list(provider['model'].keys())) - # print("models_list", models_list) - # print("model_name", model_name) - # print("model", model) - if (model and model_name in models_list) or (model == "*" and model_name in models_list): - provider_rules.append(provider_name) - else: - for provider in config['providers']: - if model in provider['model'].keys(): - provider_rules.append(provider['provider'] + "/" + model) + matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) + num_matching_providers = len(matching_providers) - provider_list = [] - # print("provider_rules", provider_rules) - for item in provider_rules: - for provider in config['providers']: - if provider['provider'] in item: - if "/" in item: - if item.split("/")[1] == model_name: - provider_list.append(provider) - else: - if model_name in provider['model'].keys(): - provider_list.append(provider) - return provider_list + status_code = 500 + error_message = None - async def request_model(self, request: RequestModel, token: str): - config = app.state.config - # api_keys_db = app.state.api_keys_db - api_list = app.state.api_list + start_index = 0 + if scheduling_algorithm != "fixed_priority": + async with self.locks[request_model]: + self.last_provider_indices[request_model] = (self.last_provider_indices[request_model] + 1) % num_matching_providers + start_index = self.last_provider_indices[request_model] - model_name = request.model - matching_providers = self.get_matching_providers(model_name, token) - # import json - # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False)) - if not matching_providers: - raise HTTPException(status_code=404, detail="No matching model found") + auto_retry = safe_get(config, 'api_keys', api_index, "preferences", "AUTO_RETRY", default=True) + role = safe_get(config, 'api_keys', api_index, "role", default=safe_get(config, 'api_keys', api_index, "api", default="None")[:8]) + + index = 0 + if num_matching_providers == 1 and (count := provider_api_circular_list[matching_providers[0]['provider']].get_items_count()) > 1: + retry_count = count + else: + retry_count = int(auto_retry) + + while True: + # print("start_index", start_index) + # print("index", index) + # print("num_matching_providers", num_matching_providers) + # print("retry_count", retry_count) + if index >= num_matching_providers + retry_count: + break + current_index = (start_index + index) % num_matching_providers + index += 1 + provider = matching_providers[current_index] + + if provider['provider'].startswith("sk-") and provider['provider'] in app.state.api_list: + local_provider_api_index = app.state.api_list.index(provider['provider']) + local_provider_scheduling_algorithm = safe_get(config, 'api_keys', local_provider_api_index, "preferences", "SCHEDULING_ALGORITHM", default="fixed_priority") + local_provider_matching_providers = await get_right_order_providers(request_model, config, local_provider_api_index, local_provider_scheduling_algorithm) + local_provider_num_matching_providers = len(local_provider_matching_providers) + else: + local_provider_num_matching_providers = 1 - # 检查是否启用轮询 - api_index = api_list.index(token) - use_round_robin = True - auto_retry = True - if config['api_keys'][api_index].get("preferences"): - if config['api_keys'][api_index]["preferences"].get("USE_ROUND_ROBIN") == False: - use_round_robin = False - if config['api_keys'][api_index]["preferences"].get("AUTO_RETRY") == False: - auto_retry = False - - return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry) - - async def try_all_providers(self, request: RequestModel, providers: List[Dict], use_round_robin: bool, auto_retry: bool): - num_providers = len(providers) - start_index = self.last_provider_index + 1 if use_round_robin else 0 - - for i in range(num_providers + 1): - self.last_provider_index = (start_index + i) % num_providers - provider = providers[self.last_provider_index] try: - response = await process_request(request, provider) + response = await process_request(request, provider, endpoint, role, local_provider_num_matching_providers) return response - except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError) as e: - logger.error(f"Error with provider {provider['provider']}: {str(e)}") - if auto_retry: - continue + except (Exception, HTTPException, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e: + + # 根据异常类型设置状态码和错误消息 + if isinstance(e, httpx.ReadTimeout): + status_code = 504 # Gateway Timeout + timeout_value = e.request.extensions.get('timeout', {}).get('read', -1) + error_message = f"Request timed out after {timeout_value} seconds" + elif isinstance(e, httpx.ConnectError): + status_code = 503 # Service Unavailable + error_message = "Unable to connect to service" + elif isinstance(e, httpx.ReadError): + status_code = 502 # Bad Gateway + error_message = "Network read error" + elif isinstance(e, httpx.RemoteProtocolError): + status_code = 502 # Bad Gateway + error_message = "Remote protocol error" + elif isinstance(e, asyncio.CancelledError): + status_code = 499 # Client Closed Request + error_message = "Request was cancelled" + elif isinstance(e, HTTPException): + status_code = e.status_code + error_message = str(e.detail) else: - raise HTTPException(status_code=500, detail="Error: Current provider response failed!") + status_code = 500 # Internal Server Error + error_message = str(e) or f"Unknown error: {e.__class__.__name__}" + + channel_id = f"{provider['provider']}" + if app.state.channel_manager.cooldown_period > 0 and num_matching_providers > 1: + # 获取源模型名称（实际配置的模型名） + # source_model = list(provider['model'][0].keys())[0] + await app.state.channel_manager.exclude_model(channel_id, request_model) + matching_providers = await get_right_order_providers(request_model, config, api_index, scheduling_algorithm) + last_num_matching_providers = num_matching_providers + num_matching_providers = len(matching_providers) + if num_matching_providers != last_num_matching_providers: + index = 0 + cooling_time = safe_get(provider, "preferences", "api_key_cooldown_period", default=0) + api_key_count = provider_api_circular_list[channel_id].get_items_count() + current_api = await provider_api_circular_list[channel_id].after_next_current() + if cooling_time > 0 and api_key_count > 1: + await provider_api_circular_list[channel_id].set_cooling(current_api, cooling_time=cooling_time) - raise HTTPException(status_code=500, detail=f"All providers failed: {request.model}") + logger.error(f"Error {status_code} with provider {channel_id} API key: {current_api}: {error_message}") + if is_debug: + import traceback + traceback.print_exc() + if auto_retry and status_code != 413: + continue + else: + return JSONResponse( + status_code=status_code, + content={"error": f"Error: Current provider response failed: {error_message}"} + ) + + current_info = request_info.get() + current_info["first_response_time"] = -1 + current_info["success"] = False + current_info["provider"] = None + return JSONResponse( + status_code=status_code, + content={"error": f"All {request.model} error: {error_message}"} + ) model_handler = ModelRequestHandler() -# 安全性依赖 security = HTTPBearer() def verify_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): api_list = app.state.api_list token = credentials.credentials - if token not in api_list: + api_index = None + try: + api_index = api_list.index(token) + except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + if api_index is None: raise HTTPException(status_code=403, detail="Invalid or missing API Key") + return api_index + +def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)): + api_list = app.state.api_list + token = credentials.credentials + api_index = None + try: + api_index = api_list.index(token) + except ValueError: + # 如果 token 不在 api_list 中，检查是否以 api_list 中的任何一个开头 + api_index = next((i for i, api in enumerate(api_list) if token.startswith(api)), None) + if api_index is None: + raise HTTPException(status_code=403, detail="Invalid or missing API Key") + # for api_key in app.state.api_keys_db: + # if token.startswith(api_key['api']): + if app.state.api_keys_db[api_index].get('role') != "admin": + raise HTTPException(status_code=403, detail="Permission denied") return token @app.post("/v1/chat/completions") -async def request_model(request: RequestModel, token: str = Depends(verify_api_key)): - return await model_handler.request_model(request, token) +async def request_model(request: RequestModel, api_index: int = Depends(verify_api_key)): + return await model_handler.request_model(request, api_index) @app.options("/v1/chat/completions") async def options_handler(): return JSONResponse(status_code=200, content={"detail": "OPTIONS allowed"}) -@app.post("/v1/models") -async def list_models(token: str = Depends(verify_api_key)): - models = post_all_models(token, app.state.config, app.state.api_list) - return JSONResponse(content={ - "object": "list", - "data": models - }) - @app.get("/v1/models") -async def list_models(): - models = get_all_models(config=app.state.config) +async def list_models(api_index: int = Depends(verify_api_key)): + models = post_all_models(api_index, app.state.config, app.state.api_list, app.state.models_list) return JSONResponse(content={ "object": "list", "data": models }) -@app.get("/generate-api-key") +@app.post("/v1/images/generations") +async def images_generations( + request: ImageGenerationRequest, + api_index: int = Depends(verify_api_key) +): + return await model_handler.request_model(request, api_index, endpoint="/v1/images/generations") + +@app.post("/v1/embeddings") +async def embeddings( + request: EmbeddingRequest, + api_index: int = Depends(verify_api_key) +): + return await model_handler.request_model(request, api_index, endpoint="/v1/embeddings") + +@app.post("/v1/audio/speech") +async def audio_speech( + request: TextToSpeechRequest, + api_index: str = Depends(verify_api_key) +): + return await model_handler.request_model(request, api_index, endpoint="/v1/audio/speech") + +@app.post("/v1/moderations") +async def moderations( + request: ModerationRequest, + api_index: int = Depends(verify_api_key) +): + return await model_handler.request_model(request, api_index, endpoint="/v1/moderations") + +from fastapi import UploadFile, File, Form, HTTPException +import io +@app.post("/v1/audio/transcriptions") +async def audio_transcriptions( + file: UploadFile = File(...), + model: str = Form(...), + api_index: int = Depends(verify_api_key) +): + try: + # 读取上传的文件内容 + content = await file.read() + file_obj = io.BytesIO(content) + + # 创建AudioTranscriptionRequest对象 + request = AudioTranscriptionRequest( + file=(file.filename, file_obj, file.content_type), + model=model + ) + + return await model_handler.request_model(request, api_index, endpoint="/v1/audio/transcriptions") + except UnicodeDecodeError: + raise HTTPException(status_code=400, detail="Invalid audio file encoding") + except Exception as e: + if is_debug: + import traceback + traceback.print_exc() + raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}") + +@app.get("/v1/generate-api-key") def generate_api_key(): - api_key = "sk-" + secrets.token_urlsafe(32) + # Define the character set (only alphanumeric) + chars = string.ascii_letters + string.digits + # Generate a random string of 36 characters + random_string = ''.join(secrets.choice(chars) for _ in range(48)) + api_key = "sk-" + random_string return JSONResponse(content={"api_key": api_key}) -# @app.get("/stats") -# async def get_stats(token: str = Depends(verify_api_key)): -# async with stats_middleware.lock: -# return { -# "request_counts": dict(stats_middleware.request_counts), -# "average_request_times": { -# endpoint: total_time / count -# for endpoint, total_time in stats_middleware.request_times.items() -# for count in [stats_middleware.request_counts[endpoint]] -# }, -# "ip_counts": { -# endpoint: dict(ips) -# for endpoint, ips in stats_middleware.ip_counts.items() -# } -# } +# 在 /stats 路由中返回成功和失败百分比 +from datetime import datetime, timedelta, timezone +from sqlalchemy import func, desc, case +from fastapi import Query + +@app.get("/v1/stats") +async def get_stats( + request: Request, + token: str = Depends(verify_admin_api_key), + hours: int = Query(default=24, ge=1, le=720, description="Number of hours to look back for stats (1-720)") +): + ''' + ## 获取统计数据 + + 使用 `/v1/stats` 获取最近 24 小时各个渠道的使用情况统计。同时带上 自己的 uni-api 的 admin API key。 + + 数据包括： + + 1. 每个渠道下面每个模型的成功率，成功率从高到低排序。 + 2. 每个渠道总的成功率，成功率从高到低排序。 + 3. 每个模型在所有渠道总的请求次数。 + 4. 每个端点的请求次数。 + 5. 每个ip请求的次数。 + + `/v1/stats?hours=48` 参数 `hours` 可以控制返回最近多少小时的数据统计，不传 `hours` 这个参数，默认统计最近 24 小时的统计数据。 + + 还有其他统计数据，可以自己写sql在数据库自己查。其他数据包括：首字时间，每个请求的总处理时间，每次请求是否成功，每次请求是否符合道德审查，每次请求的文本内容，每次请求的 API key，每次请求的输入 token，输出 token 数量。 + ''' + if DISABLE_DATABASE: + return JSONResponse(content={"stats": {}}) + async with async_session() as session: + # 计算指定时间范围的开始时间 + start_time = datetime.now(timezone.utc) - timedelta(hours=hours) + + # 1. 每个渠道下面每个模型的成功率 + channel_model_stats = await session.execute( + select( + ChannelStat.provider, + ChannelStat.model, + func.count().label('total'), + func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') + ) + .where(ChannelStat.timestamp >= start_time) + .group_by(ChannelStat.provider, ChannelStat.model) + ) + channel_model_stats = channel_model_stats.fetchall() + + # 2. 每个渠道总的成功率 + channel_stats = await session.execute( + select( + ChannelStat.provider, + func.count().label('total'), + func.sum(case((ChannelStat.success == True, 1), else_=0)).label('success_count') + ) + .where(ChannelStat.timestamp >= start_time) + .group_by(ChannelStat.provider) + ) + channel_stats = channel_stats.fetchall() + + # 3. 每个模型在所有渠道总的请求次数 + model_stats = await session.execute( + select(RequestStat.model, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.model) + .order_by(desc('count')) + ) + model_stats = model_stats.fetchall() + + # 4. 每个端点的请求次数 + endpoint_stats = await session.execute( + select(RequestStat.endpoint, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.endpoint) + .order_by(desc('count')) + ) + endpoint_stats = endpoint_stats.fetchall() + + # 5. 每个ip请求的次数 + ip_stats = await session.execute( + select(RequestStat.client_ip, func.count().label('count')) + .where(RequestStat.timestamp >= start_time) + .group_by(RequestStat.client_ip) + .order_by(desc('count')) + ) + ip_stats = ip_stats.fetchall() + + # 处理统计数据并返回 + stats = { + "time_range": f"Last {hours} hours", + "channel_model_success_rates": [ + { + "provider": stat.provider, + "model": stat.model, + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0, + "total_requests": stat.total + } for stat in sorted(channel_model_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) + ], + "channel_success_rates": [ + { + "provider": stat.provider, + "success_rate": stat.success_count / stat.total if stat.total > 0 else 0, + "total_requests": stat.total + } for stat in sorted(channel_stats, key=lambda x: x.success_count / x.total if x.total > 0 else 0, reverse=True) + ], + "model_request_counts": [ + { + "model": stat.model, + "count": stat.count + } for stat in model_stats + ], + "endpoint_request_counts": [ + { + "endpoint": stat.endpoint, + "count": stat.count + } for stat in endpoint_stats + ], + "ip_request_counts": [ + { + "ip": stat.client_ip, + "count": stat.count + } for stat in ip_stats + ] + } + + return JSONResponse(content=stats) + + + +from fastapi import FastAPI, Request +from fastapi import Form as FastapiForm, HTTPException, Depends +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from fastapi.security import APIKeyHeader +from typing import Optional, List + +from xue import HTML, Head, Body, Div, xue_initialize, Script, Ul, Li +from xue.components import input, dropdown, sheet, form, button, checkbox, sidebar, chart +from xue.components.model_config_row import model_config_row +# import sys +# import os +# sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from components.provider_table import data_table + +from ruamel.yaml import YAML +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +frontend_router = APIRouter() + +API_KEY_NAME = "X-API-Key" +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) +async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_key_header)): + if not x_api_key: + x_api_key = request.cookies.get("x_api_key") or request.query_params.get("x_api_key") + # print(f"Cookie x_api_key: {request.cookies.get('x_api_key')}") # 添加此行 + # print(f"Query param x_api_key: {request.query_params.get('x_api_key')}") # 添加此行 + # print(f"Header x_api_key: {x_api_key}") # 添加此行 + # logger.info(f"x_api_key: {x_api_key} {x_api_key == 'your_admin_api_key'}") + + if not hasattr(app.state, 'config'): + await ensure_config(request, lambda: None) + + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + return x_api_key + else: + return None + +async def frontend_rate_limit_dependency(request: Request, x_api_key: str = Depends(get_api_key)): + token = x_api_key if x_api_key else None + + # 使用 IP 地址和 token（如果有）作为限制键 + client_ip = request.client.host + rate_limit_key = f"{client_ip}:{token}" if token else client_ip + + limits = [(100, 60)] + if await rate_limiter.is_rate_limited(rate_limit_key, limits): + raise HTTPException(status_code=429, detail="Too many requests") + +# def get_backend_router_api_list(): +# api_list = [] +# for route in frontend_router.routes: +# api_list.append({ +# "path": f"/api{route.path}", # 加上前缀 +# "method": route.methods, +# "name": route.name, +# "summary": route.summary +# }) +# return api_list + +# @app.get("/backend-router-api-list") +# async def backend_router_api_list(): +# return get_backend_router_api_list() + +xue_initialize(tailwind=True) + +data_table_columns = [ + # {"label": "Status", "value": "status", "sortable": True}, + {"label": "Provider", "value": "provider", "sortable": True}, + {"label": "Base url", "value": "base_url", "sortable": True}, + # {"label": "Engine", "value": "engine", "sortable": True}, + {"label": "Tools", "value": "tools", "sortable": True}, +] + +@frontend_router.get("/login", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def login_page(): + return HTML( + Head(title="登录"), + Body( + Div( + form.Form( + form.FormField("API Key", "x_api_key", type="password", placeholder="输入API密钥", required=True), + Div(id="error-message", class_="text-red-500 mt-2"), + Div( + button.button("提交", variant="primary", type="submit"), + class_="flex justify-end mt-4" + ), + hx_post="/verify-api-key", + hx_target="#error-message", + hx_swap="innerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-md" + ) + ) + ).render() + + +@frontend_router.post("/verify-api-key", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def verify_api_key(x_api_key: str = FastapiForm(...)): + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + response = JSONResponse(content={"success": True}) + response.headers["HX-Redirect"] = "/" # 添加这一行 + response.set_cookie( + key="x_api_key", + value=x_api_key, + httponly=True, + max_age=1800, # 30分钟 + secure=False, # 在开发环境中设置为False，生产环境中使用HTTPS时设置为True + samesite="lax" # 改为"lax"以允许重定向时携带cookie + ) + return response + else: + return Div("无效的API密钥", class_="text-red-500").render() + +# 添加侧边栏配置 +sidebar_items = [ + { + "icon": "layout-dashboard", + # "label": "仪表盘", + "label": "Dashboard", + "value": "dashboard", + "hx": {"get": "/dashboard", "target": "#main-content"} + }, + # { + # "icon": "settings", + # # "label": "设置", + # "label": "Settings", + # "value": "settings", + # "hx": {"get": "/settings", "target": "#main-content"} + # }, + { + "icon": "database", + # "label": "数据", + "label": "Data", + "value": "data", + "hx": {"get": "/data", "target": "#main-content"} + }, + # { + # "icon": "scroll-text", + # # "label": "日志", + # "label": "Logs", + # "value": "logs", + # "hx": {"get": "/logs", "target": "#main-content"} + # } +] + +@frontend_router.get("/", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def root(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = HTML( + Head( + Script(""" + document.addEventListener('DOMContentLoaded', function() { + const filterInput = document.getElementById('users-table-filter'); + filterInput.addEventListener('input', function() { + const filterValue = this.value; + htmx.ajax('GET', `/filter-table?filter=${filterValue}`, '#users-table'); + }); + }); + """), + title="uni-api" + ), + Body( + Div( + sidebar.Sidebar("zap", "uni-api", sidebar_items, is_collapsed=False, active_item="dashboard"), + Div( + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # sheet加载位置 + id="main-content", + class_="ml-[200px] p-6 transition-[margin] duration-200 ease-in-out" + ), + class_="flex" + ), + class_="container mx-auto", + id="body" + ) + ).render() + # print(result) + return result + +@frontend_router.get("/sidebar/toggle", response_class=HTMLResponse) +async def toggle_sidebar(is_collapsed: bool = False): + return sidebar.Sidebar( + "zap", + "uni-api", + sidebar_items, + is_collapsed=not is_collapsed, + active_item="dashboard" + ).render() + +@app.get("/sidebar/update/{active_item}", response_class=HTMLResponse) +async def update_sidebar(active_item: str): + return sidebar.Sidebar( + "zap", + "uni-api", + sidebar_items, + is_collapsed=False, + active_item=active_item + ).render() + +@frontend_router.get("/dashboard", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def data_page(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = Div( + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # sheet加载位置 + id="main-content", + class_="ml-[200px] p-6 transition-[margin] duration-200 ease-in-out" + ).render() + + return result + +@frontend_router.get("/data", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def data_page(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + if DISABLE_DATABASE: + return HTMLResponse("数据库已禁用") + + async with async_session() as session: + # 计算过去24小时的开始时间 + start_time = datetime.now(timezone.utc) - timedelta(hours=24) + + # 按小时统计每个模型的请求数据 + model_stats = await session.execute( + select( + func.strftime('%H', RequestStat.timestamp).label('hour'), + RequestStat.model, + func.count().label('count') + ) + .where(RequestStat.timestamp >= start_time) + .group_by('hour', RequestStat.model) + .order_by('hour') + ) + model_stats = model_stats.fetchall() + + # 获取所有唯一的模型名称 + models = list(set(stat.model for stat in model_stats)) + + # 生成24小时的数据点 + chart_data = [] + current_hour = datetime.now().hour + + for i in range(24): + # 计算小时标签(从当前小时往前推24小时) + hour = (current_hour - i) % 24 + hour_str = f"{hour:02d}" + + # 创建该小时的数据点 + data_point = {"label": hour_str} + + # 添加每个模型在该小时的请求数 + for model in models: + count = next( + (stat.count for stat in model_stats + if stat.hour == f"{hour:02d}" and stat.model == model), + 0 + ) + data_point[model] = count + + chart_data.append(data_point) + + # 反转数据点顺序使其按时间正序显示 + chart_data.reverse() + + # 为每个模型配置显示属性 + chart_config = { + model: { + "label": model, + "color": f"hsl({i * 360 / len(models)}, 70%, 50%)" # 为每个模型生成不同的颜色 + } + for i, model in enumerate(models) + } + + result = HTML( + Head(title="数据统计"), + Body( + Div( + # 堆叠柱状图 + Div( + "模型使用统计 (24小时) - 按小时统计", + class_="text-2xl font-bold mb-4" + ), + Div( + chart.chart( + chart_data, + chart_config, + stacked=True, + ), + class_="mb-8 h-[400px]" # 添加固定高度 + ), + id="main-content", + class_="container ml-[200px] mx-auto p-4" + ) + ) + ).render() + print(result) + + return result + +@frontend_router.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_columns_menu(menu_id: str, row_id: str): + columns = [ + { + "label": "Edit", + "value": "edit", + "hx-get": f"/edit-sheet/{row_id}", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + { + "label": "Duplicate", + "value": "duplicate", + "hx-post": f"/duplicate/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML" + }, + { + "label": "Delete", + "value": "delete", + "hx-delete": f"/delete/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML", + "hx-confirm": "Are you sure you want to delete this configuration?" + }, + ] + result = dropdown.dropdown_menu_content(menu_id, columns).render() + print(result) + return result + +@frontend_router.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_columns_menu(menu_id: str): + result = dropdown.dropdown_menu_content(menu_id, data_table_columns).render() + print(result) + return result + +@frontend_router.get("/filter-table", response_class=HTMLResponse) +async def filter_table(filter: str = ""): + filtered_data = [ + (i, provider) for i, provider in enumerate(app.state.config["providers"]) + if filter.lower() in str(provider["provider"]).lower() or + filter.lower() in str(provider["base_url"]).lower() or + filter.lower() in str(provider["tools"]).lower() + ] + return data_table(data_table_columns, [p for _, p in filtered_data], "users-table", with_filter=False, row_ids=[i for i, _ in filtered_data]).render() + +@frontend_router.post("/add-model", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def add_model(): + new_model_id = f"model{hash(str(time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +def render_api_keys(row_id, api_keys): + return Ul( + *[Li( + Div( + Div( + input.input( + type="text", + placeholder="Enter API key", + value=api_key, + name=f"api_key_{i}", + class_="flex-grow w-full" + ), + class_="flex-grow" + ), + button.button( + "Delete", + variant="outline", + type="button", + class_="ml-2", + hx_delete=f"/delete-api-key/{row_id}/{i}", + hx_target="#api-keys-container", + hx_swap="outerHTML" + ), + class_="flex items-center mb-2 w-full" + ) + ) for i, api_key in enumerate(api_keys)], + id="api-keys-container", + class_="space-y-2 w-full" + ) + +@frontend_router.get("/edit-sheet/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): + row_data = get_row_data(row_id) + print("row_data", row_data) + + model_list = [] + for index, model in enumerate(row_data["model"]): + if isinstance(model, str): + model_list.append(model_config_row(f"model{index}", model, "", True)) + if isinstance(model, dict): + # print("model", model, list(model.items())[0]) + key, value = list(model.items())[0] + model_list.append(model_config_row(f"model{index}", key, value, True)) + + # 处理多个 API keys + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + api_key_inputs = render_api_keys(row_id, api_keys) + + sheet_id = "edit-sheet" + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", value=row_data["provider"], placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", value=row_data["base_url"], placeholder="Enter base URL", required=True), + # form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + Div( + Div("API Keys", class_="text-lg font-semibold mb-2"), + api_key_inputs, + button.button( + "Add API Key", + class_="mt-2", + hx_post=f"/add-api-key/{row_id}", + hx_target="#api-keys-container", + hx_swap="outerHTML" + ), + class_="mb-4" + ), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + *model_list, + id="models-container", + class_="space-y-2 max-h-[40vh] overflow-y-auto" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=row_data["tools"], name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", value=row_data.get("notes", ""), placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), + class_="flex justify-end mt-4" + ), + hx_post=f"/submit/{row_id}", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ), + class_="max-h-[90vh] overflow-y-auto" + ) + + result = sheet.Sheet( + sheet_id, + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +@frontend_router.post("/add-api-key/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def add_api_key(row_id: str): + row_data = get_row_data(row_id) + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + api_keys.append("") # 添加一个空的API key + + api_key_inputs = render_api_keys(row_id, api_keys) + + return api_key_inputs.render() + +@frontend_router.delete("/delete-api-key/{row_id}/{index}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def delete_api_key(row_id: str, index: int): + row_data = get_row_data(row_id) + api_keys = row_data["api"] if isinstance(row_data["api"], list) else [row_data["api"]] + if len(api_keys) > 1: + del api_keys[index] + + api_key_inputs = render_api_keys(row_id, api_keys) + + return api_key_inputs.render() + +@frontend_router.get("/add-provider-sheet", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def get_add_provider_sheet(): + sheet_id = "add-provider-sheet" + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Add New Provider"), + sheet.SheetDescription("Enter details for the new provider.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div(id="models-container"), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), + class_="flex justify-end mt-4" + ), + hx_post="/submit/new", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + sheet_id, + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +def get_row_data(row_id): + index = int(row_id) + # print(app.state.config["providers"]) + return app.state.config["providers"][index] + +def update_row_data(row_id, updated_data): + print(row_id, updated_data) + index = int(row_id) + app.state.config["providers"][index] = updated_data + +@frontend_router.post("/submit/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def submit_form( + row_id: str, + request: Request, + provider: str = FastapiForm(...), + base_url: str = FastapiForm(...), + # api_key: Optional[str] = FastapiForm(None), + tools: Optional[str] = FastapiForm(None), + notes: Optional[str] = FastapiForm(None), + x_api_key: str = Depends(get_api_key) +): + form_data = await request.form() + + api_keys = [value for key, value in form_data.items() if key.startswith("api_key_") and value] + + # 收集模型数据 + models = [] + for key, value in form_data.items(): + if key.startswith("model_name_"): + model_id = key.split("_")[-1] + enabled = form_data.get(f"model_enabled_{model_id}") == "on" + rename = form_data.get(f"model_rename_{model_id}") + if value: + if rename: + models.append({value: rename}) + else: + models.append(value) + + updated_data = { + "provider": provider, + "base_url": base_url, + "api": api_keys[0] if len(api_keys) == 1 else api_keys, # 如果只有一个 API key，就不使用列表 + "model": models, + "tools": tools == "on", + "notes": notes, + } + + print("updated_data", updated_data) + + if row_id == "new": + # 添加新提供者 + app.state.config["providers"].append(updated_data) + else: + # 更新现有提供者 + update_row_data(row_id, updated_data) + + # 保存更新后的配置 + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) + + return await root() + +@frontend_router.post("/duplicate/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def duplicate_row(row_id: str): + index = int(row_id) + original_data = app.state.config["providers"][index] + new_data = original_data.copy() + new_data["provider"] += "-copy" + app.state.config["providers"].insert(index + 1, new_data) + + # 保存更新后的配置 + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) + + return await root() + +@frontend_router.delete("/delete/{row_id}", response_class=HTMLResponse, dependencies=[Depends(frontend_rate_limit_dependency)]) +async def delete_row(row_id: str): + index = int(row_id) + del app.state.config["providers"][index] + + # 保存更新后的配置 + if not DISABLE_DATABASE: + save_api_yaml(app.state.config) + + return await root() + +app.include_router(frontend_router, tags=["frontend"]) # async def on_fetch(request, env): # import asgi # return await asgi.fetch(app, request, env) +from fastapi.staticfiles import StaticFiles +# 添加静态文件挂载 +app.mount("/", StaticFiles(directory="./static", html=True), name="static") + if __name__ == '__main__': import uvicorn + import os + PORT = int(os.getenv("PORT", "8000")) uvicorn.run( "__main__:app", host="0.0.0.0", - port=8000, + port=PORT, reload=True, + reload_dirs=["./"], + reload_includes=["*.py", "api.yaml"], ws="none", # log_level="warning" ) \ No newline at end of file diff --git a/models.py b/models.py index b69d3bf5..751e4747 100644 --- a/models.py +++ b/models.py @@ -1,22 +1,27 @@ -from pydantic import BaseModel, Field -from typing import List, Dict, Optional, Union +from io import IOBase +from pydantic import BaseModel, Field, model_validator, ConfigDict +from typing import List, Dict, Optional, Union, Tuple, Literal, Any +from log_config import logger class FunctionParameter(BaseModel): type: str - properties: Dict[str, Dict[str, str]] - required: List[str] + properties: Dict[str, Dict[str, Any]] + required: List[str] = None -# 定义 Function 模型 class Function(BaseModel): name: str - description: str + description: str = Field(default=None) parameters: Optional[FunctionParameter] = Field(default=None, exclude=None) -# 定义 Tool 模型 class Tool(BaseModel): type: str function: Function + @classmethod + def parse_raw(cls, json_str: str) -> 'Tool': + """从JSON字符串解析Tool对象""" + return cls.model_validate_json(json_str) + class FunctionCall(BaseModel): name: str arguments: str @@ -51,7 +56,30 @@ class Message(BaseModel): class Config: extra = "allow" # 允许额外的字段 -class RequestModel(BaseModel): +class FunctionChoice(BaseModel): + name: str + +class ToolChoice(BaseModel): + type: str + function: Optional[FunctionChoice] = None + +class BaseRequest(BaseModel): + request_type: Optional[Literal["chat", "image", "audio", "moderation"]] = Field(default=None, exclude=True) + +import warnings +warnings.filterwarnings("ignore", category=UserWarning, message=".*shadows an attribute.*") + +class JsonSchema(BaseModel): + name: str + schema: Dict[str, Any] = Field(validation_alias='schema') + + model_config = ConfigDict(protected_namespaces=()) + +class ResponseFormat(BaseModel): + type: Literal["text", "json_object", "json_schema"] + json_schema: Optional[JsonSchema] = None + +class RequestModel(BaseRequest): model: str messages: List[Message] logprobs: Optional[bool] = None @@ -61,9 +89,132 @@ class RequestModel(BaseModel): temperature: Optional[float] = 0.5 top_p: Optional[float] = 1.0 max_tokens: Optional[int] = None + max_completion_tokens: Optional[int] = None presence_penalty: Optional[float] = 0.0 frequency_penalty: Optional[float] = 0.0 n: Optional[int] = 1 user: Optional[str] = None - tool_choice: Optional[str] = None - tools: Optional[List[Tool]] = None \ No newline at end of file + tool_choice: Optional[Union[str, ToolChoice]] = None + tools: Optional[List[Tool]] = None + response_format: Optional[ResponseFormat] = None # 新增字段 + + def get_last_text_message(self) -> Optional[str]: + for message in reversed(self.messages): + if message.content: + if isinstance(message.content, str): + return message.content + elif isinstance(message.content, list): + for item in reversed(message.content): + if item.type == "text" and item.text: + return item.text + return "" + + def model_dump(self, **kwargs): + data = super().model_dump(**kwargs) + + # 检查并处理 tools 字段 + if 'tools' in data and data['tools']: + for tool in data['tools']: + if 'function' in tool: + function_data = tool['function'] + # 如果 parameters 为空或没有 properties，则移除 + if 'parameters' in function_data and ( + function_data['parameters'] is None or + not function_data['parameters'].get('properties') + ): + function_data.pop('parameters', None) + + return data + +class ImageGenerationRequest(BaseRequest): + prompt: str + model: Optional[str] = "dall-e-3" + n: Optional[int] = 1 + response_format: Optional[str] = "url" + size: Optional[str] = "1024x1024" + stream: bool = False + +class EmbeddingRequest(BaseRequest): + input: Union[str, List[Union[str, int, List[int]]]] # 支持字符串或数组 + model: str + encoding_format: Optional[str] = "float" + dimensions: Optional[int] = None + user: Optional[str] = None + stream: bool = False + +class AudioTranscriptionRequest(BaseRequest): + file: Tuple[str, IOBase, str] + model: str + language: Optional[str] = None + prompt: Optional[str] = None + response_format: Optional[str] = None + temperature: Optional[float] = None + stream: bool = False + + class Config: + arbitrary_types_allowed = True + +class ModerationRequest(BaseRequest): + input: Union[str, List[str]] + model: Optional[str] = "text-moderation-latest" + stream: bool = False + +class TextToSpeechRequest(BaseRequest): + model: str + input: str + voice: str + response_format: Optional[str] = "mp3" + speed: Optional[float] = 1.0 + stream: Optional[bool] = False # Add this line + +class UnifiedRequest(BaseModel): + data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest, TextToSpeechRequest] + + @model_validator(mode='before') + @classmethod + def set_request_type(cls, values): + if isinstance(values, dict): + if "messages" in values: + values["data"] = RequestModel(**values) + values["data"].request_type = "chat" + elif "prompt" in values: + values["data"] = ImageGenerationRequest(**values) + values["data"].request_type = "image" + elif "file" in values: + values["data"] = AudioTranscriptionRequest(**values) + values["data"].request_type = "audio" + elif "tts" in values.get("model", ""): + logger.info(f"TextToSpeechRequest: {values}") + values["data"] = TextToSpeechRequest(**values) + values["data"].request_type = "tts" + elif "text-embedding" in values.get("model", ""): + values["data"] = EmbeddingRequest(**values) + values["data"].request_type = "embedding" + elif "input" in values: + values["data"] = ModerationRequest(**values) + values["data"].request_type = "moderation" + else: + raise ValueError("无法确定请求类型") + return values + +if __name__ == "__main__": + # 示例JSON字符串 + json_str = ''' + { + "type": "function", + "function": { + "name": "clock-time____getCurrentTime____standalone", + "description": "获取当前时间", + "parameters": { + "type": "object", + "properties": {} + } + } + } + ''' + + # 解析JSON字符串为Tool对象 + tool = Tool.parse_raw(json_str) + + # parameters 字段将被自动排除 + print(tool.model_dump(exclude_unset=True)) \ No newline at end of file diff --git a/request.py b/request.py index f1008aad..f9f4b07e 100644 --- a/request.py +++ b/request.py @@ -1,52 +1,164 @@ +import os +import re import json +import httpx +import base64 +import urllib.parse +from PIL import Image +import io + from models import RequestModel -from utils import c35s, c3s, c3o, c3h, gem, CircularList +from utils import c35s, c3s, c3o, c3h, gemini1, gemini2, BaseAPI, get_model_dict, provider_api_circular_list, safe_get, ThreadSafeCircularList + +def get_image_format(file_content): + try: + img = Image.open(io.BytesIO(file_content)) + return img.format.lower() + except: + return None + +def encode_image(image_path): + with open(image_path, "rb") as image_file: + file_content = image_file.read() + img_format = get_image_format(file_content) + if not img_format: + raise ValueError("无法识别的图片格式") + base64_encoded = base64.b64encode(file_content).decode('utf-8') + + if img_format == 'png': + return f"data:image/png;base64,{base64_encoded}" + elif img_format in ['jpg', 'jpeg']: + return f"data:image/jpeg;base64,{base64_encoded}" + else: + raise ValueError(f"不支持的图片格式: {img_format}") + +async def get_doc_from_url(url): + filename = urllib.parse.unquote(url.split("/")[-1]) + transport = httpx.AsyncHTTPTransport( + http2=True, + verify=False, + retries=1 + ) + async with httpx.AsyncClient(transport=transport) as client: + try: + response = await client.get( + url, + timeout=30.0 + ) + with open(filename, 'wb') as f: + f.write(response.content) + + except httpx.RequestError as e: + print(f"An error occurred while requesting {e.request.url!r}.") + + return filename + +async def get_encode_image(image_url): + filename = await get_doc_from_url(image_url) + image_path = os.getcwd() + "/" + filename + base64_image = encode_image(image_path) + os.remove(image_path) + return base64_image + +# from PIL import Image +# import io +# def validate_image(image_data, image_type): +# try: +# decoded_image = base64.b64decode(image_data) +# image = Image.open(io.BytesIO(decoded_image)) + +# # 检查图片格式是否与声明的类型匹配 +# # print("image.format", image.format) +# if image_type == "image/png" and image.format != "PNG": +# raise ValueError("Image is not a valid PNG") +# elif image_type == "image/jpeg" and image.format not in ["JPEG", "JPG"]: +# raise ValueError("Image is not a valid JPEG") + +# # 如果没有异常,则图片有效 +# return True +# except Exception as e: +# print(f"Image validation failed: {str(e)}") +# return False async def get_image_message(base64_image, engine = None): - if "gpt" == engine: + if base64_image.startswith("http"): + base64_image = await get_encode_image(base64_image) + colon_index = base64_image.index(":") + semicolon_index = base64_image.index(";") + image_type = base64_image[colon_index + 1:semicolon_index] + + if image_type == "image/webp": + # 将webp转换为png + + # 解码base64获取图片数据 + image_data = base64.b64decode(base64_image.split(",")[1]) + + # 使用PIL打开webp图片 + image = Image.open(io.BytesIO(image_data)) + + # 转换为PNG格式 + png_buffer = io.BytesIO() + image.save(png_buffer, format="PNG") + png_base64 = base64.b64encode(png_buffer.getvalue()).decode('utf-8') + + # 返回PNG格式的base64 + base64_image = f"data:image/png;base64,{png_base64}" + image_type = "image/png" + + if "gpt" == engine or "openrouter" == engine or "azure" == engine: return { "type": "image_url", "image_url": { "url": base64_image, } } - if "claude" == engine: + if "claude" == engine or "vertex-claude" == engine: + # if not validate_image(base64_image.split(",")[1], image_type): + # raise ValueError(f"Invalid image format. Expected {image_type}") return { "type": "image", "source": { "type": "base64", - "media_type": "image/jpeg", + "media_type": image_type, "data": base64_image.split(",")[1], } } - if "gemini" == engine: + if "gemini" == engine or "vertex-gemini" == engine: return { "inlineData": { - "mimeType": "image/jpeg", + "mimeType": image_type, "data": base64_image.split(",")[1], } } raise ValueError("Unknown engine") async def get_text_message(role, message, engine = None): - if "gpt" == engine or "claude" == engine or "openrouter" == engine: + if "gpt" == engine or "claude" == engine or "openrouter" == engine or "vertex-claude" == engine or "o1" == engine or "azure" == engine: return {"type": "text", "text": message} - if "gemini" == engine: + if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} + if engine == "cloudflare": + return message + if engine == "cohere": + return message raise ValueError("Unknown engine") async def get_gemini_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } - model = provider['model'][request.model] - if request.stream: - gemini_stream = "streamGenerateContent" + model_dict = get_model_dict(provider) + model = model_dict[request.model] + gemini_stream = "streamGenerateContent" url = provider['base_url'] - if url.endswith("v1beta"): - url = "https://generativelanguage.googleapis.com/v1beta/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api']) - if url.endswith("v1"): - url = "https://generativelanguage.googleapis.com/v1/models/{model}:{stream}?key={api_key}".format(model=model, stream=gemini_stream, api_key=provider['api']) + parsed_url = urllib.parse.urlparse(url) + # print("parsed_url", parsed_url) + if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): + api_version = parsed_url.path.split('/')[-1] # 获取 v1 或 v1beta + else: + api_version = "v1beta" + # https://generativelanguage.googleapis.com/v1beta/models/ + url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}/models/{model}:{gemini_stream}?key={await provider_api_circular_list[provider['provider']].next(model)}" messages = [] systemInstruction = None @@ -61,7 +173,7 @@ async def get_gemini_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -103,71 +215,119 @@ async def get_gemini_payload(request, engine, provider): elif msg.role != "system": messages.append({"role": msg.role, "parts": content}) elif msg.role == "system": + content[0]["text"] = re.sub(r"_+", "_", content[0]["text"]) systemInstruction = {"parts": content} + if "gemini-2.0-flash-exp" in model or "gemini-1.5" in model: + safety_settings = "OFF" + else: + safety_settings = "BLOCK_NONE" payload = { - "contents": messages, + "contents": messages or [{"role": "user", "parts": [{"text": "No messages"}]}], "safetySettings": [ { "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE" + "threshold": safety_settings }, { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE" + "threshold": safety_settings } ] } + if systemInstruction: - payload["systemInstruction"] = systemInstruction + if api_version == "v1beta": + payload["systemInstruction"] = systemInstruction + if api_version == "v1": + first_message = safe_get(payload, "contents", 0, "parts", 0, "text", default=None) + system_instruction = safe_get(systemInstruction, "parts", 0, "text", default=None) + if first_message and system_instruction: + payload["contents"][0]["parts"][0]["text"] = system_instruction + "\n" + first_message miss_fields = [ 'model', 'messages', 'stream', 'tool_choice', - 'temperature', - 'top_p', - 'max_tokens', 'presence_penalty', 'frequency_penalty', 'n', 'user', 'include_usage', 'logprobs', - 'top_logprobs' + 'top_logprobs', + 'response_format' ] + generation_config = {} for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: if field == "tools": - payload.update({ - "tools": [{ - "function_declarations": [tool["function"] for tool in value] - }], - "tool_config": { - "function_calling_config": { - "mode": "AUTO" + # 处理每个工具的 function 定义 + processed_tools = [] + for tool in value: + function_def = tool["function"] + # 处理 parameters.properties 中的 default 字段 + if safe_get(function_def, "parameters", "properties", default=None): + for prop_value in function_def["parameters"]["properties"].values(): + if "default" in prop_value: + # 将 default 值添加到 description 中 + default_value = prop_value["default"] + description = prop_value.get("description", "") + prop_value["description"] = f"{description}\nDefault: {default_value}" + # 删除 default 字段 + del prop_value["default"] + if function_def["name"] != "googleSearch" and function_def["name"] != "googleSearch": + processed_tools.append({"function": function_def}) + + if processed_tools: + payload.update({ + "tools": [{ + "function_declarations": [tool["function"] for tool in processed_tools] + }], + "tool_config": { + "function_calling_config": { + "mode": "AUTO" + } } - } - }) + }) + elif field == "temperature": + generation_config["temperature"] = value + elif field == "max_tokens": + generation_config["maxOutputTokens"] = value + elif field == "top_p": + generation_config["topP"] = value else: payload[field] = value + if generation_config: + payload["generationConfig"] = generation_config + if "maxOutputTokens" not in generation_config: + payload["generationConfig"]["maxOutputTokens"] = 8192 + + if request.model.endswith("-search"): + if "tools" not in payload: + payload["tools"] = [{ + "googleSearch": {} + }] + else: + payload["tools"].append({ + "googleSearch": {} + }) + return url, headers, payload import time -import httpx -import base64 from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.serialization import load_pem_private_key @@ -232,11 +392,19 @@ async def get_vertex_gemini_payload(request, engine, provider): if provider.get("project_id"): project_id = provider.get("project_id") - if request.stream: - gemini_stream = "streamGenerateContent" - model = provider['model'][request.model] - location = gem - url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) + gemini_stream = "streamGenerateContent" + model_dict = get_model_dict(provider) + model = model_dict[request.model] + search_tool = None + + if "gemini-2.0" in model or "gemini-exp" in model: + location = gemini2 + search_tool = {"googleSearch": {}} + else: + location = gemini1 + search_tool = {"googleSearchRetrieval": {}} + + url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/google/models/{MODEL_ID}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL_ID=model, stream=gemini_stream) messages = [] systemInstruction = None @@ -251,7 +419,7 @@ async def get_vertex_gemini_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -316,12 +484,6 @@ async def get_vertex_gemini_payload(request, engine, provider): # "threshold": "BLOCK_NONE" # } # ] - "generationConfig": { - "temperature": 0.5, - "max_output_tokens": 8192, - "top_k": 40, - "top_p": 0.95 - }, } if systemInstruction: payload["system_instruction"] = systemInstruction @@ -331,9 +493,6 @@ async def get_vertex_gemini_payload(request, engine, provider): 'messages', 'stream', 'tool_choice', - 'temperature', - 'top_p', - 'max_tokens', 'presence_penalty', 'frequency_penalty', 'n', @@ -342,6 +501,7 @@ async def get_vertex_gemini_payload(request, engine, provider): 'logprobs', 'top_logprobs' ] + generation_config = {} for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: @@ -356,9 +516,26 @@ async def get_vertex_gemini_payload(request, engine, provider): } } }) + elif field == "temperature": + generation_config["temperature"] = value + elif field == "max_tokens": + generation_config["max_output_tokens"] = value + elif field == "top_p": + generation_config["top_p"] = value else: payload[field] = value + if generation_config: + payload["generationConfig"] = generation_config + if "max_output_tokens" not in generation_config: + payload["generationConfig"]["max_output_tokens"] = 8192 + + if request.model.endswith("-search"): + if "tools" not in payload: + payload["tools"] = [search_tool] + else: + payload["tools"].append(search_tool) + return url, headers, payload async def get_vertex_claude_payload(request, engine, provider): @@ -371,7 +548,8 @@ async def get_vertex_claude_payload(request, engine, provider): if provider.get("project_id"): project_id = provider.get("project_id") - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] if "claude-3-5-sonnet" in model: location = c35s elif "claude-3-opus" in model: @@ -381,45 +559,58 @@ async def get_vertex_claude_payload(request, engine, provider): elif "claude-3-haiku" in model: location = c3h - if request.stream: - claude_stream = "streamRawPredict" - url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/anthropic/models/{MODEL}:{stream}".format(LOCATION=location.next(), PROJECT_ID=project_id, MODEL=model, stream=claude_stream) + claude_stream = "streamRawPredict" + url = "https://{LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/anthropic/models/{MODEL}:{stream}".format(LOCATION=await location.next(), PROJECT_ID=project_id, MODEL=model, stream=claude_stream) messages = [] system_prompt = None + tool_id = None for msg in request.messages: - tool_calls = None tool_call_id = None + tool_calls = None if isinstance(msg.content, list): content = [] for item in msg.content: if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: content = msg.content tool_calls = msg.tool_calls + tool_id = tool_calls[0].id if tool_calls else None or tool_id tool_call_id = msg.tool_call_id if tool_calls: tool_calls_list = [] - for tool_call in tool_calls: - tool_calls_list.append({ - "type": "tool_use", - "id": tool_call.id, - "name": tool_call.function.name, - "input": json.loads(tool_call.function.arguments), - }) - messages.append({"role": msg.role, "content": tool_calls_list}) + tool_call = tool_calls[0] + tool_calls_list.append({ + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": json.loads(tool_call.function.arguments), + }) + messages.append({"role": msg.role, "content": tool_calls_list}) elif tool_call_id: messages.append({"role": "user", "content": [{ "type": "tool_result", - "tool_use_id": tool_call.id, + "tool_use_id": tool_id, "content": content }]}) + elif msg.role == "function": + messages.append({"role": "assistant", "content": [{ + "type": "tool_use", + "id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "name": msg.name, + "input": {"prompt": "..."} + }]}) + messages.append({"role": "user", "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "content": msg.content + }]}) elif msg.role != "system": messages.append({"role": msg.role, "content": content}) elif msg.role == "system": @@ -443,19 +634,17 @@ async def get_vertex_claude_payload(request, engine, provider): else: message_index = message_index + 1 - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "anthropic_version": "vertex-2023-10-16", "messages": messages, "system": system_prompt or "You are Claude, a large language model trained by Anthropic.", + "max_tokens": 8192 if "claude-3-5-sonnet" in model else 4096, } - # 檢查是否需要添加 max_tokens - if 'max_tokens' not in payload: - if "claude-3-5-sonnet" in model: - payload['max_tokens'] = 8192 - elif "claude-3" in model: # 處理其他 Claude 3 模型 - payload['max_tokens'] = 4096 + if request.max_tokens: + payload["max_tokens"] = int(request.max_tokens) miss_fields = [ 'model', @@ -478,9 +667,21 @@ async def get_vertex_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - payload["tool_choice"] = { - "type": "auto" - } + if isinstance(payload["tool_choice"], dict): + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } + if isinstance(payload["tool_choice"], str): + if payload["tool_choice"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"] == "none": + payload["tool_choice"] = { + "type": "any" + } if provider.get("tools") == False: payload.pop("tools", None) @@ -492,8 +693,14 @@ async def get_gpt_payload(request, engine, provider): headers = { 'Content-Type': 'application/json', } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api']}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + elif provider['provider'].startswith("sk-"): + headers['Authorization'] = f"Bearer {provider['provider']}" + url = provider['base_url'] messages = [] @@ -506,7 +713,7 @@ async def get_gpt_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True) and "o1-mini" not in model: image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -525,13 +732,18 @@ async def get_gpt_payload(request, engine, provider): "arguments": tool_call.function.arguments } }) - messages.append({"role": msg.role, "tool_calls": tool_calls_list}) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_calls": tool_calls_list}) elif tool_call_id: - messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) else: messages.append({"role": msg.role, "content": content}) - model = provider['model'][request.model] + if ("o1-mini" in model or "o1-preview" in model) and len(messages) > 1 and messages[0]["role"] == "system": + system_msg = messages.pop(0) + messages[0]["content"] = system_msg["content"] + messages[0]["content"] + payload = { "model": model, "messages": messages, @@ -539,14 +751,140 @@ async def get_gpt_payload(request, engine, provider): miss_fields = [ 'model', - 'messages' + 'messages', ] for field, value in request.model_dump(exclude_unset=True).items(): if field not in miss_fields and value is not None: - payload[field] = value + if field == "max_tokens" and ("o1" in model or "o3" in model): + payload["max_completion_tokens"] = value + else: + payload[field] = value - if provider.get("tools") == False: + if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model or "grok" in model: + payload.pop("tools", None) + payload.pop("tool_choice", None) + if "o1" in model and "models.inference.ai.azure.com" in url: + payload["stream"] = False + # request.stream = False + payload.pop("stream_options", None) + + if "o3-mini" in model: + if request.model.endswith("high"): + payload["reasoning_effort"] = "high" + elif request.model.endswith("low"): + payload["reasoning_effort"] = "low" + else: + payload["reasoning_effort"] = "medium" + + if "o3-mini" in model or "o1" in model: + if "temperature" in payload: + payload.pop("temperature") + + if request.model.endswith("-search") and "gemini" in request.model: + if "tools" not in payload: + payload["tools"] = [{ + "type": "function", + "function": { + "name": "googleSearch", + "description": "googleSearch" + } + }] + else: + if not any(tool["function"]["name"] == "googleSearch" for tool in payload["tools"]): + payload["tools"].append({ + "type": "function", + "function": { + "name": "googleSearch", + "description": "googleSearch" + } + }) + + return url, headers, payload + +def build_azure_endpoint(base_url, deployment_id, function="chat/completions", api_version="2024-10-21"): + # 移除base_url末尾的斜杠(如果有) + base_url = base_url.rstrip('/') + + # 构建路径 + path = f"/openai/deployments/{deployment_id}/{function}" + + # 使用urljoin拼接base_url和path + full_url = urllib.parse.urljoin(base_url, path) + + # 添加api-version查询参数 + final_url = f"{full_url}?api-version={api_version}" + + return final_url + +async def get_azure_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json', + } + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers['api-key'] = f"{provider['api']}" + + url = build_azure_endpoint( + base_url=provider['base_url'], + deployment_id=model, + ) + + messages = [] + for msg in request.messages: + tool_calls = None + tool_call_id = None + if isinstance(msg.content, list): + content = [] + for item in msg.content: + if item.type == "text": + text_message = await get_text_message(msg.role, item.text, engine) + content.append(text_message) + elif item.type == "image_url" and provider.get("image", True) and "o1-mini" not in model: + image_message = await get_image_message(item.image_url.url, engine) + content.append(image_message) + else: + content = msg.content + tool_calls = msg.tool_calls + tool_call_id = msg.tool_call_id + + if tool_calls: + tool_calls_list = [] + for tool_call in tool_calls: + tool_calls_list.append({ + "id": tool_call.id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments + } + }) + if provider.get("tools"): + messages.append({"role": msg.role, "tool_calls": tool_calls_list}) + elif tool_call_id: + if provider.get("tools"): + messages.append({"role": msg.role, "tool_call_id": tool_call_id, "content": content}) + else: + messages.append({"role": msg.role, "content": content}) + + payload = { + "model": model, + "messages": messages, + } + + miss_fields = [ + 'model', + 'messages', + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + if field == "max_tokens" and "o1" in model: + payload["max_completion_tokens"] = value + else: + payload[field] = value + + if provider.get("tools") == False or "o1" in model or "chatgpt-4o-latest" in model or "grok" in model: payload.pop("tools", None) payload.pop("tool_choice", None) @@ -556,8 +894,13 @@ async def get_openrouter_payload(request, engine, provider): headers = { 'Content-Type': 'application/json' } + model_dict = get_model_dict(provider) + model = model_dict[request.model] if provider.get("api"): - headers['Authorization'] = f"Bearer {provider['api']}" + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + elif provider['provider'].startswith("sk-"): + headers['Authorization'] = f"Bearer {provider['provider']}" url = provider['base_url'] @@ -570,7 +913,7 @@ async def get_openrouter_payload(request, engine, provider): if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: @@ -585,11 +928,10 @@ async def get_openrouter_payload(request, engine, provider): if item["type"] == "text": messages.append({"role": msg.role, "content": item["text"]}) elif item["type"] == "image_url": - messages.append({"role": msg.role, "content": item["url"]}) + messages.append({"role": msg.role, "content": [await get_image_message(item["image_url"]["url"], engine)]}) else: messages.append({"role": msg.role, "content": content}) - model = provider['model'][request.model] payload = { "model": model, "messages": messages, @@ -618,6 +960,123 @@ async def get_openrouter_payload(request, engine, provider): return url, headers, payload +async def get_cohere_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + model_dict = get_model_dict(provider) + model = model_dict[request.model] + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + url = provider['base_url'] + + role_map = { + "user": "USER", + "assistant" : "CHATBOT", + "system": "SYSTEM" + } + + messages = [] + for msg in request.messages: + if isinstance(msg.content, list): + content = [] + for item in msg.content: + if item.type == "text": + text_message = await get_text_message(msg.role, item.text, engine) + content.append(text_message) + else: + content = msg.content + + if isinstance(content, list): + for item in content: + if item["type"] == "text": + messages.append({"role": role_map[msg.role], "message": item["text"]}) + else: + messages.append({"role": role_map[msg.role], "message": content}) + + chat_history = messages[:-1] + query = messages[-1].get("message") + payload = { + "model": model, + "message": query, + } + + if chat_history: + payload["chat_history"] = chat_history + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + +async def get_cloudflare_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + model_dict = get_model_dict(provider) + model = model_dict[request.model] + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + url = "https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model_id}".format(cf_account_id=provider['cf_account_id'], cf_model_id=model) + + msg = request.messages[-1] + messages = [] + content = None + if isinstance(msg.content, list): + for item in msg.content: + if item.type == "text": + content = await get_text_message(msg.role, item.text, engine) + else: + content = msg.content + name = msg.name + + payload = { + "prompt": content, + } + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + async def gpt2claude_tools_json(json_dict): import copy json_dict = copy.deepcopy(json_dict) @@ -638,10 +1097,11 @@ async def gpt2claude_tools_json(json_dict): return json_dict async def get_claude_payload(request, engine, provider): - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] headers = { "content-type": "application/json", - "x-api-key": f"{provider['api']}", + "x-api-key": f"{await provider_api_circular_list[provider['provider']].next(model)}", "anthropic-version": "2023-06-01", "anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15" if "claude-3-5-sonnet" in model else "tools-2024-05-16", } @@ -649,39 +1109,53 @@ async def get_claude_payload(request, engine, provider): messages = [] system_prompt = None + tool_id = None for msg in request.messages: - tool_calls = None tool_call_id = None + tool_calls = None if isinstance(msg.content, list): content = [] for item in msg.content: if item.type == "text": text_message = await get_text_message(msg.role, item.text, engine) content.append(text_message) - elif item.type == "image_url": + elif item.type == "image_url" and provider.get("image", True): image_message = await get_image_message(item.image_url.url, engine) content.append(image_message) else: content = msg.content tool_calls = msg.tool_calls + tool_id = tool_calls[0].id if tool_calls else None or tool_id tool_call_id = msg.tool_call_id if tool_calls: tool_calls_list = [] - for tool_call in tool_calls: - tool_calls_list.append({ - "type": "tool_use", - "id": tool_call.id, - "name": tool_call.function.name, - "input": json.loads(tool_call.function.arguments), - }) - messages.append({"role": msg.role, "content": tool_calls_list}) + tool_call = tool_calls[0] + tool_calls_list.append({ + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": json.loads(tool_call.function.arguments), + }) + messages.append({"role": msg.role, "content": tool_calls_list}) elif tool_call_id: messages.append({"role": "user", "content": [{ "type": "tool_result", - "tool_use_id": tool_call.id, + "tool_use_id": tool_id, "content": content }]}) + elif msg.role == "function": + messages.append({"role": "assistant", "content": [{ + "type": "tool_use", + "id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "name": msg.name, + "input": {"prompt": "..."} + }]}) + messages.append({"role": "user", "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_017r5miPMV6PGSNKmhvHPic4", + "content": msg.content + }]}) elif msg.role != "system": messages.append({"role": msg.role, "content": content}) elif msg.role == "system": @@ -705,13 +1179,18 @@ async def get_claude_payload(request, engine, provider): else: message_index = message_index + 1 - model = provider['model'][request.model] + model_dict = get_model_dict(provider) + model = model_dict[request.model] payload = { "model": model, "messages": messages, "system": system_prompt or "You are Claude, a large language model trained by Anthropic.", + "max_tokens": 8192 if "claude-3-5-sonnet" in model else 4096, } + if request.max_tokens: + payload["max_tokens"] = int(request.max_tokens) + miss_fields = [ 'model', 'messages', @@ -734,9 +1213,21 @@ async def get_claude_payload(request, engine, provider): tools.append(json_tool) payload["tools"] = tools if "tool_choice" in payload: - payload["tool_choice"] = { - "type": "auto" - } + if isinstance(payload["tool_choice"], dict): + if payload["tool_choice"]["type"] == "function": + payload["tool_choice"] = { + "type": "tool", + "name": payload["tool_choice"]["function"]["name"] + } + if isinstance(payload["tool_choice"], str): + if payload["tool_choice"] == "auto": + payload["tool_choice"] = { + "type": "auto" + } + if payload["tool_choice"] == "none": + payload["tool_choice"] = { + "type": "any" + } if provider.get("tools") == False: payload.pop("tools", None) @@ -746,18 +1237,158 @@ async def get_claude_payload(request, engine, provider): return url, headers, payload +async def get_dalle_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + url = provider['base_url'] + url = BaseAPI(url).image_url + + payload = { + "model": model, + "prompt": request.prompt, + "n": request.n, + "response_format": request.response_format, + "size": request.size + } + + return url, headers, payload + +async def get_whisper_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + # "Content-Type": "multipart/form-data", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + url = provider['base_url'] + url = BaseAPI(url).audio_transcriptions + + payload = { + "model": model, + "file": request.file, + } + + if request.prompt: + payload["prompt"] = request.prompt + if request.response_format: + payload["response_format"] = request.response_format + if request.temperature: + payload["temperature"] = request.temperature + if request.language: + payload["language"] = request.language + + return url, headers, payload + +async def get_moderation_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + url = provider['base_url'] + url = BaseAPI(url).moderations + + payload = { + "input": request.input, + } + + return url, headers, payload + +async def get_embedding_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + "Content-Type": "application/json", + } + url = provider['base_url'] + is_azure = url.endswith(".azure.com") + if provider.get("api"): + if is_azure: + headers['api-key'] = f"{await provider_api_circular_list[provider['provider']].next(model)}" + else: + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + + if is_azure: + url = build_azure_endpoint(url, model, function="embeddings") + else: + url = BaseAPI(url).embeddings + + payload = { + "input": request.input, + "model": model, + } + + if request.encoding_format: + if url.startswith("https://api.jina.ai"): + payload["embedding_type"] = request.encoding_format + else: + payload["encoding_format"] = request.encoding_format + + return url, headers, payload + +async def get_tts_payload(request, engine, provider): + model_dict = get_model_dict(provider) + model = model_dict[request.model] + headers = { + "Content-Type": "application/json", + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}" + url = provider['base_url'] + url = BaseAPI(url).audio_speech + + payload = { + "model": model, + "input": request.input, + "voice": request.voice, + } + + if request.response_format: + payload["response_format"] = request.response_format + if request.speed: + payload["speed"] = request.speed + if request.stream is not None: + payload["stream"] = request.stream + + return url, headers, payload + + async def get_payload(request: RequestModel, engine, provider): if engine == "gemini": return await get_gemini_payload(request, engine, provider) - elif engine == "vertex" and "gemini" in provider['model'][request.model]: + elif engine == "vertex-gemini": return await get_vertex_gemini_payload(request, engine, provider) - elif engine == "vertex" and "claude" in provider['model'][request.model]: + elif engine == "vertex-claude": return await get_vertex_claude_payload(request, engine, provider) + elif engine == "azure": + return await get_azure_payload(request, engine, provider) elif engine == "claude": return await get_claude_payload(request, engine, provider) elif engine == "gpt": return await get_gpt_payload(request, engine, provider) elif engine == "openrouter": return await get_openrouter_payload(request, engine, provider) + elif engine == "cloudflare": + return await get_cloudflare_payload(request, engine, provider) + elif engine == "cohere": + return await get_cohere_payload(request, engine, provider) + elif engine == "dalle": + return await get_dalle_payload(request, engine, provider) + elif engine == "whisper": + return await get_whisper_payload(request, engine, provider) + elif engine == "tts": + return await get_tts_payload(request, engine, provider) + elif engine == "moderation": + return await get_moderation_payload(request, engine, provider) + elif engine == "embedding": + return await get_embedding_payload(request, engine, provider) else: - raise ValueError("Unknown payload") \ No newline at end of file + raise ValueError("Unknown payload") diff --git a/requirements.txt b/requirements.txt index 40317098..b0717ad8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,15 @@ -pyyaml +xue pytest +pillow uvicorn fastapi +aiofiles +greenlet +aiosqlite +sqlalchemy +watchfiles +ruamel.yaml httpx[http2] -cryptography \ No newline at end of file +httpx-socks==0.9.2 +cryptography==43.0.3 +python-multipart \ No newline at end of file diff --git a/response.py b/response.py index bf3c3e35..379741af 100644 --- a/response.py +++ b/response.py @@ -1,70 +1,70 @@ import json import httpx +import random +import string from datetime import datetime from log_config import logger +from utils import safe_get, generate_sse_response, generate_no_stream_response, end_of_line -async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, tokens_use=None, total_tokens=None): - sample_data = { - "id": "chatcmpl-9ijPeRHa0wtyA2G8wq5z8FC3wGMzc", - "object": "chat.completion.chunk", - "created": timestamp, - "model": model, - "system_fingerprint": "fp_d576307f90", - "choices": [ - { - "index": 0, - "delta": {"content": content}, - "logprobs": None, - "finish_reason": None - } - ], - "usage": None - } - if function_call_content: - sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]} - if tools_id and function_call_name: - sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"id": tools_id,"type":"function","function":{"name": function_call_name, "arguments":""}}]} - # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]} - if role: - sample_data["choices"][0]["delta"] = {"role": role, "content": ""} - json_data = json.dumps(sample_data, ensure_ascii=False) - - # 构建SSE响应 - sse_response = f"data: {json_data}\n\n" - - return sse_response +async def check_response(response, error_log): + if response and not (200 <= response.status_code < 300): + error_message = await response.aread() + error_str = error_message.decode('utf-8', errors='replace') + try: + error_json = json.loads(error_str) + except json.JSONDecodeError: + error_json = error_str + return {"error": f"{error_log} HTTP Error", "status_code": response.status_code, "details": error_json} + return None async def fetch_gemini_response_stream(client, url, headers, payload, model): - timestamp = datetime.timestamp(datetime.now()) + timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_gemini_response_stream") + if error_message: + yield error_message + return buffer = "" revicing_function_call = False function_full_response = "{" need_function_call = False + is_finish = False + # line_index = 0 + # last_text_line = 0 + # if "thinking" in model: + # is_thinking = True + # else: + # is_thinking = False async for chunk in response.aiter_text(): buffer += chunk + while "\n" in buffer: line, buffer = buffer.split("\n", 1) + # line_index += 1 + if line and '\"finishReason\": \"' in line: + is_finish = True + break # print(line) if line and '\"text\": \"' in line: try: json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) - sse_string = await generate_sse_response(timestamp, model, content) + # content = content.replace("\n", "\n\n") + # if last_text_line == 0 and is_thinking: + # content = "> " + content.lstrip() + # if is_thinking: + # content = content.replace("\n", "\n> ") + # if last_text_line == line_index - 3: + # is_thinking = False + # content = "\n\n\n" + content.lstrip() + sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") + # last_text_line = line_index if line and ('\"functionCall\": {' in line or revicing_function_call): revicing_function_call = True @@ -75,6 +75,9 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response += line + if is_finish: + break + if need_function_call: function_call = json.loads(function_full_response) function_call_name = function_call["functionCall"]["name"] @@ -83,18 +86,16 @@ async def fetch_gemini_response_stream(client, url, headers, payload, model): function_full_response = json.dumps(function_call["functionCall"]["args"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id="chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV", function_call_name=None, function_call_content=function_full_response) yield sse_string + yield "data: [DONE]" + end_of_line async def fetch_vertex_claude_response_stream(client, url, headers, payload, model): - timestamp = datetime.timestamp(datetime.now()) + timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} + error_message = await check_response(response, "fetch_vertex_claude_response_stream") + if error_message: + yield error_message + return + buffer = "" revicing_function_call = False function_full_response = "{" @@ -103,13 +104,13 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod buffer += chunk while "\n" in buffer: line, buffer = buffer.split("\n", 1) - logger.info(f"{line}") + # logger.info(f"{line}") if line and '\"text\": \"' in line: try: json_data = json.loads( "{" + line + "}") content = json_data.get('text', '') content = "\n".join(content.split("\\n")) - sse_string = await generate_sse_response(timestamp, model, content) + sse_string = await generate_sse_response(timestamp, model, content=content) yield sse_string except json.JSONDecodeError: logger.error(f"无法解析JSON: {line}") @@ -132,68 +133,125 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod function_full_response = json.dumps(function_call["input"]) sse_string = await generate_sse_response(timestamp, model, content=None, tools_id=function_call_id, function_call_name=None, function_call_content=function_full_response) yield sse_string + yield "data: [DONE]" + end_of_line -async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): - redirect_count = 0 - while redirect_count < max_redirects: - # logger.info(f"fetch_gpt_response_stream: {url}") - async with client.stream('POST', url, headers=headers, json=payload) as response: - if response.status_code != 200: - error_message = await response.aread() - error_str = error_message.decode('utf-8', errors='replace') - try: - error_json = json.loads(error_str) - except json.JSONDecodeError: - error_json = error_str - yield {"error": f"fetch_gpt_response_stream HTTP Error {response.status_code}", "details": error_json} - return - - buffer = "" - try: - async for chunk in response.aiter_text(): - # logger.info(f"chunk: {repr(chunk)}") - buffer += chunk - if chunk.startswith("= t: + hourly_counts[t] += 1 + break + + # 准备绘图数据 + hours = [t.strftime('%Y-%m-%d %H:00') for t in time_range] + counts = [hourly_counts[t] for t in time_range] + + # 创建柱状图 + plt.figure(figsize=(15, 6)) + plt.bar(hours, counts) + plt.title(f'{key} 端点请求量 (过去24小时)') + plt.xlabel('时间') + plt.ylabel('请求数') + plt.xticks(rotation=45, ha='right') + plt.tight_layout() + + # 保存图片 + plt.savefig(f'{key.replace("/", "")}.png') + +def create_pie_chart(model_counts): + models = list(model_counts.keys()) + counts = list(model_counts.values()) + + # 设置颜色和排列顺序 + colors = plt.cm.Set3(np.linspace(0, 1, len(models))) + sorted_data = sorted(zip(counts, models, colors), reverse=True) + counts, models, colors = zip(*sorted_data) + + # 创建饼图 + fig, ax = plt.subplots(figsize=(16, 10)) + wedges, _ = ax.pie(counts, colors=colors, startangle=90, wedgeprops=dict(width=0.5)) + + # 添加圆环效果 + centre_circle = plt.Circle((0, 0), 0.35, fc='white') + fig.gca().add_artist(centre_circle) + + # 计算总数 + total = sum(counts) + + # 准备标注 + bbox_props = dict(boxstyle="round,pad=0.3", fc="w", ec="k", lw=0.72) + kw = dict(xycoords='data', textcoords='data', arrowprops=dict(arrowstyle="-"), bbox=bbox_props, zorder=0) + + left_labels = [] + right_labels = [] + + for i, p in enumerate(wedges): + ang = (p.theta2 - p.theta1) / 2. + p.theta1 + y = np.sin(np.deg2rad(ang)) + x = np.cos(np.deg2rad(ang)) + + percentage = counts[i] / total * 100 + label = f"{models[i]}: {percentage:.1f}%" + + if x > 0: + right_labels.append((x, y, label)) + else: + left_labels.append((x, y, label)) + + # 绘制左侧标注 + for i, (x, y, label) in enumerate(left_labels): + ax.annotate(label, xy=(x, y), xytext=(-1.2, 0.9 - i * 0.15), **kw) + + # 绘制右侧标注 + for i, (x, y, label) in enumerate(right_labels): + ax.annotate(label, xy=(x, y), xytext=(1.2, 0.9 - i * 0.15), **kw) + + plt.title("各模型使用次数对比", size=16) + ax.set_xlim(-1.5, 1.5) + ax.set_ylim(-1.2, 1.2) + ax.axis('off') + plt.tight_layout() + plt.savefig('model_usage_pie_chart.png', bbox_inches='tight', pad_inches=0.5) + +if __name__ == '__main__': + model_counts = { + "model_counts": { + "claude-3-5-sonnet": 94, + "o1-preview": 71, + "gpt-4o": 512, + "gpt-4o-mini": 5, + "gemini-1.5-pro": 5, + "deepseek-chat": 7, + "grok-2-mini": 1, + "grok-2": 9, + "o1-mini": 8 + } + } + # create_pic(request_arrivals, 'POST /v1/chat/completions') + + create_pie_chart(model_counts["model_counts"]) diff --git a/test/test_nostream.py b/test/test_nostream.py new file mode 100644 index 00000000..febb248f --- /dev/null +++ b/test/test_nostream.py @@ -0,0 +1,120 @@ +import requests +import base64 +import json +import os +from datetime import datetime + +# 設置API密鑰和自定義base URL +API_KEY = '' +BASE_URL = 'http://localhost:8000/v1' +SAVE_DIR = 'safe_output' # 保存 JSON 輸出的目錄 + +def ensure_save_directory(): + if not os.path.exists(SAVE_DIR): + os.makedirs(SAVE_DIR) + +def image_to_base64(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + +def get_model_response(image_base64): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {API_KEY}" + } + + tools = [ + { + "type": "function", + "function": { + "name": "extract_underlined_text", + "description": "從圖片中提取紅色下劃線的文字", + "parameters": { + "type": "object", + "properties": { + "underlined_text": { + "type": "array", + "items": {"type": "string"}, + "description": "紅色下劃線的文字列表" + } + }, + "required": ["underlined_text"] + } + } + } + ] + + payload = { + "model": "claude-3-5-sonnet", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "請仔細分析圖片，並提取所有使用紅色筆在單字、單詞或句子下方畫有橫線的文字。只提取有紅色下劃線的文字，忽略其他未標記的文字。將結果以 JSON 格式輸出，格式為 {\"underlined_text\": [\"文字1\", \"文字2\", ...]}。" + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64}" + } + } + ] + } + ], + # "stream": True, + "tools": tools, + "tool_choice": {"type": "function", "function": {"name": "extract_underlined_text"}}, + "max_tokens": 1000 + } + + try: + response = requests.post(f"{BASE_URL}/chat/completions", headers=headers, json=payload, timeout=30) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + return f"Error: {e}" + +def save_json_output(data): + ensure_save_directory() + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + filename = f"{SAVE_DIR}/output_{timestamp}.json" + with open(filename, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + return filename + +def main(image_path): + image_base64 = image_to_base64(image_path) + + response = get_model_response(image_base64) + + print("模型回應:") + print(json.dumps(response, indent=2, ensure_ascii=False)) + + if isinstance(response, str) and response.startswith("Error"): + print(response) + return + + if 'choices' in response and response['choices']: + message = response['choices'][0]['message'] + if 'tool_calls' in message: + tool_call = message['tool_calls'][0] + if tool_call['function']['name'] == 'extract_underlined_text': + function_args = json.loads(tool_call['function']['arguments']) + print("\n提取的紅色下劃線文字:") + print(json.dumps(function_args, indent=2, ensure_ascii=False)) + + # 保存 JSON 輸出 + saved_file = save_json_output(function_args) + print(f"\nJSON 輸出已保存至: {saved_file}") + else: + print("\n模型調用了未預期的函數。") + else: + print("\n模型沒有調用工具。") + else: + print("\n無法解析回應。") + +if __name__ == "__main__": + image_path = "1.jpg" # 替換為您的圖像路徑 + main(image_path) \ No newline at end of file diff --git a/test/test_rate_limit.py b/test/test_rate_limit.py new file mode 100644 index 00000000..48803203 --- /dev/null +++ b/test/test_rate_limit.py @@ -0,0 +1,41 @@ +import re + +def parse_rate_limit(limit_string): + # 定义时间单位到秒的映射 + time_units = { + 's': 1, 'sec': 1, 'second': 1, + 'm': 60, 'min': 60, 'minute': 60, + 'h': 3600, 'hr': 3600, 'hour': 3600, + 'd': 86400, 'day': 86400, + 'mo': 2592000, 'month': 2592000, + 'y': 31536000, 'year': 31536000 + } + + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit_string) + if not match: + raise ValueError(f"Invalid rate limit format: {limit_string}") + + count, unit = match.groups() + count = int(count) + + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") + + seconds = time_units[unit] + + return (count, seconds) + +# 测试函数 +test_cases = [ + "2/min", "5/hour", "10/day", "1/second", "3/mo", "1/year", + "20/s", "15/m", "8/h", "100/d", "50/mo", "2/y" +] + +for case in test_cases: + try: + result = parse_rate_limit(case) + print(f"{case} => {result}") + except ValueError as e: + print(f"Error parsing {case}: {str(e)}") \ No newline at end of file diff --git a/test/test_request.py b/test/test_request.py new file mode 100644 index 00000000..a366acbb --- /dev/null +++ b/test/test_request.py @@ -0,0 +1,87 @@ +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from models import RequestModel +from request import get_payload +import json + +async def test_gemini_payload(): + # 构造测试请求 + request_data = { + "model": "gemini-1.5-pro-latest", + "messages": [ + { + "role": "system", + "content": "\n\nDisplay a clock to show current time\n获取当前时间\n\n" + }, + { + "role": "user", + "content": "几点了？" + } + ], + "stream": True, + "temperature": 1.3, + "top_p": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "user": "d5a7516e-e919-45f0-81a1-42bad3da6125", + "tools": [ + { + "type": "function", + "function": { + "name": "clock-time____getCurrentTime____standalone", + "description": "获取当前时间", + "parameters": { + "type": "object", + "properties": {} + } + } + } + ] + } + + request = RequestModel(**request_data) + provider = { + "provider": "gemini", + "base_url": "https://generativelanguage.googleapis.com", + "api": "your-api-key", # 测试时替换为实际的 API key + "model": ["gemini-1.5-pro-latest"], + "project_id": "your-project-id" + } + + url, headers, payload = await get_payload(request, "vertex-gemini", provider) + # url, headers, payload = await get_payload(request, "gemini", provider) + + print("payload", json.dumps(payload, indent=4, ensure_ascii=False)) + + # 验证生成的 payload 结构 + assert "contents" in payload + assert "tools" in payload + assert len(payload["tools"]) == 1 + assert "function_declarations" in payload["tools"][0] + + # 验证工具配置 + assert payload["tools"][0]["function_declarations"][0]["name"] == "clock-time____getCurrentTime____standalone" + assert payload["tools"][0]["function_declarations"][0]["description"] == "获取当前时间" + + # 验证消息内容 + assert len(payload["contents"]) == 2 + assert payload["contents"][0]["role"] == "system" + assert payload["contents"][1]["role"] == "user" + + # 验证其他参数 + assert payload["temperature"] == 1.3 + assert payload["top_p"] == 1.0 + + # 验证安全设置 + assert "safetySettings" in payload + assert len(payload["safetySettings"]) == 4 + + # 验证工具配置 + assert "tool_config" in payload + assert payload["tool_config"]["function_calling_config"]["mode"] == "AUTO" + +if __name__ == "__main__": + import asyncio + asyncio.run(test_gemini_payload()) \ No newline at end of file diff --git a/test/test_ruamel_yaml.py b/test/test_ruamel_yaml.py new file mode 100644 index 00000000..d91820c9 --- /dev/null +++ b/test/test_ruamel_yaml.py @@ -0,0 +1,44 @@ +from ruamel.yaml import YAML + +# 假设我们有以下 YAML 内容 +yaml_content = """ +# 这是顶级注释 +key1: value1 # 行尾注释 +key2: value2 + +# 这是嵌套结构的注释 +nested: + subkey1: subvalue1 + subkey2: subvalue2 # 嵌套的行尾注释 + +# 列表的注释 +list_key: + - item1 + - item2 # 列表项的注释 +""" + +# 创建 YAML 对象 +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +with open('api.yaml', 'r', encoding='utf-8') as file: + data = yaml.load(file) + +# data = yaml.load(yaml_content) +# 加载 YAML 数据 +print(data) + +# # 修改数据 +# data['key1'] = 'new_value1' +# data['nested']['subkey1'] = 'new_subvalue1' +# data['list_key'].append('new_item') + +# 将修改后的数据写回文件（这里我们使用 StringIO 来模拟文件操作） +# from io import StringIO +# output = StringIO() +# yaml.dump(data, output) +# print(output.getvalue()) + +with open('formatted.yaml', 'w', encoding='utf-8') as file: + yaml.dump(data, file) \ No newline at end of file diff --git a/test/test_vertex copy.py b/test/test_vertex.py similarity index 100% rename from test/test_vertex copy.py rename to test/test_vertex.py diff --git a/test/test_weights.py b/test/test_weights.py new file mode 100644 index 00000000..e1732cb2 --- /dev/null +++ b/test/test_weights.py @@ -0,0 +1,33 @@ +def weighted_round_robin(weights): + provider_names = list(weights.keys()) + current_weights = {name: 0 for name in provider_names} + num_selections = total_weight = sum(weights.values()) + weighted_provider_list = [] + + for _ in range(num_selections): + max_ratio = -1 + selected_letter = None + + for name in provider_names: + current_weights[name] += weights[name] + ratio = current_weights[name] / weights[name] + + if ratio > max_ratio: + max_ratio = ratio + selected_letter = name + + weighted_provider_list.append(selected_letter) + current_weights[selected_letter] -= total_weight + + return weighted_provider_list + +# 权重和选择次数 +weights = {'a': 5, 'b': 3, 'c': 2} +index = {'a', 'c'} + +result = dict(filter(lambda item: item[0] in index, weights.items())) +print(result) +# result = {k: weights[k] for k in index if k in weights} +# print(result) +weighted_provider_list = weighted_round_robin(weights) +print(weighted_provider_list) diff --git a/test/xue/test_dropdown_sheet.py b/test/xue/test_dropdown_sheet.py new file mode 100644 index 00000000..2f65e4cf --- /dev/null +++ b/test/xue/test_dropdown_sheet.py @@ -0,0 +1,75 @@ +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue.components import dropdown, sheet, button, form, input + +xue_initialize(tailwind=True) +app = FastAPI() + +@app.get("/", response_class=HTMLResponse) +async def root(): + result = HTML( + Head( + title="Dropdown with Edit Sheet Example", + ), + Body( + Div( + dropdown.dropdown_menu("Actions"), + Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 + class_="container mx-auto p-4" + ) + ) + ).render() + print(result) + return result + +@app.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse) +async def get_dropdown_menu_content(menu_id: str): + items = [ + { + "icon": "pencil", + "label": "Edit", + "hx-get": "/edit-sheet", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + {"icon": "trash", "label": "Delete"}, + {"icon": "copy", "label": "Duplicate"}, + ] + result = dropdown.dropdown_menu_content(menu_id, items).render() + print("dropdown-menu result", result) + return result + +@app.get("/edit-sheet", response_class=HTMLResponse) +async def get_edit_sheet(): + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + form.Form( + form.FormField("Name", "name", placeholder="Enter item name"), + form.FormField("Description", "description", placeholder="Enter item description"), + Div( + button.button("Save", class_="bg-blue-500 text-white"), + button.button("Cancel", class_="bg-gray-300 text-gray-700 ml-2", data_close_sheet="true"), + class_="flex justify-end mt-4" + ), + class_="space-y-4" + ) + ) + ) + + result = sheet.Sheet( + "edit-sheet", + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/test/xue/test_form_uni_api.py b/test/xue/test_form_uni_api.py new file mode 100644 index 00000000..9aaeb832 --- /dev/null +++ b/test/xue/test_form_uni_api.py @@ -0,0 +1,116 @@ +from fastapi import FastAPI, Form as FastAPIForm +from fastapi.responses import HTMLResponse +from xue import HTML, Head, Body, Div, xue_initialize, Strong, Span, Ul, Li +from xue.components import form, button, checkbox, input +from xue.components.model_config_row import model_config_row +from typing import List, Optional +import time + +xue_initialize(tailwind=True) +app = FastAPI() + +@app.get("/", response_class=HTMLResponse) +async def root(): + result = HTML( + Head( + title="Provider Configuration Form" + ), + Body( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="password", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + model_config_row("model1", "gpt-4o: deepbricks-gpt-4o-mini", True), + model_config_row("model2", "gpt-4o"), + model_config_row("model3", "gpt-3.5-turbo"), + model_config_row("model4", "claude-3-5-sonnet-20240620: claude-3-5-sonnet"), + model_config_row("model5", "o1-mini-all"), + model_config_row("model6", "o1-preview-all"), + model_config_row("model7", "whisper-1"), + id="models-container" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=True), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", class_="bg-blue-500 text-white"), + button.button("Cancel", class_="bg-gray-300 text-gray-700 ml-2"), + class_="flex justify-end mt-4" + ), + hx_post="/submit", + hx_swap="outerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ).render() + print(result) + return result + +@app.post("/add-model", response_class=HTMLResponse) +async def add_model(): + new_model_id = f"model{hash(str(time.time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +def form_success_message(provider, base_url, api_key, models, tools_enabled, notes): + return Div( + Strong("Success!", class_="font-bold"), + Span("Form submitted successfully.", class_="block sm:inline"), + Ul( + Li(f"Provider: {provider}"), + Li(f"Base URL: {base_url}"), + Li(f"API Key: {'*' * len(api_key)}"), + Li(f"Models: {', '.join(models)}"), + Li(f"Tools Enabled: {'Yes' if tools_enabled else 'No'}"), + Li(f"Notes: {notes}"), + class_="mt-3" + ), + class_="bg-green-100 border border-green-400 text-green-700 px-4 py-3 rounded relative", + role="alert" + ) + +@app.post("/submit", response_class=HTMLResponse) +async def submit_form( + provider: str = FastAPIForm(...), + base_url: str = FastAPIForm(...), + api_key: str = FastAPIForm(...), + models: List[str] = FastAPIForm([]), + tools: Optional[str] = FastAPIForm(None), + notes: Optional[str] = FastAPIForm(None) +): + # 处理提交的数据 + print(f"Received: provider={provider}, base_url={base_url}, api_key={api_key}") + print(f"Models: {models}") + print(f"Tools Enabled: {tools is not None}") + print(f"Notes: {notes}") + + # 返回处理结果 + return form_success_message( + provider, + base_url, + api_key, + models, + tools is not None, + notes or "No notes provided" + ).render() + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/test/xue/test_home.py b/test/xue/test_home.py new file mode 100644 index 00000000..231280a8 --- /dev/null +++ b/test/xue/test_home.py @@ -0,0 +1,472 @@ +from fastapi import FastAPI, Request +from fastapi import Form as FastapiForm, HTTPException, Depends +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from fastapi.security import APIKeyHeader +from typing import Optional, List + +from xue import HTML, Head, Body, Div, xue_initialize, Script +from xue.components.menubar import ( + Menubar, MenubarMenu, MenubarTrigger, MenubarContent, + MenubarItem, MenubarSeparator +) +from xue.components import input +from xue.components import dropdown, sheet, form, button, checkbox +from xue.components.model_config_row import model_config_row +import time + +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from components.provider_table import data_table + + +from ruamel.yaml import YAML +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +xue_initialize(tailwind=True) + +from starlette.middleware.base import BaseHTTPMiddleware +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +API_YAML_PATH = "./api.yaml" +class RequestBodyLoggerMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + if request.method == "POST" and request.url.path.startswith("/submit/"): + # if request.method == "POST": + body = await request.body() + logger.info(f"Request body for {request.url.path}: {body.decode()}") + + response = await call_next(request) + return response + +from utils import load_config +from contextlib import asynccontextmanager +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.config, app.state.api_keys_db, app.state.api_list = await load_config() + for item in app.state.api_keys_db: + if item.get("role") == "admin": + app.state.admin_api_key = item.get("api") + if not hasattr(app.state, "admin_api_key"): + if len(app.state.api_keys_db) >= 1: + app.state.admin_api_key = app.state.api_keys_db[0].get("api") + else: + raise Exception("No admin API key found") + + yield + # 关闭时的代码 + await app.state.client.aclose() + +app = FastAPI(lifespan=lifespan) +# app.add_middleware(RequestBodyLoggerMiddleware) +app.add_middleware(RequestBodyLoggerMiddleware) + +data_table_columns = [ + # {"label": "Status", "value": "status", "sortable": True}, + {"label": "Provider", "value": "provider", "sortable": True}, + {"label": "Base url", "value": "base_url", "sortable": True}, + # {"label": "Engine", "value": "engine", "sortable": True}, + {"label": "Tools", "value": "tools", "sortable": True}, +] + +API_KEY_NAME = "X-API-Key" +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) + +@app.get("/login", response_class=HTMLResponse) +async def login_page(): + return HTML( + Head(title="登录"), + Body( + Div( + form.Form( + form.FormField("API Key", "x_api_key", type="password", placeholder="输入API密钥", required=True), + Div(id="error-message", class_="text-red-500 mt-2"), + Div( + button.button("提交", variant="primary", type="submit"), + class_="flex justify-end mt-4" + ), + hx_post="/verify-api-key", + hx_target="#error-message", + hx_swap="innerHTML", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-md" + ) + ) + ).render() + + +@app.post("/verify-api-key", response_class=HTMLResponse) +async def verify_api_key(x_api_key: str = FastapiForm(...)): + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + response = JSONResponse(content={"success": True}) + response.headers["HX-Redirect"] = "/" # 添加这一行 + response.set_cookie( + key="x_api_key", + value=x_api_key, + httponly=True, + max_age=1800, # 30分钟 + secure=False, # 在开发环境中设置为False，生产环境中使用HTTPS时设置为True + samesite="lax" # 改为"lax"以允许重定向时携带cookie + ) + return response + else: + return Div("无效的API密钥", class_="text-red-500").render() + +async def get_api_key(request: Request, x_api_key: Optional[str] = Depends(api_key_header)): + if not x_api_key: + x_api_key = request.cookies.get("x_api_key") or request.query_params.get("x_api_key") + # print(f"Cookie x_api_key: {request.cookies.get('x_api_key')}") # 添加此行 + # print(f"Query param x_api_key: {request.query_params.get('x_api_key')}") # 添加此行 + # print(f"Header x_api_key: {x_api_key}") # 添加此行 + # logger.info(f"x_api_key: {x_api_key} {x_api_key == 'your_admin_api_key'}") + + if x_api_key == app.state.admin_api_key: # 替换为实际的管理员API密钥 + return x_api_key + else: + return None + +@app.get("/", response_class=HTMLResponse) +async def root(x_api_key: str = Depends(get_api_key)): + if not x_api_key: + return RedirectResponse(url="/login", status_code=303) + + result = HTML( + Head( + Script(""" + document.addEventListener('DOMContentLoaded', function() { + const filterInput = document.getElementById('users-table-filter'); + filterInput.addEventListener('input', function() { + const filterValue = this.value; + htmx.ajax('GET', `/filter-table?filter=${filterValue}`, '#users-table'); + }); + }); + """), + title="Menubar Example" + ), + Body( + Div( + Menubar( + MenubarMenu( + MenubarTrigger("File", "file-menu"), + MenubarContent( + MenubarItem("New Tab", shortcut="⌘T"), + MenubarItem("New Window", shortcut="⌘N"), + MenubarItem("New Incognito Window", disabled=True), + MenubarSeparator(), + MenubarItem("Print...", shortcut="⌘P"), + ), + id="file-menu" + ), + MenubarMenu( + MenubarTrigger("Edit", "edit-menu"), + MenubarContent( + MenubarItem("Undo", shortcut="⌘Z"), + MenubarItem("Redo", shortcut="⇧⌘Z"), + MenubarSeparator(), + MenubarItem("Cut"), + MenubarItem("Copy"), + MenubarItem("Paste"), + ), + id="edit-menu" + ), + MenubarMenu( + MenubarTrigger("View", "view-menu"), + MenubarContent( + MenubarItem("Always Show Bookmarks Bar"), + MenubarItem("Always Show Full URLs"), + MenubarSeparator(), + MenubarItem("Reload", shortcut="⌘R"), + MenubarItem("Force Reload", shortcut="⇧⌘R", disabled=True), + MenubarSeparator(), + MenubarItem("Toggle Fullscreen"), + MenubarItem("Hide Sidebar"), + ), + id="view-menu" + ), + ), + class_="p-4" + ), + Div( + data_table(data_table_columns, app.state.config["providers"], "users-table"), + class_="p-4" + ), + Div(id="sheet-container"), # 这里是 sheet 将被加载的地方 + class_="container mx-auto", + id="body" + ) + ).render() + # print(result) + return result + +@app.get("/dropdown-menu/{menu_id}/{row_id}", response_class=HTMLResponse) +async def get_columns_menu(menu_id: str, row_id: str): + columns = [ + { + "label": "Edit", + "value": "edit", + "hx-get": f"/edit-sheet/{row_id}", + "hx-target": "#sheet-container", + "hx-swap": "innerHTML" + }, + { + "label": "Duplicate", + "value": "duplicate", + "hx-post": f"/duplicate/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML" + }, + { + "label": "Delete", + "value": "delete", + "hx-delete": f"/delete/{row_id}", + "hx-target": "body", + "hx-swap": "outerHTML", + "hx-confirm": "确定要删除这个配置吗？" + }, + ] + result = dropdown.dropdown_menu_content(menu_id, columns).render() + print(result) + return result + +@app.get("/dropdown-menu/{menu_id}", response_class=HTMLResponse) +async def get_columns_menu(menu_id: str): + result = dropdown.dropdown_menu_content(menu_id, data_table_columns).render() + print(result) + return result + +@app.get("/filter-table", response_class=HTMLResponse) +async def filter_table(filter: str = ""): + filtered_data = [ + provider for provider in app.state.config["providers"] + if filter.lower() in str(provider["provider"]).lower() or + filter.lower() in str(provider["base_url"]).lower() or + filter.lower() in str(provider["tools"]).lower() + ] + return data_table(data_table_columns, filtered_data, "users-table", with_filter=False).render() + +@app.post("/add-model", response_class=HTMLResponse) +async def add_model(): + new_model_id = f"model{hash(str(time.time()))}" # 生成一个唯一的ID + new_model = model_config_row(new_model_id).render() + return new_model + +@app.get("/edit-sheet/{row_id}", response_class=HTMLResponse) +async def get_edit_sheet(row_id: str, x_api_key: str = Depends(get_api_key)): + row_data = get_row_data(row_id) + print("row_data", row_data) + + model_list = [] + for index, model in enumerate(row_data["model"]): + if isinstance(model, str): + model_list.append(model_config_row(f"model{index}", model, "", True)) + if isinstance(model, dict): + # print("model", model, list(model.items())[0]) + key, value = list(model.items())[0] + model_list.append(model_config_row(f"model{index}", key, value, True)) + + sheet_id = "edit-sheet" + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Edit Item"), + sheet.SheetDescription("Make changes to your item here.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", value=row_data["provider"], placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", value=row_data["base_url"], placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", value=row_data["api"], type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div( + *model_list, + id="models-container" + ), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", checked=row_data["tools"], name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", value=row_data.get("notes", ""), placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", type="button", class_="ml-2", onclick=f"toggleSheet('{sheet_id}')"), + class_="flex justify-end mt-4" + ), + hx_post=f"/submit/{row_id}", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + sheet_id, + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +@app.get("/add-provider-sheet", response_class=HTMLResponse) +async def get_add_provider_sheet(): + edit_sheet_content = sheet.SheetContent( + sheet.SheetHeader( + sheet.SheetTitle("Add New Provider"), + sheet.SheetDescription("Enter details for the new provider.") + ), + sheet.SheetBody( + Div( + form.Form( + form.FormField("Provider", "provider", placeholder="Enter provider name", required=True), + form.FormField("Base URL", "base_url", placeholder="Enter base URL", required=True), + form.FormField("API Key", "api_key", type="text", placeholder="Enter API key"), + Div( + Div("Models", class_="text-lg font-semibold mb-2"), + Div(id="models-container"), + button.button( + "Add Model", + class_="mt-2", + hx_post="/add-model", + hx_target="#models-container", + hx_swap="beforeend" + ), + class_="mb-4" + ), + Div( + checkbox.checkbox("tools", "Enable Tools", name="tools"), + class_="mb-4" + ), + form.FormField("Notes", "notes", placeholder="Enter any additional notes"), + Div( + button.button("Submit", variant="primary", type="submit"), + button.button("Cancel", variant="outline", class_="ml-2"), + class_="flex justify-end mt-4" + ), + hx_post="/submit/new", + hx_swap="outerHTML", + hx_target="body", + class_="space-y-4" + ), + class_="container mx-auto p-4 max-w-2xl" + ) + ) + ) + + result = sheet.Sheet( + "add-provider-sheet", + Div(), + edit_sheet_content, + width="80%", + max_width="800px" + ).render() + return result + +def get_row_data(row_id): + index = int(row_id) + # print(app.state.config["providers"]) + return app.state.config["providers"][index] + +def update_row_data(row_id, updated_data): + print(row_id, updated_data) + index = int(row_id) + app.state.config["providers"][index] = updated_data + save_api_yaml() + +def save_api_yaml(): + with open(API_YAML_PATH, "w", encoding="utf-8") as f: + yaml.dump(app.state.config, f) + +@app.post("/submit/{row_id}", response_class=HTMLResponse) +async def submit_form( + row_id: str, + request: Request, + provider: str = FastapiForm(...), + base_url: str = FastapiForm(...), + api_key: Optional[str] = FastapiForm(None), + tools: Optional[str] = FastapiForm(None), + notes: Optional[str] = FastapiForm(None), + x_api_key: str = Depends(get_api_key) +): + form_data = await request.form() + + # 收集模型数据 + models = [] + for key, value in form_data.items(): + if key.startswith("model_name_"): + model_id = key.split("_")[-1] + enabled = form_data.get(f"model_enabled_{model_id}") == "on" + rename = form_data.get(f"model_rename_{model_id}") + if value: + if rename: + models.append({value: rename}) + else: + models.append(value) + + updated_data = { + "provider": provider, + "base_url": base_url, + "api": api_key, + "model": models, + "tools": tools == "on", + "notes": notes, + } + + print("updated_data", updated_data) + + if row_id == "new": + # 添加新提供者 + app.state.config["providers"].append(updated_data) + else: + # 更新现有提供者 + update_row_data(row_id, updated_data) + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +@app.post("/duplicate/{row_id}", response_class=HTMLResponse) +async def duplicate_row(row_id: str): + index = int(row_id) + original_data = app.state.config["providers"][index] + new_data = original_data.copy() + new_data["provider"] += "-copy" + app.state.config["providers"].insert(index + 1, new_data) + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +@app.delete("/delete/{row_id}", response_class=HTMLResponse) +async def delete_row(row_id: str): + index = int(row_id) + del app.state.config["providers"][index] + + # 保存更新后的配置 + save_api_yaml() + + return await root() + +if __name__ == "__main__": + import uvicorn + uvicorn.run("__main__:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/utils.py b/utils.py index 6b18ae51..1199dc56 100644 --- a/utils.py +++ b/utils.py @@ -1,68 +1,426 @@ import json from fastapi import HTTPException import httpx +from urllib.parse import urlparse from log_config import logger -def update_config(config_data): +import re +from time import time +def parse_rate_limit(limit_string): + # 定义时间单位到秒的映射 + time_units = { + 's': 1, 'sec': 1, 'second': 1, + 'm': 60, 'min': 60, 'minute': 60, + 'h': 3600, 'hr': 3600, 'hour': 3600, + 'd': 86400, 'day': 86400, + 'mo': 2592000, 'month': 2592000, + 'y': 31536000, 'year': 31536000 + } + + # 处理多个限制条件 + limits = [] + for limit in limit_string.split(','): + limit = limit.strip() + # 使用正则表达式匹配数字和单位 + match = re.match(r'^(\d+)/(\w+)$', limit) + if not match: + raise ValueError(f"Invalid rate limit format: {limit}") + + count, unit = match.groups() + count = int(count) + + # 转换单位到秒 + if unit not in time_units: + raise ValueError(f"Unknown time unit: {unit}") + + seconds = time_units[unit] + limits.append((count, seconds)) + + return limits + +from collections import defaultdict +class InMemoryRateLimiter: + def __init__(self): + self.requests = defaultdict(list) + + async def is_rate_limited(self, key: str, limits) -> bool: + now = time() + + # 检查所有速率限制条件 + for limit, period in limits: + # 计算在当前时间窗口内的请求数量 + recent_requests = sum(1 for req in self.requests[key] if req > now - period) + if recent_requests >= limit: + return True + + # 清理太旧的请求记录（比最长时间窗口还要老的记录） + max_period = max(period for _, period in limits) + self.requests[key] = [req for req in self.requests[key] if req > now - max_period] + + # 记录新的请求 + self.requests[key].append(now) + return False + +rate_limiter = InMemoryRateLimiter() + +import asyncio + +class ThreadSafeCircularList: + def __init__(self, items = [], rate_limit={"default": "999999/min"}, schedule_algorithm="round_robin"): + if schedule_algorithm == "random": + import random + self.items = random.sample(items, len(items)) + self.schedule_algorithm = "random" + elif schedule_algorithm == "round_robin": + self.items = items + self.schedule_algorithm = "round_robin" + elif schedule_algorithm == "fixed_priority": + self.items = items + self.schedule_algorithm = "fixed_priority" + else: + self.items = items + logger.warning(f"Unknown schedule algorithm: {schedule_algorithm}, use (round_robin, random, fixed_priority) instead") + self.schedule_algorithm = "round_robin" + self.index = 0 + self.lock = asyncio.Lock() + # 修改为二级字典，第一级是item，第二级是model + self.requests = defaultdict(lambda: defaultdict(list)) + self.cooling_until = defaultdict(float) + self.rate_limits = {} + if isinstance(rate_limit, dict): + for rate_limit_model, rate_limit_value in rate_limit.items(): + self.rate_limits[rate_limit_model] = parse_rate_limit(rate_limit_value) + elif isinstance(rate_limit, str): + self.rate_limits["default"] = parse_rate_limit(rate_limit) + else: + logger.error(f"Error ThreadSafeCircularList: Unknown rate_limit type: {type(rate_limit)}, rate_limit: {rate_limit}") + + async def set_cooling(self, item: str, cooling_time: int = 60): + """设置某个 item 进入冷却状态 + + Args: + item: 需要冷却的 item + cooling_time: 冷却时间(秒)，默认60秒 + """ + if item == None: + return + now = time() + async with self.lock: + self.cooling_until[item] = now + cooling_time + # 清空该 item 的请求记录 + # self.requests[item] = [] + logger.warning(f"API key {item} 已进入冷却状态，冷却时间 {cooling_time} 秒") + + async def is_rate_limited(self, item, model: str = None) -> bool: + now = time() + # 检查是否在冷却中 + if now < self.cooling_until[item]: + return True + + # 获取适用的速率限制 + + if model: + model_key = model + else: + model_key = "default" + + rate_limit = None + # 先尝试精确匹配 + if model and model in self.rate_limits: + rate_limit = self.rate_limits[model] + else: + # 如果没有精确匹配，尝试模糊匹配 + for limit_model in self.rate_limits: + if limit_model != "default" and model and limit_model in model: + rate_limit = self.rate_limits[limit_model] + break + + # 如果都没匹配到，使用默认值 + if rate_limit is None: + rate_limit = self.rate_limits.get("default", [(999999, 60)]) # 默认限制 + + # 检查所有速率限制条件 + for limit_count, limit_period in rate_limit: + # 使用特定模型的请求记录进行计算 + recent_requests = sum(1 for req in self.requests[item][model_key] if req > now - limit_period) + if recent_requests >= limit_count: + logger.warning(f"API key {item} 对模型 {model_key} 已达到速率限制 ({limit_count}/{limit_period}秒)") + return True + + # 清理太旧的请求记录 + max_period = max(period for _, period in rate_limit) + self.requests[item][model_key] = [req for req in self.requests[item][model_key] if req > now - max_period] + + # 记录新的请求 + self.requests[item][model_key].append(now) + return False + + async def next(self, model: str = None): + async with self.lock: + if self.schedule_algorithm == "fixed_priority": + self.index = 0 + start_index = self.index + while True: + item = self.items[self.index] + self.index = (self.index + 1) % len(self.items) + + if not await self.is_rate_limited(item, model): + return item + + # 如果已经检查了所有的 API key 都被限制 + if self.index == start_index: + logger.warning(f"All API keys are rate limited!") + raise HTTPException(status_code=429, detail="Too many requests") + + async def after_next_current(self): + # 返回当前取出的 API，因为已经调用了 next，所以当前API应该是上一个 + if len(self.items) == 0: + return None + async with self.lock: + item = self.items[(self.index - 1) % len(self.items)] + return item + + def get_items_count(self) -> int: + """返回列表中的项目数量 + + Returns: + int: items列表的长度 + """ + return len(self.items) + +def circular_list_encoder(obj): + if isinstance(obj, ThreadSafeCircularList): + return obj.to_dict() + raise TypeError(f'Object of type {obj.__class__.__name__} is not JSON serializable') + +provider_api_circular_list = defaultdict(ThreadSafeCircularList) + +def get_model_dict(provider): + model_dict = {} + for model in provider['model']: + if type(model) == str: + model_dict[model] = model + if isinstance(model, dict): + model_dict.update({new: old for old, new in model.items()}) + return model_dict + +def update_initial_model(provider): + try: + engine, stream_mode = get_engine(provider, endpoint=None, original_model="") + # print("engine", engine, provider) + api_url = provider['base_url'] + api = provider['api'] + + if engine == "gemini": + url = "https://generativelanguage.googleapis.com/v1beta/models" + params = {"key": api} + + with httpx.Client() as client: + response = client.get(url, params=params) + + original_models = response.json() + if original_models.get("error"): + raise Exception({"error": original_models.get("error"), "endpoint": url, "api": api}) + + models = {"data": []} + for model in original_models["models"]: + models["data"].append({ + "id": model["name"].split("models/")[-1], + }) + else: + endpoint = BaseAPI(api_url=api_url) + endpoint_models_url = endpoint.v1_models + if isinstance(api, list): + api = api[0] + headers = {"Authorization": f"Bearer {api}"} + response = httpx.get( + endpoint_models_url, + headers=headers, + ) + models = response.json() + if models.get("error"): + raise Exception({"error": models.get("error"), "endpoint": endpoint_models_url, "api": api}) + + # print(models) + models_list = models["data"] + models_id = [model["id"] for model in models_list] + set_models = set() + for model_item in models_id: + set_models.add(model_item) + models_id = list(set_models) + # print(models_id) + return models_id + except Exception as e: + # print("error:", e) + import traceback + traceback.print_exc() + return [] + +from ruamel.yaml import YAML, YAMLError +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=2, sequence=4, offset=2) + +API_YAML_PATH = "./api.yaml" +yaml_error_message = None + +def save_api_yaml(config_data): + with open(API_YAML_PATH, "w", encoding="utf-8") as f: + yaml.dump(config_data, f) + +def update_config(config_data, use_config_url=False): for index, provider in enumerate(config_data['providers']): - model_dict = {} - for model in provider['model']: - if type(model) == str: - model_dict[model] = model - if type(model) == dict: - model_dict.update({new: old for old, new in model.items()}) - provider['model'] = model_dict if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' + if provider.get('cf_account_id'): + provider['base_url'] = 'https://api.cloudflare.com/' + + if isinstance(provider['provider'], int): + provider['provider'] = str(provider['provider']) + + provider_api = provider.get('api', None) + if provider_api: + if isinstance(provider_api, int): + provider_api = str(provider_api) + if isinstance(provider_api, str): + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( + [provider_api], + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}), + safe_get(provider, "preferences", "api_key_schedule_algorithm", default="round_robin") + ) + if isinstance(provider_api, list): + provider_api_circular_list[provider['provider']] = ThreadSafeCircularList( + provider_api, + safe_get(provider, "preferences", "api_key_rate_limit", default={"default": "999999/min"}), + safe_get(provider, "preferences", "api_key_schedule_algorithm", default="round_robin") + ) + + if "models.inference.ai.azure.com" in provider['base_url'] and not provider.get("model"): + provider['model'] = [ + "gpt-4o", + "gpt-4o-mini", + "o1-mini", + "o1-preview", + "text-embedding-3-small", + "text-embedding-3-large", + ] + + if not provider.get("model"): + model_list = update_initial_model(provider) + if model_list: + provider["model"] = model_list + if not use_config_url: + save_api_yaml(config_data) + + if provider.get("tools") == None: + provider["tools"] = True + config_data['providers'][index] = provider + api_keys_db = config_data['api_keys'] + + for index, api_key in enumerate(config_data['api_keys']): + weights_dict = {} + models = [] + if api_key.get('model'): + for model in api_key.get('model'): + if isinstance(model, dict): + key, value = list(model.items())[0] + provider_name = key.split("/")[0] + model_name = key.split("/")[1] + + for provider_item in config_data["providers"]: + if provider_item['provider'] != provider_name: + continue + model_dict = get_model_dict(provider_item) + if model_name in model_dict.keys(): + weights_dict.update({provider_name + "/" + model_name: int(value)}) + elif model_name == "*": + weights_dict.update({provider_name + "/" + model_name: int(value) for model_item in model_dict.keys()}) + + models.append(key) + if isinstance(model, str): + models.append(model) + if weights_dict: + config_data['api_keys'][index]['weights'] = weights_dict + config_data['api_keys'][index]['model'] = models + api_keys_db[index]['model'] = models + else: + # Default to all models if 'model' field is not set + config_data['api_keys'][index]['model'] = ["all"] + api_keys_db[index]['model'] = ["all"] + api_list = [item["api"] for item in api_keys_db] # logger.info(json.dumps(config_data, indent=4, ensure_ascii=False)) return config_data, api_keys_db, api_list # 读取YAML配置文件 -async def load_config(app): - import yaml +async def load_config(app=None): + import os try: - with open('./api.yaml', 'r') as f: - # 判断是否为空文件 - conf = yaml.safe_load(f) - # conf = None - if conf: - config, api_keys_db, api_list = update_config(conf) - else: - # logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") - config, api_keys_db, api_list = [], [], [] + with open(API_YAML_PATH, 'r', encoding='utf-8') as file: + conf = yaml.load(file) + + if conf: + config, api_keys_db, api_list = update_config(conf, use_config_url=False) + else: + logger.error("配置文件 'api.yaml' 为空。请检查文件内容。") + config, api_keys_db, api_list = {}, {}, [] except FileNotFoundError: - logger.error("配置文件 'api.yaml' 未找到。请确保文件存在于正确的位置。") - config, api_keys_db, api_list = [], [], [] - except yaml.YAMLError: - logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。") - config, api_keys_db, api_list = [], [], [] + if not os.environ.get('CONFIG_URL'): + logger.error("'api.yaml' not found. Please check the file path.") + config, api_keys_db, api_list = {}, {}, [] + except YAMLError as e: + logger.error("配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。%s", e) + global yaml_error_message + yaml_error_message = "配置文件 'api.yaml' 格式不正确。请检查 YAML 格式。" + config, api_keys_db, api_list = {}, {}, [] + except OSError as e: + logger.error(f"open 'api.yaml' failed: {e}") + config, api_keys_db, api_list = {}, {}, [] - if config != []: + if config != {}: return config, api_keys_db, api_list - import os # 新增： 从环境变量获取配置URL并拉取配置 config_url = os.environ.get('CONFIG_URL') if config_url: try: - response = await app.state.client.get(config_url) + default_config = { + "headers": { + "User-Agent": "curl/7.68.0", + "Accept": "*/*", + }, + "http2": True, + "verify": True, + "follow_redirects": True + } + # 初始化客户端管理器 + timeout = httpx.Timeout( + connect=15.0, + read=100, + write=30.0, + pool=200 + ) + client = httpx.AsyncClient( + timeout=timeout, + **default_config + ) + response = await client.get(config_url) # logger.info(f"Fetching config from {response.text}") response.raise_for_status() - config_data = yaml.safe_load(response.text) + config_data = yaml.load(response.text) # 更新配置 # logger.info(config_data) if config_data: - config, api_keys_db, api_list = update_config(config_data) + config, api_keys_db, api_list = update_config(config_data, use_config_url=True) else: logger.error(f"Error fetching or parsing config from {config_url}") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] except Exception as e: logger.error(f"Error fetching or parsing config from {config_url}: {str(e)}") - config, api_keys_db, api_list = [], [], [] + config, api_keys_db, api_list = {}, {}, [] return config, api_keys_db, api_list def ensure_string(item): @@ -75,63 +433,115 @@ def ensure_string(item): else: return str(item) +def identify_audio_format(file_bytes): + # 读取开头的字节 + if file_bytes.startswith(b'\xFF\xFB') or file_bytes.startswith(b'\xFF\xF3'): + return "MP3" + elif file_bytes.startswith(b'ID3'): + return "MP3 with ID3" + elif file_bytes.startswith(b'OpusHead'): + return "OPUS" + elif file_bytes.startswith(b'ADIF'): + return "AAC (ADIF)" + elif file_bytes.startswith(b'\xFF\xF1') or file_bytes.startswith(b'\xFF\xF9'): + return "AAC (ADTS)" + elif file_bytes.startswith(b'fLaC'): + return "FLAC" + elif file_bytes.startswith(b'RIFF') and file_bytes[8:12] == b'WAVE': + return "WAV" + return "Unknown/PCM" + import asyncio -async def error_handling_wrapper(generator, status_code=200): +import time as time_module +async def error_handling_wrapper(generator, channel_id, engine, stream, error_triggers): + start_time = time_module.time() try: first_item = await generator.__anext__() + first_response_time = time_module.time() - start_time first_item_str = first_item - # logger.info("first_item_str: %s", first_item_str) + # logger.info("first_item_str: %s :%s", type(first_item_str), first_item_str) if isinstance(first_item_str, (bytes, bytearray)): - first_item_str = first_item_str.decode("utf-8") + if identify_audio_format(first_item_str) in ["MP3", "MP3 with ID3", "OPUS", "AAC (ADIF)", "AAC (ADTS)", "FLAC", "WAV"]: + return first_item, first_response_time + else: + first_item_str = first_item_str.decode("utf-8") if isinstance(first_item_str, str): - if first_item_str.startswith("data: "): - first_item_str = first_item_str[6:] - elif first_item_str.startswith("data:"): - first_item_str = first_item_str[5:] + if first_item_str.startswith("data:"): + first_item_str = first_item_str.lstrip("data: ") if first_item_str.startswith("[DONE]"): - logger.error("error_handling_wrapper [DONE]!") + logger.error(f"provider: {channel_id:<11} error_handling_wrapper [DONE]!") + raise StopAsyncIteration + try: + encode_first_item_str = first_item_str.encode().decode('unicode-escape') + except UnicodeDecodeError: + encode_first_item_str = first_item_str + logger.error(f"provider: {channel_id:<11} error UnicodeDecodeError: %s", first_item_str) + if any(x in encode_first_item_str for x in error_triggers): + logger.error(f"provider: {channel_id:<11} error const string: %s", encode_first_item_str) raise StopAsyncIteration try: first_item_str = json.loads(first_item_str) except json.JSONDecodeError: - logger.error("error_handling_wrapper JSONDecodeError!" + repr(first_item_str)) + logger.error(f"provider: {channel_id:<11} error_handling_wrapper JSONDecodeError! {repr(first_item_str)}") raise StopAsyncIteration - if isinstance(first_item_str, dict) and 'error' in first_item_str: + if isinstance(first_item_str, dict) and 'error' in first_item_str and first_item_str.get('error') != {"message": "","type": "","param": "","code": None}: # 如果第一个 yield 的项是错误信息，抛出 HTTPException - raise HTTPException(status_code=status_code, detail=f"{first_item_str}"[:300]) + status_code = first_item_str.get('status_code', 500) + detail = first_item_str.get('details', f"{first_item_str}") + raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) + + if isinstance(first_item_str, dict) and safe_get(first_item_str, "choices", 0, "error", default=None): + # 如果第一个 yield 的项是错误信息，抛出 HTTPException + status_code = safe_get(first_item_str, "choices", 0, "error", "code", default=500) + detail = safe_get(first_item_str, "choices", 0, "error", "message", default=f"{first_item_str}") + raise HTTPException(status_code=status_code, detail=f"{detail}"[:300]) + + if isinstance(first_item_str, dict) and engine not in ["tts", "embedding", "dalle", "moderation", "whisper"] and stream == False: + if any(x in str(first_item_str) for x in error_triggers): + logger.error(f"provider: {channel_id:<11} error const string: %s", first_item_str) + raise StopAsyncIteration + content = safe_get(first_item_str, "choices", 0, "message", "content", default=None) + if content == "" or content is None: + raise StopAsyncIteration # 如果不是错误，创建一个新的生成器，首先yield第一个项，然后yield剩余的项 async def new_generator(): + # print("type(first_item)", type(first_item)) + # print("first_item", ensure_string(first_item)) yield ensure_string(first_item) try: async for item in generator: yield ensure_string(item) - except (httpx.ReadError, asyncio.CancelledError) as e: - logger.error(f"Network error in new_generator: {e}") - raise + except asyncio.CancelledError: + # 客户端断开连接是正常行为，不需要记录错误日志 + logger.debug(f"provider: {channel_id:<11} Stream cancelled by client") + return + except (httpx.ReadError, httpx.RemoteProtocolError) as e: + # 只记录真正的网络错误 + # logger.error(f"provider: {channel_id:<11} Network error in new_generator: {e}") + raise HTTPException(status_code=502, detail=f"Network error in new_generator: {e}") - return new_generator() + return new_generator(), first_response_time except StopAsyncIteration: - raise HTTPException(status_code=status_code, detail="data: {'error': 'No data returned'}") + raise HTTPException(status_code=400, detail="data: {'error': 'No data returned'}") -def post_all_models(token, config, api_list): +def post_all_models(api_index, config, api_list, models_list): all_models = [] unique_models = set() - if token not in api_list: - raise HTTPException(status_code=403, detail="Invalid or missing API Key") - api_index = api_list.index(token) if config['api_keys'][api_index]['model']: for model in config['api_keys'][api_index]['model']: + if model == "all": + # 如果模型名为 all，则返回所有模型 + all_models = get_all_models(config) + return all_models if "/" in model: provider = model.split("/")[0] model = model.split("/")[1] if model == "*": - for provider_item in config["providers"]: - if provider_item['provider'] != provider: - continue - for model_item in provider_item['model'].keys(): + if provider.startswith("sk-") and provider in api_list: + for model_item in models_list[provider]: if model_item not in unique_models: unique_models.add(model_item) model_info = { @@ -139,23 +549,53 @@ def post_all_models(token, config, api_list): "object": "model", "created": 1720524448858, "owned_by": "uni-api" - # "owned_by": provider_item['provider'] } all_models.append(model_info) + else: + for provider_item in config["providers"]: + if provider_item['provider'] != provider: + continue + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys(): + if model_item not in unique_models: + unique_models.add(model_item) + model_info = { + "id": model_item, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + # "owned_by": provider_item['provider'] + } + all_models.append(model_info) else: - for provider_item in config["providers"]: - if provider_item['provider'] != provider: - continue - for model_item in provider_item['model'].keys() : - if model_item not in unique_models and model_item == model: - unique_models.add(model_item) - model_info = { - "id": model_item, - "object": "model", - "created": 1720524448858, - "owned_by": "uni-api" - } - all_models.append(model_info) + if provider.startswith("sk-") and provider in api_list: + if model in models_list[provider] and model not in unique_models: + unique_models.add(model) + model_info = { + "id": model, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + } + all_models.append(model_info) + else: + for provider_item in config["providers"]: + if provider_item['provider'] != provider: + continue + model_dict = get_model_dict(provider_item) + for model_item in model_dict.keys(): + if model_item not in unique_models and model_item == model: + unique_models.add(model_item) + model_info = { + "id": model_item, + "object": "model", + "created": 1720524448858, + "owned_by": "uni-api" + } + all_models.append(model_info) + continue + + if model.startswith("sk-") and model in api_list: continue if model not in unique_models: @@ -164,7 +604,7 @@ def post_all_models(token, config, api_list): "id": model, "object": "model", "created": 1720524448858, - "owned_by": model + "owned_by": "uni-api" } all_models.append(model_info) @@ -175,7 +615,8 @@ def get_all_models(config): unique_models = set() for provider in config["providers"]: - for model in provider['model'].keys(): + model_dict = get_model_dict(provider) + for model in model_dict.keys(): if model not in unique_models: unique_models.add(model) model_info = { @@ -189,6 +630,8 @@ def get_all_models(config): return all_models # 【GCP-Vertex AI 目前有這些區域可用】 https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude?hl=zh_cn +# https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations?hl=zh-cn#available-regions + # c3.5s # us-east5 # europe-west1 @@ -206,20 +649,187 @@ def get_all_models(config): # us-central1 # europe-west1 # europe-west4 -from collections import deque -class CircularList: - def __init__(self, items): - self.queue = deque(items) - def next(self): - if not self.queue: - return None - item = self.queue.popleft() - self.queue.append(item) - return item -c35s = CircularList(["us-east5", "europe-west1"]) -c3s = CircularList(["us-east5", "us-central1", "asia-southeast1"]) -c3o = CircularList(["us-east5"]) -c3h = CircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) -gem = CircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) \ No newline at end of file +c35s = ThreadSafeCircularList(["us-east5", "europe-west1"]) +c3s = ThreadSafeCircularList(["us-east5", "us-central1", "asia-southeast1"]) +c3o = ThreadSafeCircularList(["us-east5"]) +c3h = ThreadSafeCircularList(["us-east5", "us-central1", "europe-west1", "europe-west4"]) +gemini1 = ThreadSafeCircularList(["us-central1", "us-east4", "us-west1", "us-west4", "europe-west1", "europe-west2"]) +gemini2 = ThreadSafeCircularList(["us-central1"]) + +class BaseAPI: + def __init__( + self, + api_url: str = "https://api.openai.com/v1/chat/completions", + ): + if api_url == "": + api_url = "https://api.openai.com/v1/chat/completions" + self.source_api_url: str = api_url + from urllib.parse import urlparse, urlunparse + parsed_url = urlparse(self.source_api_url) + # print("parsed_url", parsed_url) + if parsed_url.scheme == "": + raise Exception("Error: API_URL is not set") + if parsed_url.path != '/': + before_v1 = parsed_url.path.split("chat/completions")[0] + else: + before_v1 = "" + self.base_url: str = urlunparse(parsed_url[:2] + ("",) + ("",) * 3) + self.v1_url: str = urlunparse(parsed_url[:2]+ (before_v1,) + ("",) * 3) + self.v1_models: str = urlunparse(parsed_url[:2] + (before_v1 + "models",) + ("",) * 3) + self.chat_url: str = urlunparse(parsed_url[:2] + (before_v1 + "chat/completions",) + ("",) * 3) + self.image_url: str = urlunparse(parsed_url[:2] + (before_v1 + "images/generations",) + ("",) * 3) + self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "audio/transcriptions",) + ("",) * 3) + self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "moderations",) + ("",) * 3) + self.embeddings: str = urlunparse(parsed_url[:2] + (before_v1 + "embeddings",) + ("",) * 3) + self.audio_speech: str = urlunparse(parsed_url[:2] + (before_v1 + "audio/speech",) + ("",) * 3) + +def safe_get(data, *keys, default=None): + for key in keys: + try: + data = data[key] if isinstance(data, (dict, list)) else data.get(key) + except (KeyError, IndexError, AttributeError, TypeError): + return default + return data + + +# end_of_line = "\n\r\n" +# end_of_line = "\r\n" +# end_of_line = "\n\r" +end_of_line = "\n\n" +# end_of_line = "\r" +# end_of_line = "\n" + +import random +import string +async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) + sample_data = { + "id": f"chatcmpl-{random_str}", + "object": "chat.completion.chunk", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"content": content} if content else {}, + "logprobs": None, + "finish_reason": None if content else "stop" + } + ], + "usage": None, + "system_fingerprint": "fp_d576307f90", + } + if function_call_content: + sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]} + if tools_id and function_call_name: + sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"id": tools_id,"type":"function","function":{"name": function_call_name, "arguments":""}}]} + # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]} + if role: + sample_data["choices"][0]["delta"] = {"role": role, "content": ""} + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + sample_data["choices"] = [] + json_data = json.dumps(sample_data, ensure_ascii=False) + + # 构建SSE响应 + sse_response = f"data: {json_data}" + end_of_line + + return sse_response + +async def generate_no_stream_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None, role=None, total_tokens=0, prompt_tokens=0, completion_tokens=0): + random.seed(timestamp) + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=29)) + sample_data = { + "id": f"chatcmpl-{random_str}", + "object": "chat.completion", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": role, + "content": content, + "refusal": None + }, + "logprobs": None, + "finish_reason": "stop" + } + ], + "usage": None, + "system_fingerprint": "fp_a7d06e42a7" + } + if total_tokens: + total_tokens = prompt_tokens + completion_tokens + sample_data["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens} + json_data = json.dumps(sample_data, ensure_ascii=False) + + return json_data + +def get_engine(provider, endpoint=None, original_model=""): + parsed_url = urlparse(provider['base_url']) + # print("parsed_url", parsed_url) + engine = None + stream = None + if parsed_url.path.endswith("/v1beta") or parsed_url.path.endswith("/v1"): + engine = "gemini" + elif parsed_url.netloc == 'aiplatform.googleapis.com': + engine = "vertex" + elif parsed_url.netloc.rstrip('/').endswith('openai.azure.com'): + engine = "azure" + elif parsed_url.netloc == 'api.cloudflare.com': + engine = "cloudflare" + elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): + engine = "claude" + elif parsed_url.netloc == 'api.cohere.com': + engine = "cohere" + stream = True + else: + engine = "gpt" + + if original_model \ + and "claude" not in original_model \ + and "gpt" not in original_model \ + and "deepseek" not in original_model \ + and "o1" not in original_model \ + and "o3" not in original_model \ + and "gemini" not in original_model \ + and "learnlm" not in original_model \ + and "grok" not in original_model \ + and parsed_url.netloc != 'api.cloudflare.com' \ + and parsed_url.netloc != 'api.cohere.com': + engine = "openrouter" + + if "claude" in original_model and engine == "vertex": + engine = "vertex-claude" + + if "gemini" in original_model and engine == "vertex": + engine = "vertex-gemini" + + if provider.get("engine"): + engine = provider["engine"] + + if endpoint == "/v1/images/generations" or "stable-diffusion" in original_model: + engine = "dalle" + stream = False + + if endpoint == "/v1/audio/transcriptions": + engine = "whisper" + stream = False + + if endpoint == "/v1/moderations": + engine = "moderation" + stream = False + + if endpoint == "/v1/embeddings": + engine = "embedding" + + if endpoint == "/v1/audio/speech": + engine = "tts" + stream = False + + return engine, stream \ No newline at end of file diff --git a/vercel.json b/vercel.json new file mode 100644 index 00000000..6e08ab58 --- /dev/null +++ b/vercel.json @@ -0,0 +1,14 @@ +{ + "builds": [ + { + "src": "main.py", + "use": "@vercel/python" + } + ], + "routes": [ + { + "src": "/(.*)", + "dest": "main.py" + } + ] +} \ No newline at end of file