接入 Amazon Bedrock 新一代推理引擎 Mantle：用 LiteLLM 网关统一调用 GPT-5.6 与 Claude

curl http://<PROXY_HOST>/v1/chat/completions \
  -H "Authorization: Bearer <VIRTUAL_KEY>" \
  -H 'Content-Type: application/json' \
  -d '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":"你好"}]}'

curl http://<PROXY_HOST>/v1/responses \
  -H "Authorization: Bearer <VIRTUAL_KEY>" \
  -H 'Content-Type: application/json' \
  -d '{"model":"gpt-5.6-sol","input":"你好"}'

{
  "model": "gpt-5.6-sol",
  "status": "completed",
  "output": [{"content": [{"type": "output_text", "text": "你好！有什么我可以帮你的吗？"}]}],
  "usage": {"input_tokens": 7, "output_tokens": 13, "total_tokens": 20}
}

opencode run -m litellm-east1/gpt-5.6-sol "重构这个函数"
opencode run -m litellm-east1/claude-sonnet-4-5 "解释这段代码"

codex exec -m gpt-5.6-sol -c model_provider=litellm-east1 "写单元测试"

REGION=us-east-1
ACCOUNT=$(aws sts get-caller-identity --query Account --output text)

# 1. 信任策略
cat > /tmp/trust.json <<'EOF'
{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
 "Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}
EOF

# 2. Bedrock 推理权限（bedrock-runtime + bedrock-mantle 两套 action）
cat > /tmp/bedrock.json <<EOF
{"Version":"2012-10-17","Statement":[
 {"Effect":"Allow",
  "Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream",
            "bedrock:Converse","bedrock:ConverseStream"],
  "Resource":["arn:aws:bedrock:*::foundation-model/*",
              "arn:aws:bedrock:*:${ACCOUNT}:inference-profile/*"]},
 {"Effect":"Allow",
  "Action":["bedrock-mantle:CreateInference","bedrock-mantle:CreateModelInference",
            "bedrock-mantle:ListModels","bedrock-mantle:GetModel"],
  "Resource":"*"}]}
EOF

# 3. 建角色与实例配置文件
aws iam create-role --role-name litellm-gateway-role \
  --assume-role-policy-document file:///tmp/trust.json
aws iam put-role-policy --role-name litellm-gateway-role \
  --policy-name bedrock-invoke --policy-document file:///tmp/bedrock.json
aws iam create-instance-profile --instance-profile-name litellm-gateway-profile
aws iam add-role-to-instance-profile \
  --instance-profile-name litellm-gateway-profile --role-name litellm-gateway-role

aws ec2 run-instances --region $REGION \
  --image-id $(aws ssm get-parameter --region $REGION \
      --name /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 \
      --query Parameter.Value --output text) \
  --instance-type t3.large \
  --key-name <YOUR_KEYPAIR> \
  --subnet-id <YOUR_PUBLIC_SUBNET> \
  --security-group-ids <YOUR_SG> \
  --associate-public-ip-address \
  --iam-instance-profile Name=litellm-gateway-profile \
  --metadata-options "HttpTokens=required,HttpPutResponseHopLimit=2,HttpEndpoint=enabled" \
  --block-device-mappings 'DeviceName=/dev/xvda,Ebs={VolumeSize=30,VolumeType=gp3,Encrypted=true}' \
  --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=litellm-gateway}]'

sudo mkdir -p /opt/litellm && cd /opt/litellm
sudo tee .env >/dev/null <<EOF
LITELLM_MASTER_KEY=sk-$(openssl rand -hex 24)
LITELLM_SALT_KEY=sk-$(openssl rand -hex 24)
PG_PASSWORD=$(openssl rand -hex 20)
EOF
sudo chmod 600 .env

model_list:
  # Claude：bedrock/ 前缀 -> bedrock-runtime
  - model_name: claude-sonnet-4-5
    litellm_params:
      model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
      aws_region_name: us-east-1
  - model_name: claude-haiku-4-5
    litellm_params:
      model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
      aws_region_name: us-east-1

  # GPT-5.x：bedrock_mantle/ 前缀 -> Mantle 引擎，无需 api_key
  - model_name: gpt-5.6-sol
    litellm_params:
      model: bedrock_mantle/openai.gpt-5.6-sol
      aws_region_name: us-east-1
  - model_name: gpt-5.6-terra
    litellm_params:
      model: bedrock_mantle/openai.gpt-5.6-terra
      aws_region_name: us-east-1
  - model_name: gpt-5.6-luna
    litellm_params:
      model: bedrock_mantle/openai.gpt-5.6-luna
      aws_region_name: us-east-1
  - model_name: gpt-5.5
    litellm_params:
      model: bedrock_mantle/openai.gpt-5.5
      aws_region_name: us-east-1
  - model_name: gpt-5.4
    litellm_params:
      model: bedrock_mantle/openai.gpt-5.4
      aws_region_name: us-east-1

  # gpt-oss：同在 Mantle 上，但只支持 Chat Completions
  - model_name: gpt-oss-120b
    litellm_params:
      model: bedrock_mantle/openai.gpt-oss-120b
      aws_region_name: us-east-1
  - model_name: gpt-oss-20b
    litellm_params:
      model: bedrock_mantle/openai.gpt-oss-20b
      aws_region_name: us-east-1

litellm_settings:
  drop_params: true

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY

services:
  litellm:
    image: docker.litellm.ai/berriai/litellm-database:v1.90.2
    container_name: litellm
    restart: unless-stopped
    ports:
      - "80:4000"
    volumes:
      - ./config.yaml:/app/config.yaml:ro
    environment:
      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY}
      DATABASE_URL: postgresql://litellm:${PG_PASSWORD}@postgres:5432/litellm
      STORE_MODEL_IN_DB: "True"
    command: ["--config", "/app/config.yaml"]
    depends_on:
      postgres: {condition: service_healthy}

  postgres:
    image: postgres:16
    container_name: litellm-postgres
    restart: unless-stopped
    environment:
      POSTGRES_USER: litellm
      POSTGRES_PASSWORD: ${PG_PASSWORD}
      POSTGRES_DB: litellm
    volumes: [pgdata:/var/lib/postgresql/data]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
      interval: 5s
      retries: 20

volumes:
  pgdata:

cd /opt/litellm
sudo docker compose --env-file /opt/litellm/.env up -d

# 等就绪（首次启动要跑数据库迁移，约 1 分钟）
until curl -sf http://localhost/health/liveliness >/dev/null; do sleep 5; done
curl -s http://localhost/health/readiness   # 期望 {"status":"healthy","db":"connected"}

curl -X POST http://localhost/key/generate \
  -H "Authorization: Bearer <MASTER_KEY>" \
  -H 'Content-Type: application/json' \
  -d '{"models":["gpt-5.6-sol","gpt-5.6-terra","gpt-5.6-luna","gpt-5.5","gpt-5.4",
                 "gpt-oss-120b","gpt-oss-20b","claude-sonnet-4-5","claude-haiku-4-5"],
       "max_budget":50,"rpm_limit":100,"key_alias":"coding-agents"}'

curl -X POST http://<PROXY_HOST>/model/update \
  -H "Authorization: Bearer <MASTER_KEY>" -H 'Content-Type: application/json' \
  -d '{"model_name":"gpt-5.6-sol",
       "model_info":{"id":"<列表里的 model_id>"},
       "litellm_params":{"model":"bedrock_mantle/openai.gpt-5.6-sol",
                         "aws_region_name":"us-east-1",
                         "custom_llm_provider":"bedrock_mantle"}}'

400 validation_error: invalid request body:
Invalid 'input': value did not match any expected variant

if not found:
    # No additional_tools item present → non-Codex or already standard.
    return data

# 1. 文件放到宿主机
sudo cp codex_additional_tools_flatten.py /opt/litellm/

# 2. docker-compose.yml 的 litellm 服务下增加挂载
#      - ./codex_additional_tools_flatten.py:/app/codex_additional_tools_flatten.py:ro

# 3. config.yaml 的 litellm_settings 下注册 callback
#    callbacks:
#      - codex_additional_tools_flatten.codex_additional_tools_flatten_instance

# 4. 必须用 up -d 重建（restart 不会挂载新卷）
sudo docker compose --env-file /opt/litellm/.env up -d

"""
Codex `additional_tools` → top-level `tools` flatten hook for LiteLLM.

Newer Codex (>=26.707, "Responses Lite" serialization) packs its tool list into
a non-standard input item of shape:

    {"type": "additional_tools", "role": "developer", "tools": [ ... ]}

Bedrock Mantle's Responses API (bedrock-mantle) only accepts the STANDARD OpenAI
Responses schema, whose `input` items are message/reasoning/function_call/etc.
`additional_tools` is a Codex-private variant, so Mantle rejects the whole body
with `400 validation_error: Invalid 'input': value did not match any expected
variant`.

Verified out-of-band: Mantle DOES accept those same tools when placed in the
standard top-level `tools` field (including Codex's `custom`/`namespace` tool
types). So this hook rewrites the request pre-call:

  1. find every input item with type == "additional_tools"
  2. merge each one's `tools` into the top-level `data["tools"]`
  3. drop the additional_tools item(s) from `data["input"]`

Scope guard:
  - Only touches `aresponses` calls whose input actually contains an
    `additional_tools` item. Every other request returns untouched.
  - Wrapped in try/except: a rewrite failure logs and passes the ORIGINAL data
    through rather than dropping the request.
"""
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger

# call_types that carry a Responses-API body (data["input"] / data["tools"])
_RESPONSES_CALL_TYPES = {"aresponses"}


class CodexAdditionalToolsFlatten(CustomLogger):
    async def async_pre_call_hook(
        self, user_api_key_dict, cache, data, call_type
    ):
        if call_type not in _RESPONSES_CALL_TYPES:
            return data
        try:
            input_items = data.get("input")
            if not isinstance(input_items, list):
                return data
            # Collect additional_tools items without mutating during iteration.
            # NOTE: `found` (did we see an additional_tools item?) is tracked
            # SEPARATELY from `extra_tools` (did that item carry any tools?).
            # A later Codex turn re-sends the additional_tools item with an
            # EMPTY tools list (tools already established earlier in the thread).
            # That empty item is still a non-standard variant Mantle rejects, so
            # it MUST be dropped from input even when there is nothing to merge —
            # keying the rewrite off `extra_tools` alone left the empty item in
            # place and produced 400 "Invalid 'input'" on every follow-up turn.
            found = False
            extra_tools = []
            kept_items = []
            for item in input_items:
                if isinstance(item, dict) and item.get("type") == "additional_tools":
                    found = True
                    tools = item.get("tools")
                    if isinstance(tools, list):
                        extra_tools.extend(tools)
                    # drop the item (do not keep it in input)
                else:
                    kept_items.append(item)
            if not found:
                # No additional_tools item present → non-Codex or already standard.
                return data
            # Drop the additional_tools item(s) from input unconditionally.
            data["input"] = kept_items
            # Merge any carried tools into top-level tools (preserve existing).
            if extra_tools:
                existing = data.get("tools")
                merged = (existing if isinstance(existing, list) else []) + extra_tools
                data["tools"] = merged
            verbose_logger.info(
                "CodexAdditionalToolsFlatten: removed additional_tools item(s), "
                "flattened %d tool(s) into top-level tools (model=%s)"
                % (len(extra_tools), str(data.get("model", "")))
            )
        except Exception as e:
            # Never fail a request over this rewrite — log and pass original through.
            verbose_logger.warning(
                "CodexAdditionalToolsFlatten: flatten failed, passing through: %s" % e
            )
        return data


# Module-level instance referenced by config.yaml callbacks:
#   callbacks: ["codex_additional_tools_flatten.codex_additional_tools_flatten_instance"]
codex_additional_tools_flatten_instance = CodexAdditionalToolsFlatten()