my-free-code 多 Provider 编码网关

1from fastapi import FastAPI
2from .api.routes import router
3
4app = FastAPI(title="My Free Code", version="0.8.0")
5app.include_router(router)
6
7def main():
8    import uvicorn
9    from .config import settings
10    uvicorn.run(app, host=settings.host, port=settings.port)

10class ProviderRuntime:
11    def __init__(self, settings: Settings):
12        self.settings = settings
13        self.semaphores = {p.id: asyncio.Semaphore(settings.max_concurrency) for p in PROVIDER_MAP.values()}
14        self.calls = {p.id: deque() for p in PROVIDER_MAP.values()}
15        self.unhealthy_until = {}
16
17    def context(self, provider_id, remote_model):
18        p = PROVIDER_MAP[provider_id]
19        base = p.base_url
20        if provider_id == "ollama":
21            base = self.settings.key("OLLAMA_BASE_URL") or base
22        elif provider_id == "lmstudio":
23            base = self.settings.key("LM_STUDIO_BASE_URL") or base
24        elif provider_id == "llamacpp":
25            base = self.settings.key("LLAMACPP_BASE_URL") or base
26        if not base:
27            raise RuntimeError(f"{p.name} requires a specialized adapter")
28        return ProviderContext(provider_id, remote_model, self.settings.key(p.key), base, self.settings.http_timeout)

3def select_model(requested: str | None, s: Settings) -> str:
4    value = (requested or "").lower()
5    if "/" in value:
6        return requested or s.model
7    if "opus" in value and s.opus:
8        return s.opus
9    if "sonnet" in value and s.sonnet:
10        return s.sonnet
11    if "haiku" in value and s.haiku:
12        return s.haiku
13    if "fable" in value and s.fable:
14        return s.fable
15    return s.model
16
17def candidates(primary: str, s: Settings) -> list[str]:
18    return list(dict.fromkeys([primary, *s.fallbacks]))
19
20def split_model(model: str) -> tuple[str, str]:
21    provider, sep, remote = model.partition("/")
22    if not sep or not provider or not remote:
23        raise ValueError("model must have the form provider/model")
24    return provider, remote

7@dataclass(frozen=True)
8class Settings:
9    host: str = os.getenv("HOST", "127.0.0.1")
10    port: int = int(os.getenv("PORT", "8082"))
11    auth_token: str = os.getenv("PROXY_AUTH_TOKEN", "local")
12    model: str = os.getenv("MODEL", "open_router/openrouter/free")
13    fable: str = os.getenv("MODEL_FABLE", "")
14    opus: str = os.getenv("MODEL_OPUS", "")
15    sonnet: str = os.getenv("MODEL_SONNET", "")
16    haiku: str = os.getenv("MODEL_HAIKU", "")
17    fallbacks: list[str] = field(default_factory=lambda: [
18        x.strip() for x in os.getenv("FALLBACK_MODELS", "").split(",") if x.strip()
19    ])
20    max_concurrency: int = int(os.getenv("PROVIDER_MAX_CONCURRENCY", "5"))
21    rate_limit: int = int(os.getenv("PROVIDER_RATE_LIMIT", "10"))
22    rate_window: float = float(os.getenv("PROVIDER_RATE_WINDOW", "3"))
23    http_timeout: float = float(os.getenv("HTTP_TIMEOUT", "180"))
24
25    def key(self, env_name: str | None) -> str:
26        return os.getenv(env_name, "").strip() if env_name else ""

3def build_gateway_models(config):
4    return [
5        GatewayModel("claude-opus", "opus", True),
6        GatewayModel("claude-opus-no-thinking", "opus", False),
7        GatewayModel("claude-sonnet", "sonnet", True),
8        GatewayModel("claude-sonnet-no-thinking", "sonnet", False),
9        GatewayModel("claude-haiku", "haiku", True),
10        GatewayModel("claude-haiku-no-thinking", "haiku", False),
11        GatewayModel("claude-fable", "fable", True),
12    ]
13
14def public_catalog(config):
15    direct = [x for x in {
16        config.model, config.fable, config.opus, config.sonnet, config.haiku,
17        *config.fallbacks
18    } if x and "/" in x]
19    gateways = [x.id for x in build_gateway_models(config)]
20    return list(dict.fromkeys(gateways + direct))

11def anthropic_to_openai(body: dict, remote_model: str) -> dict:
12    messages = []
13    system = body.get("system")
14    if system:
15        messages.append({"role": "system", "content": content_text(system)})
16
17    for msg in body.get("messages", []):
18        role = msg.get("role", "user")
19        content = msg.get("content", "")
20        if isinstance(content, str):
21            messages.append({"role": role, "content": content})
22            continue
23
24        parts = []
25        tool_calls = []
26        for block in content:
27            kind = block.get("type")
28            if kind == "text":
29                parts.append({"type": "text", "text": block.get("text", "")})
30            elif kind == "image":
31                src = block.get("source", {})
32                if src.get("type") == "base64":
33                    parts.append({
34                        "type": "image_url",
35                        "image_url": {
36                            "url": f"data:{src.get('media_type','image/png')};base64,{src.get('data','')}"
37                        }
38                    })
39            elif kind == "tool_use":
40                tool_calls.append({
41                    "id": block.get("id"),
42                    "type": "function",
43                    "function": {
44                        "name": block.get("name"),
45                        "arguments": json.dumps(block.get("input", {}))
46                    }
47                })
48            elif kind == "tool_result":
49                messages.append({
50                    "role": "tool",
51                    "tool_call_id": block.get("tool_use_id"),
52                    "content": content_text(block.get("content", ""))
53                })
54
55        item = {"role": role, "content": parts}
56        if tool_calls:
57            item["tool_calls"] = tool_calls
58        messages.append(item)
59
60    out = {
61        "model": remote_model,
62        "messages": messages,
63        "stream": bool(body.get("stream")),
64        "max_tokens": body.get("max_tokens", 4096),
65    }
66    for key in ("temperature", "top_p", "stop"):
67        if key in body:
68            out[key] = body[key]
69
70    if body.get("tools"):
71        out["tools"] = [{
72            "type": "function",
73            "function": {
74                "name": t["name"],
75                "description": t.get("description", ""),
76                "parameters": t.get("input_schema", {}),
77            }
78        } for t in body["tools"]]
79
80    if body.get("tool_choice"):
81        out["tool_choice"] = body["tool_choice"]
82
83    return out

85def openai_to_anthropic(data: dict, requested_model: str) -> dict:
86    choice = (data.get("choices") or [{}])[0]
87    message = choice.get("message") or {}
88    blocks = []
89
90    if message.get("content"):
91        blocks.append({"type": "text", "text": message["content"]})
92
93    for call in message.get("tool_calls") or []:
94        fn = call.get("function") or {}
95        raw = fn.get("arguments", "{}")
96        try:
97            args = json.loads(raw)
98        except (ValueError, TypeError):
99            args = {"raw": raw}
100        blocks.append({
101            "type": "tool_use",
102            "id": call.get("id"),
103            "name": fn.get("name"),
104            "input": args,
105        })
106
107    reason = {
108        "stop": "end_turn",
109        "length": "max_tokens",
110        "tool_calls": "tool_use",
111    }.get(choice.get("finish_reason"), "end_turn")
112
113    usage = data.get("usage") or {}
114    return {
115        "id": data.get("id", "msg_proxy"),
116        "type": "message",
117        "role": "assistant",
118        "model": requested_model,
119        "content": blocks or [{"type": "text", "text": ""}],
120        "stop_reason": reason,
121        "stop_sequence": None,
122        "usage": {
123            "input_tokens": usage.get("prompt_tokens", 0),
124            "output_tokens": usage.get("completion_tokens", 0),
125        },
126    }

1def responses_to_messages(body: dict) -> dict:
2    # Minimal normalization for Responses clients. Rich response items are
3    # intentionally preserved as text/tool content rather than discarded.
4    input_value = body.get("input", "")
5    messages = []
6    if isinstance(input_value, str):
7        messages.append({"role": "user", "content": input_value})
8    elif isinstance(input_value, list):
9        for item in input_value:
10            if isinstance(item, dict):
11                role = item.get("role", "user")
12                content = item.get("content", "")
13                messages.append({"role": role, "content": content})
14            else:
15                messages.append({"role": "user", "content": str(item)})
16
17    return {
18        "messages": messages,
19        "model": body.get("model", ""),
20        "max_tokens": body.get("max_output_tokens", 4096),
21        "stream": bool(body.get("stream")),
22    }
23
24def messages_to_response(message: dict) -> dict:
25    text = "".join(
26        b.get("text", "")
27        for b in message.get("content", [])
28        if isinstance(b, dict) and b.get("type") == "text"
29    )
30    return {
31        "id": message.get("id"),
32        "object": "response",
33        "status": "completed",
34        "model": message.get("model"),
35        "output": [{
36            "type": "message",
37            "role": "assistant",
38            "content": [{"type": "output_text", "text": text}],
39        }],
40        "usage": {
41            "input_tokens": message.get("usage", {}).get("input_tokens", 0),
42            "output_tokens": message.get("usage", {}).get("output_tokens", 0),
43            "total_tokens": (
44                message.get("usage", {}).get("input_tokens", 0)
45                + message.get("usage", {}).get("output_tokens", 0)
46            ),
47        },
48    }

49@router.post("/v1/messages")
50async def messages(body: dict, authorization: str | None = Header(None)):
51    require_auth(authorization)
52    selected = select_model(body.get("model"), settings)
53
54    if not body.get("stream"):
55        last = None
56        for model in candidates(selected, settings):
57            try:
58                return await runtime.complete(body, model)
59            except Exception as exc:
60                last = exc
61        raise HTTPException(status_code=502, detail=str(last))
62
63    return StreamingResponse(
64        stream_messages(body, selected),
65        media_type="text/event-stream",
66        headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
67    )

69async def stream_messages(body, selected):
70    message_id = "msg_" + uuid.uuid4().hex[:20]
71    start = {
72        "type": "message_start",
73        "message": {
74            "id": message_id,
75            "type": "message",
76            "role": "assistant",
77            "content": [],
78            "model": selected,
79            "stop_reason": None,
80            "usage": {"input_tokens": 0, "output_tokens": 0},
81        },
82    }
83    yield f"event: message_start\ndata: {json.dumps(start)}\n\n"
84    yield 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n'
85
86    started = False
87    error = None
88    for model in candidates(selected, settings):
89        try:
90            async for chunk in runtime.stream(body, model):
91                delta = ((chunk.get("choices") or [{}])[0].get("delta") or {})
92                text = delta.get("content")
93                if text:
94                    started = True
95                    event = {
96                        "type": "content_block_delta",
97                        "index": 0,
98                        "delta": {"type": "text_delta", "text": text},
99                    }
100                    yield f"event: content_block_delta\ndata: {json.dumps(event)}\n\n"
101            error = None
102            break
103        except Exception as exc:
104            error = exc
105            if started:
106                break
107
108    if error and not started:
109        event = {"type":"error","error":{"type":"api_error","message":str(error)}}
110        yield f"event: error\ndata: {json.dumps(event)}\n\n"
111        return
112
113    yield 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n'
114    yield 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\n'
115    yield 'event: message_stop\ndata: {"type":"message_stop"}\n\n'

58    async def stream(self, anthropic_body, public_model):
59        provider_id, remote = public_model.split("/", 1)
60        sem = await self.admit(provider_id)
61        await sem.acquire()
62        try:
63            adapter = self.adapter(provider_id)
64            async for item in adapter.stream(
65                self.context(provider_id, remote),
66                anthropic_to_openai(anthropic_body, remote)
67            ):
68                yield item
69        except Exception:
70            self.unhealthy_until[provider_id] = time.monotonic() + 2
71            raise
72        finally:
73            sem.release()

1from my_free_code.core.router import select_model, candidates
2from my_free_code.config import Settings
3
4def test_model_routing():
5    s = Settings(model="open_router/free", sonnet="deepseek/deepseek-chat")
6    assert select_model("sonnet", s) == "deepseek/deepseek-chat"
7
8def test_explicit_model():
9    s = Settings(model="open_router/free")
10    assert select_model("groq/foo", s) == "groq/foo"
11
12def test_fallbacks_are_unique():
13    s = Settings(model="a/b", fallbacks=["c/d","a/b","c/d"])
14    assert candidates("a/b", s) == ["a/b","c/d"]

1from my_free_code.core.anthropic import anthropic_to_openai, openai_to_anthropic, rough_token_count
2
3def test_anthropic_conversion():
4    body = {
5        "model": "sonnet",
6        "system": "You are helpful",
7        "messages": [{"role":"user","content":"hello"}],
8        "max_tokens": 100,
9    }
10    out = anthropic_to_openai(body, "test-model")
11    assert out["model"] == "test-model"
12    assert out["messages"][0]["role"] == "system"
13    assert out["messages"][1]["content"] == "hello"
14
15def test_response_conversion():
16    data = {
17        "id": "x",
18        "choices": [{"message":{"content":"hello"},"finish_reason":"stop"}],
19        "usage":{"prompt_tokens":3,"completion_tokens":2},
20    }
21    out = openai_to_anthropic(data, "provider/test")
22    assert out["content"][0]["text"] == "hello"
23    assert out["usage"]["output_tokens"] == 2
24
25def test_token_counter():
26    assert rough_token_count({"messages":[{"role":"user","content":"hello"}]}) > 0

1from my_free_code.core.reasoning import resolve_reasoning
2from my_free_code.core.model_catalog import public_catalog 
3from my_free_code.core.streaming import openai_delta_to_anthropic
4from my_free_code.config import Settings
5
6def test_reasoning_disabled():
7    p = resolve_reasoning({"thinking":{"type":"disabled"}})
8    assert p.mode == "off"
9
10def test_reasoning_budget():
11    p = resolve_reasoning({"thinking":{"type":"enabled","budget_tokens":1234}})
12    assert p.budget_tokens == 1234
13
14def test_catalog_has_gateway_models():
15    data = public_catalog(Settings(model="x/y"))
16    assert "claude-sonnet" in data
17    assert "x/y" in data
18
19def test_stream_delta():
20    events = openai_delta_to_anthropic({"content":"hi"})
21    assert "text_delta" in events[0]