GitHub

AI

Generate text and objects, call tools and stream responses through a LanguageModel service any provider can fill.

pyfect.ai follows Effect's effect/ai. A program asks the LanguageModel service for text, a decoded object or a stream, and a provider's layer supplies the service. So the same program runs on Claude, on any OpenAI-compatible API, or on a scripted model in a test. Failures are an AiError whose reason says what went wrong (a rate limit, an invalid request, output that does not match its schema). A request that never reached the provider is a NetworkError with its method and url; its cause is the HttpClientError it failed with. Errors carry the response and its body, never the request's headers.

Language models and providers

language_model.generate_text(prompt) needs a LanguageModel and gives a GenerateTextResponse: its content parts, with text, tool_calls, tool_results, finish_reason and usage. A prompt is a string, a list of messages or a Prompt.

pyfect.ai.anthropic calls Anthropic's Messages API, pyfect.ai.openai OpenAI's Responses API and pyfect.ai.openai_compat any Chat Completions API (a local server, a gateway). Each has a client layer, with the API key as a Redacted (or from Config with layer_config), and a language_model whose model(name).layer provides the LanguageModel. They all send their requests through the HttpClient service, so they need no vendor SDK: provide httpx_client.layer (the pyfect-dev[httpx] extra) in production, or a test client, as here.

generate.py
import json

import pyfect as fx
from pyfect.ai import language_model
from pyfect.ai.anthropic import client
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact


# The program needs a LanguageModel, from any provider.
@fx.fn
def program():
    reply = yield from language_model.generate_text("Name a prime number.")
    print(reply.text, reply.finish_reason, reply.usage.output_tokens.total)


# Claude through Anthropic's Messages API, over the HttpClient service.
Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
    fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)


# In production, provide an HttpClient such as httpx_client.layer. Here a test
# client answers with what the API would send.
def messages_api(request: HttpClientRequest):
    message = {
        "id": "msg_1",
        "type": "message",
        "role": "assistant",
        "model": "claude-sonnet-4-5",
        "content": [{"type": "text", "text": "7"}],
        "stop_reason": "end_turn",
        "stop_sequence": None,
        "usage": {"input_tokens": 12, "output_tokens": 1},
    }
    return fx.succeed(response.make(request, json.dumps(message)))


fx.run_main(
    program().pipe(fx.provide(Claude)).pipe(fx.provide(http_client.layer_test(messages_api)))
)

Anthropic

pyfect.ai.anthropic.tool has Anthropic's tools. Anthropic runs web search, web fetch, code execution and tool search, so they need no handler; bash, the text editor, computer use and memory are run by your handler. To keep a large toolkit out of a request's context, annotate its tools with tool.DeferLoading: they are sent with defer_loading, and Claude sees one only once tool search finds it.

anthropic_tools.py
import json
from typing import TypedDict

import pyfect as fx
from pyfect import schema as S
from pyfect.ai import language_model, toolkit
from pyfect.ai import tool as ai_tool
from pyfect.ai.anthropic import client, tool
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.body import Bytes
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact


class Search(TypedDict):
    name: str


ListContacts = ai_tool.make("ListContacts", parameters=S.Struct(Search))

# ListContacts stays out of the request's context until tool search finds it.
Tools = toolkit.make(ListContacts.annotate(tool.DeferLoading, True), tool.ToolSearchBM25_20251119())


@fx.fn
def program():
    reply = yield from language_model.generate_text(
        "Who is in my contacts?", toolkit=Tools, disable_tool_call_resolution=True
    )
    print(reply.text)


Handlers = Tools.to_layer(ListContacts.handler(lambda _search, _ctx: fx.succeed(None)))


Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
    fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)


def messages_api(request: HttpClientRequest):
    assert isinstance(request.body, Bytes)
    for sent in json.loads(request.body.body)["tools"]:
        print(sent["name"], sent.get("defer_loading", False))
    message = {
        "id": "msg_1",
        "type": "message",
        "role": "assistant",
        "model": "claude-sonnet-4-5",
        "content": [{"type": "text", "text": "Anna and Ben."}],
        "stop_reason": "end_turn",
        "stop_sequence": None,
        "usage": {"input_tokens": 12, "output_tokens": 4},
    }
    return fx.succeed(response.make(request, json.dumps(message)))


fx.run_main(
    program()
    .pipe(fx.provide(Handlers))
    .pipe(fx.provide(Claude))
    .pipe(fx.provide(http_client.layer_test(messages_api)))
)

Tools are strict by default on models with structured outputs, and a strict tool's parameters can't be recursive. Set "strict_json_schema": False in config, or annotate the tool with pyfect.ai.tool.Strict set to False, to send such a schema as it is.

OpenAI

pyfect.ai.openai.language_model.model(name, config) takes the Responses API's own request fields as config (store, reasoning, max_output_tokens, ...) and gives back reasoning, citations and OpenAI's metadata on the response parts. pyfect.ai.openai.tool has OpenAI's built-in tools (WebSearch, FileSearch, CodeInterpreter, Mcp, ImageGeneration, ...): OpenAI runs them, so their calls and results arrive in the response and the toolkit's to_layer() takes no handler for them. ApplyPatch, Shell and LocalShell are ones your handler runs. effect.pipe(client.with_web_socket_mode) sends the streams of effect over one WebSocket connection instead of a request each.

openai.py
import json
from typing import Any

import pyfect as fx
from pyfect.ai import language_model, toolkit
from pyfect.ai.openai import client, tool
from pyfect.ai.openai import language_model as openai
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact

# A provider-defined tool runs on OpenAI's side, so its toolkit's layer
# takes no handler.
Search = toolkit.make(tool.WebSearch())


@fx.fn
def program():
    reply = yield from language_model.generate_text("What is Effect?", toolkit=Search)
    for call in reply.tool_calls:
        print(call.name, call.params)
    print(reply.text, reply.finish_reason)


# GPT through OpenAI's Responses API; `store` keeps the response for later turns.
Gpt = openai.model("gpt-5", {"store": True}).layer.pipe(
    fx.layer.provide(client.layer(api_key=redact("sk-...")))
)


# A test client answers with what the Responses API would send.
def responses_api(request: HttpClientRequest):
    action = {"type": "search", "query": "Effect TypeScript"}
    body: dict[str, Any] = {
        "id": "resp_1",
        "object": "response",
        "created_at": 1,
        "model": "gpt-5",
        "status": "completed",
        "output": [
            {"type": "web_search_call", "id": "ws_1", "status": "completed", "action": action},
            {
                "type": "message",
                "id": "msg_1",
                "role": "assistant",
                "status": "completed",
                "content": [
                    {"type": "output_text", "text": "A TypeScript library.", "annotations": []}
                ],
            },
        ],
        "usage": {"input_tokens": 9, "output_tokens": 5, "total_tokens": 14},
    }
    return fx.succeed(response.make(request, json.dumps(body)))


fx.run_main(
    program()
    .pipe(fx.provide(Search.to_layer()))
    .pipe(fx.provide(Gpt))
    .pipe(fx.provide(http_client.layer_test(responses_api)))
)

For the rest of OpenAI's API, pyfect.ai.openai.client_generated.OpenAiClientGenerated is a service with one method per operation (379 of them, such as list_assistants or download_file_stream), generated from Effect's client. It takes the same key, URL and OpenAiConfig transforms as the language model's client, and an operation fails with the HttpClientError, the SchemaError or the OpenAiClientError of a documented status rather than AiError.

OpenRouter

pyfect.ai.openrouter.language_model.model(name, config) reaches any model OpenRouter routes to, named as OpenRouter names it (anthropic/claude-sonnet-4.5). config takes the Chat Completions request's own fields (provider routing, reasoning, plugins, ...). Tool and response schemas are written for the model's family: Anthropic's and OpenAI's strict forms for their models, the default for the others. A generate_object response format is strict only for OpenAI models; for the others it goes with strict unset, as in Effect, and some (Gemini, Claude) then answer in prose and fail with a StructuredOutputError. Set "strict_json_schema": True in config to make it strict for every model. Replayed reasoning is sent once per request, and {"openrouter": {"cacheControl": ...}} in a message's or part's options sets a cache breakpoint. pyfect.ai.openrouter.decision_model provides a DecisionModel over OpenRouter's Decisions API, and the OpenRouterClient service's client field is a generated client of every other OpenRouter operation.

openrouter.py
import json
from typing import Any

import pyfect as fx
from pyfect.ai import language_model
from pyfect.ai.openrouter import client
from pyfect.ai.openrouter import language_model as openrouter
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact


@fx.fn
def program():
    reply = yield from language_model.generate_text("What is Effect?")
    print(reply.reasoning_text)
    print(reply.text, reply.finish_reason)


# Claude through OpenRouter, served by Anthropic only, with a little reasoning.
# `site_title` ranks the app on openrouter.ai.
Claude = openrouter.model(
    "anthropic/claude-sonnet-4.5",
    {
        "provider": {"order": ["anthropic"], "allow_fallbacks": False},
        "reasoning": {"effort": "low"},
    },
).layer.pipe(fx.layer.provide(client.layer(api_key=redact("sk-or-..."), site_title="Docs")))


# A test client answers with what the Chat Completions API would send.
def chat_completions(request: HttpClientRequest):
    body: dict[str, Any] = {
        "id": "gen-1",
        "object": "chat.completion",
        "created": 1,
        "model": "anthropic/claude-sonnet-4.5",
        "system_fingerprint": None,
        "choices": [
            {
                "index": 0,
                "finish_reason": "stop",
                "message": {
                    "role": "assistant",
                    "content": "A TypeScript library.",
                    "reasoning_details": [
                        {"type": "reasoning.text", "text": "Effect is a library."}
                    ],
                },
            }
        ],
        "usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14},
    }
    return fx.succeed(response.make(request, json.dumps(body)))


fx.run_main(
    program().pipe(fx.provide(Claude)).pipe(fx.provide(http_client.layer_test(chat_completions)))
)

language_model.make(generate_text=..., stream_text=...) builds a LanguageModel from two functions of a request's ProviderOptions (its prompt, tools and response format). The examples below use it to script the model's answers, as a test does.

Tools and toolkits

tool.make(name, parameters=..., success=..., failure=...) declares a tool; the model sees its description and its parameters' JSON Schema. toolkit.make(*tools) groups tools, and to_layer(*handlers) provides their handlers: pyright checks GetWeather.handler(f) against the tool's schemas and the services it declares in dependencies.

With toolkit=Tools, generate_text offers the tools, decodes each call's parameters, runs its handler and adds the results to the response. A handler's failure fails generate_text, or, for a tool made with failure_mode="return", goes back to the model as a failure result. tool_choice is "auto", "none", "required", {"tool": name} or {"one_of": names}, and a tool with needs_approval gives an approval request in place of running.

to_model_output=f on tool.make maps a successful result to prompt.TextPart and prompt.FilePart values that the model sees instead of the encoded result, such as a screenshot. The response keeps the typed result. Anthropic and OpenAI send the parts as native content; OpenAI-compatible providers and OpenRouter send them as text, with each file replaced by a placeholder.

text, tool calls, finishtool resultstextgenerate_text(prompt,toolkit=Tools)LanguageModel: a providerdecode the calls' parametersTools handlersGenerateTextResponse
text, tool calls, finishtool resultstextgenerate_text(prompt,toolkit=Tools)LanguageModel: a providerdecode the calls' parametersTools handlersGenerateTextResponse
tools.py
from typing import TypedDict

import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import language_model, tool, toolkit
from pyfect.ai.language_model import LanguageModel, ProviderOptions


class City(TypedDict):
    city: str


class Weather(TypedDict):
    degrees: float


# A tool is a name and schemas; the model sees its JSON Schema.
GetWeather = tool.make(
    "GetWeather",
    description="The current temperature in a city",
    parameters=S.Struct(City),
    success=S.Struct(Weather),
    failure=S.String,
)
Tools = toolkit.make(GetWeather)

# Handlers are a layer: pyright checks each against its tool's schemas.
TEMPERATURES = {"Lisbon": 21.5}
ToolsLive = Tools.to_layer(
    GetWeather.handler(
        lambda params, _ctx: fx.succeed(Weather(degrees=TEMPERATURES[params["city"]]))
    )
)


@fx.fn
def program():
    reply = yield from language_model.generate_text("How warm is Lisbon?", toolkit=Tools)
    for call in reply.tool_calls:
        print("call", call.name, call.params)
    for result in reply.tool_results:
        print("result", result.name, result.result)


# A scripted model, as in a test: it asks for the tool once.
def answer(options: ProviderOptions):
    print("offered", [t.name for t in options.tools])
    call = {"type": "tool-call", "id": "c1", "name": "GetWeather", "params": {"city": "Lisbon"}}
    finish = {
        "type": "finish",
        "reason": "tool-calls",
        "usage": {"inputTokens": {"total": 9}, "outputTokens": {"total": 4}},
    }
    return fx.succeed([call, finish])


Scripted = fx.layer.effect(
    LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)

fx.run_main(program().pipe(fx.provide(ToolsLive)).pipe(fx.provide(Scripted)))

Provider tools

A provider-defined tool is one the provider knows by name, such as those in pyfect.ai.anthropic.tool. Anthropic sends and calls them under its own names (bash, code_execution), while the toolkit and the response parts use the tools' custom names (AnthropicBash, AnthropicCodeExecution); the provider maps between the two.

A client tool, such as Bash_20250124, is run by your handler: give Bash.handler(f) to the toolkit's to_layer, as with your own tools. A handler that needs a service declares it with Bash.add_dependency(Shell), so pyright checks the handler against it and the program must provide it. A server tool, such as CodeExecution_20250825, is run by Anthropic: to_layer() takes no handler for it, and its call and result arrive in the response as parts with provider_executed set. A chat keeps them in the assistant message and sends them back to Anthropic as they came, so the conversation can go on after the tool ran.

provider_tools.py
import json
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any

import pyfect as fx
from pyfect.ai import chat, toolkit
from pyfect.ai import prompt as P
from pyfect.ai.anthropic import client, tool
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.ai.anthropic.tool import BashParameters
from pyfect.ai.tool import HandlerContext
from pyfect.context import Tag
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.body import Bytes
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact


@dataclass(frozen=True)
class ShellService:
    run: Callable[[str], str]


class Shell(Tag[ShellService], key="app/Shell"):
    """Where the bash tool's commands run."""


# Bash is a client tool: your handler runs it, and it may need services it declares.
Bash = tool.Bash_20250124().add_dependency(Shell)
# Code execution is a server tool: Anthropic runs it, so it takes no handler.
Tools = toolkit.make(Bash, tool.CodeExecution_20250825())


@fx.fn
def run_bash(params: BashParameters, _ctx: HandlerContext[str]):
    shell = yield from Shell()
    return shell.run(params["command"])


Handlers = Tools.to_layer(Bash.handler(run_bash))
FakeShell = fx.layer.succeed(Shell, ShellService(run=lambda command: f"ran {command}"))


@fx.fn
def program():
    session = yield from chat.from_prompt(P.empty)
    first = yield from session.generate_text("List the files", toolkit=Tools)
    for call in first.tool_calls:
        print("call:", call.name, call.params)
    for result in first.tool_results:
        print("result:", result.name, result.result)
    second = yield from session.generate_text([], toolkit=Tools)
    for call in second.tool_calls:
        print("call:", call.name, call.params, "run by Anthropic:", call.provider_executed)
    print("text:", second.text)
    yield from session.generate_text("Thanks", toolkit=Tools)


SENT: list[dict[str, Any]] = []
REPLIES: list[list[dict[str, Any]]] = [
    [{"type": "tool_use", "id": "toolu_1", "name": "bash", "input": {"command": "ls"}}],
    [
        {
            "type": "server_tool_use",
            "id": "srvtoolu_1",
            "name": "bash_code_execution",
            "input": {"command": "python -c 'print(17 * 23)'"},
        },
        {
            "type": "bash_code_execution_tool_result",
            "tool_use_id": "srvtoolu_1",
            "content": {
                "type": "bash_code_execution_result",
                "stdout": "391\n",
                "stderr": "",
                "return_code": 0,
                "content": [],
            },
        },
        {"type": "text", "text": "17 * 23 = 391"},
    ],
    [{"type": "text", "text": "You're welcome."}],
]


def messages_api(request: HttpClientRequest):
    assert isinstance(request.body, Bytes)
    SENT.append(json.loads(request.body.body))
    content = REPLIES[len(SENT) - 1]
    message = {
        "id": f"msg_{len(SENT)}",
        "type": "message",
        "role": "assistant",
        "model": "claude-sonnet-4-5",
        "content": content,
        "stop_reason": "tool_use" if content[0]["type"] == "tool_use" else "end_turn",
        "stop_sequence": None,
        "usage": {"input_tokens": 10, "output_tokens": 5},
    }
    return fx.succeed(response.make(request, json.dumps(message)))


Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
    fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)

fx.run_main(
    program()
    .pipe(fx.provide(Handlers))
    .pipe(fx.provide(FakeShell))
    .pipe(fx.provide(Claude))
    .pipe(fx.provide(http_client.layer_test(messages_api)))
)

# The tools go out under Anthropic's names; the last request replays the history,
# the server tool's call and result inside the assistant message, as Anthropic sent them.
tools = [t["name"] for t in SENT[0]["tools"]]
print("tools:", tools)
replayed = [(m["role"], [block["type"] for block in m["content"]]) for m in SENT[-1]["messages"]]
for role, blocks in replayed:
    print(role, blocks)
assert tools == ["bash", "code_execution"]
assert replayed[3] == ("assistant", ["server_tool_use", "bash_code_execution_tool_result", "text"])

Structured output

language_model.generate_object(prompt, schema=...) asks for JSON matching a schema and decodes it into value, typed by the schema. The provider gets the schema's JSON Schema, named object_name; output that does not decode fails with an AiError.

structured.py
from typing import TypedDict

import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import language_model
from pyfect.ai.language_model import JsonFormat, LanguageModel, ProviderOptions


class Contact(TypedDict):
    name: str
    email: str


@fx.fn
def program():
    reply = yield from language_model.generate_object(
        "Ada Lovelace, ada@example.com", schema=S.Struct(Contact), object_name="contact"
    )
    contact = reply.value  # a Contact, decoded by the schema
    print(contact["name"], contact["email"])


# A scripted model: it is asked for JSON matching the schema, and answers with it.
def answer(options: ProviderOptions):
    if isinstance(options.response_format, JsonFormat):
        print("asked for", options.response_format.object_name)
    text = '{"name": "Ada Lovelace", "email": "ada@example.com"}'
    return fx.succeed([{"type": "text", "text": text}])


Scripted = fx.layer.effect(
    LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)

fx.run_main(program().pipe(fx.provide(Scripted)))

Streaming

language_model.stream_text(prompt) is a stream of the response's parts as they arrive: text, reasoning and tool-parameter deltas, tool calls, then their results, then the finish part. With a toolkit, each handler starts as soon as the stream moves past its call.

streaming.py
import pyfect as fx
from pyfect import stream
from pyfect.ai import language_model, response
from pyfect.ai.language_model import EncodedPart, LanguageModel, ProviderOptions


# Print the text as it arrives; the stream ends with a finish part.
def show(part: response.StreamPart[str, object]) -> fx.Effect[None]:
    if isinstance(part, response.TextDeltaPart):
        return fx.sync(lambda: print(part.delta, end="", flush=True))
    if isinstance(part, response.FinishPart):
        return fx.sync(lambda: print(f"\n[{part.reason}]"))
    return fx.void


program = language_model.stream_text("Count to three.").run_for_each(show)


# A scripted model that streams three deltas.
def deltas(_options: ProviderOptions):
    words = ["one, ", "two, ", "three"]
    usage = {"inputTokens": {"total": 5}, "outputTokens": {"total": 3}}
    parts: list[EncodedPart] = [
        {"type": "text-start", "id": "t"},
        *({"type": "text-delta", "id": "t", "delta": word} for word in words),
        {"type": "text-end", "id": "t"},
        {"type": "finish", "reason": "stop", "usage": usage},
    ]
    return stream.from_iterable(parts)


Scripted = fx.layer.effect(
    LanguageModel,
    language_model.make(generate_text=lambda _: fx.succeed([]), stream_text=deltas),
)

fx.run_main(program.pipe(fx.provide(Scripted)))

Conversations

A conversation is a Prompt that grows: concat adds the next user message, and prompt.from_response_parts(reply.content) turns a reply, with its tool calls and results, into the messages that continue it. set_system replaces the system message.

conversation.py
import pyfect as fx
from pyfect import stream
from pyfect.ai import language_model, prompt
from pyfect.ai.language_model import LanguageModel, ProviderOptions


# A conversation is a Prompt that grows: each reply is added back to it.
@fx.fn
def program():
    history = prompt.make("My name is Ada.").set_system("Answer in one line.")
    for question in ["Hello!", "What is my name?"]:
        history = history.concat(question)
        reply = yield from language_model.generate_text(history)
        history = history.concat(prompt.from_response_parts(reply.content))
        print(f"> {question}\n< {reply.text}")
    print([message.role for message in history.content])


# A scripted model that says how much of the conversation it was sent.
def answer(options: ProviderOptions):
    text = f"I was sent {len(options.prompt.content)} messages"
    return fx.succeed([{"type": "text", "text": text}])


Scripted = fx.layer.effect(
    LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)

fx.run_main(program().pipe(fx.provide(Scripted)))

pyfect.ai.chat does this for you. A chat (chat.empty, chat.from_prompt(prompt)) has the same generate_text, generate_object and stream_text; each call sends the history plus the new prompt and adds the reply, and calls on one chat run one at a time. export_json and chat.from_json save and restore a history. chat.Persistence, from chat.layer_persisted(store_id=...) over any persistence backing store, keeps chats by id: get_or_create(id, time_to_live=...) opens one and every call saves it.

chat.py
import pyfect as fx
from pyfect import stream
from pyfect.ai import chat, language_model
from pyfect.ai.language_model import LanguageModel, ProviderOptions
from pyfect.persistence import persistence


# A chat keeps the history: each call sends it and adds the reply to it.
@fx.fn
def program():
    chats = yield from chat.Persistence()
    session = yield from chats.get_or_create("ada")
    for question in ["My name is Ada.", "What is my name?"]:
        reply = yield from session.generate_text(question)
        print(f"> {question}\n< {reply.text}")
    # The store has the chat, so it can be opened again later.
    reopened = yield from chats.get("ada")
    history = yield from reopened.history.get()
    print([message.role for message in history.content])


def answer(options: ProviderOptions):
    text = f"I was sent {len(options.prompt.content)} messages"
    return fx.succeed([{"type": "text", "text": text}])


Scripted = fx.layer.effect(
    LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
Chats = chat.layer_persisted(store_id="chats").pipe(
    fx.layer.provide(persistence.layer_backing_memory)
)

fx.run_main(program().pipe(fx.provide(Scripted)).pipe(fx.provide(Chats)))

A tool loop

One generate_text call is one model call: it runs the tools the model asks for and adds their results to the response, but the model has not seen them yet. When reply.finish_reason == "tool-calls", call the model again with the results. With a chat, the results are already in its history, so the next call sends no new message: session.generate_text([], toolkit=Tools). Stop after a few steps, as a model can keep calling tools.

tool_loop.py
from typing import Any, TypedDict

import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import chat, language_model, tool, toolkit
from pyfect.ai.language_model import LanguageModel, ProviderOptions


class City(TypedDict):
    city: str


GetWeather = tool.make("GetWeather", parameters=S.Struct(City), success=S.String)
Tools = toolkit.make(GetWeather)
ToolsLive = Tools.to_layer(GetWeather.handler(lambda params, _: fx.succeed("21.5 °C")))

MAX_STEPS = 5


@fx.fn
def ask(question: str):
    session = yield from chat.empty
    reply = yield from session.generate_text(question, toolkit=Tools)
    steps = 1
    # The tools ran and their results are in the chat; the model has not seen them yet.
    while reply.finish_reason == "tool-calls" and steps < MAX_STEPS:
        print("step", steps, "called", [call.name for call in reply.tool_calls])
        reply = yield from session.generate_text([], toolkit=Tools)  # no new message
        steps += 1
    print("step", steps, "answered:", reply.text)


# A scripted model: it calls the tool, then answers from the tool's result.
def answer(options: ProviderOptions):
    usage = {"inputTokens": {"total": 9}, "outputTokens": {"total": 4}}
    results = [
        part
        for message in options.prompt.content
        if message.role == "tool"
        for part in message.content
    ]
    if not results:
        call = {"type": "tool-call", "id": "c1", "name": "GetWeather", "params": {"city": "Lisbon"}}
        return fx.succeed([call, {"type": "finish", "reason": "tool-calls", "usage": usage}])
    result: Any = getattr(results[0], "result", None)
    text = f"It is {result} in Lisbon."
    return fx.succeed(
        [{"type": "text", "text": text}, {"type": "finish", "reason": "stop", "usage": usage}]
    )


Scripted = fx.layer.effect(
    LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)

fx.run_main(ask("How warm is Lisbon?").pipe(fx.provide(ToolsLive)).pipe(fx.provide(Scripted)))

An EmbeddingModel's embed(text) and embed_many(texts) give vectors, and concurrent embed calls are sent as one batch; openai.embedding_model.model(name, dimensions=n).layer (or openai_compat's) provides one. Each call runs in a span, such as LanguageModel.generateText, that tracing records.

Tokenizer

A Tokenizer service counts a prompt's tokens for one model, so a program can keep a long conversation within the model's context window. tokenizer.make(tokenize=f) builds one from a function that gives a prompt's token ids; tokenize(prompt) gives them, and truncate(prompt, n) keeps the latest messages whose tokens fit in n. No provider here supplies a tokenizer, so a program brings its own f, such as a local tokenizer library for its model.

tokenizer.py
import pyfect as fx
from pyfect.ai import prompt as P
from pyfect.ai import tokenizer
from pyfect.ai.tokenizer import Tokenizer


# A stand-in tokenizer: one token per word of every text part.
def count_words(prompt: P.Prompt):
    words = [
        word
        for message in prompt.content
        for part in (message.content if isinstance(message.content, tuple) else ())
        if isinstance(part, P.TextPart)
        for word in part.text.split()
    ]
    return fx.succeed(list(range(len(words))))


Words = fx.layer.succeed(Tokenizer, tokenizer.make(tokenize=count_words))

history = [
    {"role": "user", "content": "Plan a three day trip to Lisbon in spring"},
    {"role": "assistant", "content": "Day one: Alfama and the castle"},
    {"role": "user", "content": "Add a day trip"},
]


@fx.fn
def program():
    tokens = yield from Tokenizer()
    total = yield from tokens.tokenize(history)
    print("tokens:", len(total))
    # Keep the latest messages that fit in a budget of 12 tokens.
    kept = yield from tokens.truncate(history, 12)
    print("kept:", [message.role for message in kept.content])
    assert len(total) == 19
    assert [message.role for message in kept.content] == ["assistant", "user"]


fx.run_main(program().pipe(fx.provide(Words)))

ResponseIdTracker is the other side of that problem: a provider that keeps a conversation on its server is sent the previous response's id and only the new messages. Only OpenAI's Responses language model reads it, and its WebSocket mode (client.with_web_socket_mode or client.layer_web_socket_mode()) provides it, so a program doesn't provide one itself. Anthropic's Messages API keeps no conversation state, so Anthropic ignores it.

Decisions

A DecisionModel answers judgments instead of writing text. decision.classify picks one of its labels, decision.rate places the input on ordered levels and decision.probability gives the chance that a statement is true; decision.make(input=schema, decisions={...}) names them and the schema encodes the input as JSON. decision_model.decide(definition, input=...) asks for every answer in one call. Look an answer up with its decision object, answers[team], and its label has that decision's literal type. A missing, refused or invalid answer fails with an AiError; decide_settled instead gives each decision an Ok answer or an Err(Unanswered). A provider's classify or rate answer must give a probability for every label (in [0, 1], summing to 1 within its rounding); one that leaves a label out is invalid, as in Effect.

An image goes in the input as a field of type decision.Image, holding a decision_model.Image(media_type, data). The provider gets each distinct image once, and a reference string replaces it in the state. Instructions name an image by its field path, as in `photo`. decision.locate(image="photo", instructions="the mug") asks where something is in that image and answers with a BoundingBox (edges from 0 to 1). Providers without image or locate support fail with InvalidUserInputError. TypeSafe supports both; OpenAI and Cloudflare take images but not locate.

pyfect.ai.openai.decision_model, pyfect.ai.openrouter.decision_model, pyfect.ai.cloudflare.decision_model and pyfect.ai.typesafe.decision_model provide one. decision_model.make(decide=...) builds one from a function of a call's ProviderOptions, as a test does. Its answers are plain dicts tagged by _tag ("Classify", "Rate", "Probability", "Locate" or "Refusal"), as in Effect:

decisions.py
from typing import TypedDict

import pyfect as fx
from pyfect import schema as S
from pyfect.ai import decision, decision_model


class Ticket(TypedDict):
    message: str


team = decision.classify(
    instructions="Which team should handle this?",
    criteria={"billing": "Payments and refunds", "technical": "Outages and bugs"},
)
severity = decision.rate(instructions="How severe is it?", criteria=["minor", "major"])
urgent = decision.probability(instructions="Does this need attention now?")
triage = decision.make(
    input=S.Struct(Ticket), decisions={"team": team, "severity": severity, "urgent": urgent}
)


@fx.fn
def program():
    result = yield from decision_model.decide(triage, input={"message": "Checkout is down"})
    answers = result.answers
    # Looked up by its decision, an answer's label is one of that decision's labels.
    print(answers[team].label, answers[severity].label, answers[urgent].probability)


# A provider such as openai.decision_model.model("gpt-6-luna") answers in production.
# Here a test DecisionModel answers each named decision; its answers are checked
# (distributions must sum to 1) before the program sees them.
def answer(options: decision_model.ProviderOptions):
    return fx.succeed(
        decision_model.ProviderResponse(
            {
                "team": {
                    "_tag": "Classify",
                    "label": "technical",
                    "probabilities": {"billing": 0.2, "technical": 0.8},
                },
                "severity": {
                    "_tag": "Rate",
                    "rating": 0.9,
                    "probabilities": {"minor": 0.1, "major": 0.9},
                },
                "urgent": {"_tag": "Probability", "probability": 0.97},
            }
        )
    )


TestDecisions = fx.layer.effect(decision_model.DecisionModel, decision_model.make(decide=answer))

fx.run_main(program().pipe(fx.provide(TestDecisions)))

Reference: pyfect.ai, language_model, tool, toolkit, tokenizer, pyfect.ai.anthropic, pyfect.ai.openai, pyfect.ai.openrouter, decision, decision_model, pyfect.ai.openai_compat, pyfect.ai.cloudflare, pyfect.ai.typesafe.