Generate text and objects, call tools and stream responses through a LanguageModel service any provider can fill.
pyfect.ai follows Effect's effect/ai. A program asks the LanguageModel service for
text, a decoded object or a stream, and a provider's layer supplies the service. So the
same program runs on Claude, on any OpenAI-compatible API, or on a scripted model in a
test. Failures are an AiError whose reason says what went wrong (a rate limit, an
invalid request, output that does not match its schema). A request that never reached the
provider is a NetworkError with its method and url; its cause is the
HttpClientError it failed with. Errors carry the response and its body, never the
request's headers.
Language models and providers#
language_model.generate_text(prompt) needs a LanguageModel and gives a
GenerateTextResponse: its content parts, with text, tool_calls, tool_results,
finish_reason and usage. A prompt is a string, a list of messages or a Prompt.
pyfect.ai.anthropic calls Anthropic's Messages API, pyfect.ai.openai OpenAI's
Responses API and pyfect.ai.openai_compat any Chat Completions API (a local server, a
gateway). Each has a client layer, with the API key as a Redacted (or from Config
with layer_config), and a language_model whose model(name).layer provides the
LanguageModel. They all send their requests through
the HttpClient service, so they need no vendor SDK: provide
httpx_client.layer (the pyfect-dev[httpx] extra) in production, or a test client, as here.
import json
import pyfect as fx
from pyfect.ai import language_model
from pyfect.ai.anthropic import client
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact
# The program needs a LanguageModel, from any provider.
@fx.fn
def program():
reply = yield from language_model.generate_text("Name a prime number.")
print(reply.text, reply.finish_reason, reply.usage.output_tokens.total)
# Claude through Anthropic's Messages API, over the HttpClient service.
Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)
# In production, provide an HttpClient such as httpx_client.layer. Here a test
# client answers with what the API would send.
def messages_api(request: HttpClientRequest):
message = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "7"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 1},
}
return fx.succeed(response.make(request, json.dumps(message)))
fx.run_main(
program().pipe(fx.provide(Claude)).pipe(fx.provide(http_client.layer_test(messages_api)))
)Anthropic#
pyfect.ai.anthropic.tool has Anthropic's tools. Anthropic runs web search, web fetch, code
execution and tool search, so they need no handler; bash, the text editor, computer use and
memory are run by your handler. To keep a large toolkit out of a request's context, annotate
its tools with tool.DeferLoading: they are sent with defer_loading, and Claude sees one
only once tool search finds it.
import json
from typing import TypedDict
import pyfect as fx
from pyfect import schema as S
from pyfect.ai import language_model, toolkit
from pyfect.ai import tool as ai_tool
from pyfect.ai.anthropic import client, tool
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.body import Bytes
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact
class Search(TypedDict):
name: str
ListContacts = ai_tool.make("ListContacts", parameters=S.Struct(Search))
# ListContacts stays out of the request's context until tool search finds it.
Tools = toolkit.make(ListContacts.annotate(tool.DeferLoading, True), tool.ToolSearchBM25_20251119())
@fx.fn
def program():
reply = yield from language_model.generate_text(
"Who is in my contacts?", toolkit=Tools, disable_tool_call_resolution=True
)
print(reply.text)
Handlers = Tools.to_layer(ListContacts.handler(lambda _search, _ctx: fx.succeed(None)))
Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)
def messages_api(request: HttpClientRequest):
assert isinstance(request.body, Bytes)
for sent in json.loads(request.body.body)["tools"]:
print(sent["name"], sent.get("defer_loading", False))
message = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "Anna and Ben."}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 4},
}
return fx.succeed(response.make(request, json.dumps(message)))
fx.run_main(
program()
.pipe(fx.provide(Handlers))
.pipe(fx.provide(Claude))
.pipe(fx.provide(http_client.layer_test(messages_api)))
)Tools are strict by default on models with structured outputs, and a strict tool's
parameters can't be recursive. Set "strict_json_schema": False in config, or annotate
the tool with pyfect.ai.tool.Strict set to False, to send such a schema as it is.
OpenAI#
pyfect.ai.openai.language_model.model(name, config) takes the Responses API's own
request fields as config (store, reasoning, max_output_tokens, ...) and gives back
reasoning, citations and OpenAI's metadata on the response parts. pyfect.ai.openai.tool
has OpenAI's built-in tools (WebSearch, FileSearch, CodeInterpreter, Mcp,
ImageGeneration, ...): OpenAI runs them, so their calls and results arrive in the
response and the toolkit's to_layer() takes no handler for them. ApplyPatch, Shell and LocalShell are ones your handler
runs. effect.pipe(client.with_web_socket_mode) sends the streams of effect over one
WebSocket connection instead of a request each.
import json
from typing import Any
import pyfect as fx
from pyfect.ai import language_model, toolkit
from pyfect.ai.openai import client, tool
from pyfect.ai.openai import language_model as openai
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact
# A provider-defined tool runs on OpenAI's side, so its toolkit's layer
# takes no handler.
Search = toolkit.make(tool.WebSearch())
@fx.fn
def program():
reply = yield from language_model.generate_text("What is Effect?", toolkit=Search)
for call in reply.tool_calls:
print(call.name, call.params)
print(reply.text, reply.finish_reason)
# GPT through OpenAI's Responses API; `store` keeps the response for later turns.
Gpt = openai.model("gpt-5", {"store": True}).layer.pipe(
fx.layer.provide(client.layer(api_key=redact("sk-...")))
)
# A test client answers with what the Responses API would send.
def responses_api(request: HttpClientRequest):
action = {"type": "search", "query": "Effect TypeScript"}
body: dict[str, Any] = {
"id": "resp_1",
"object": "response",
"created_at": 1,
"model": "gpt-5",
"status": "completed",
"output": [
{"type": "web_search_call", "id": "ws_1", "status": "completed", "action": action},
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [
{"type": "output_text", "text": "A TypeScript library.", "annotations": []}
],
},
],
"usage": {"input_tokens": 9, "output_tokens": 5, "total_tokens": 14},
}
return fx.succeed(response.make(request, json.dumps(body)))
fx.run_main(
program()
.pipe(fx.provide(Search.to_layer()))
.pipe(fx.provide(Gpt))
.pipe(fx.provide(http_client.layer_test(responses_api)))
)For the rest of OpenAI's API, pyfect.ai.openai.client_generated.OpenAiClientGenerated is a
service with one method per operation (379 of them, such as list_assistants or
download_file_stream), generated from Effect's client. It takes the same key, URL and
OpenAiConfig transforms as the language model's client, and an operation fails with the
HttpClientError, the SchemaError or the OpenAiClientError of a documented status rather
than AiError.
OpenRouter#
pyfect.ai.openrouter.language_model.model(name, config) reaches any model OpenRouter
routes to, named as OpenRouter names it (anthropic/claude-sonnet-4.5). config takes the
Chat Completions request's own fields (provider routing, reasoning, plugins, ...).
Tool and response schemas are written for the model's family: Anthropic's and OpenAI's
strict forms for their models, the default for the others. A generate_object response
format is strict only for OpenAI models; for the others it goes with strict unset, as in
Effect, and some (Gemini, Claude) then answer in prose and fail with a
StructuredOutputError. Set "strict_json_schema": True in config to make it strict for
every model. Replayed reasoning is sent once
per request, and {"openrouter": {"cacheControl": ...}} in a message's or part's options
sets a cache breakpoint. pyfect.ai.openrouter.decision_model provides a DecisionModel
over OpenRouter's Decisions API, and the OpenRouterClient service's client field is a
generated client of every other OpenRouter operation.
import json
from typing import Any
import pyfect as fx
from pyfect.ai import language_model
from pyfect.ai.openrouter import client
from pyfect.ai.openrouter import language_model as openrouter
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact
@fx.fn
def program():
reply = yield from language_model.generate_text("What is Effect?")
print(reply.reasoning_text)
print(reply.text, reply.finish_reason)
# Claude through OpenRouter, served by Anthropic only, with a little reasoning.
# `site_title` ranks the app on openrouter.ai.
Claude = openrouter.model(
"anthropic/claude-sonnet-4.5",
{
"provider": {"order": ["anthropic"], "allow_fallbacks": False},
"reasoning": {"effort": "low"},
},
).layer.pipe(fx.layer.provide(client.layer(api_key=redact("sk-or-..."), site_title="Docs")))
# A test client answers with what the Chat Completions API would send.
def chat_completions(request: HttpClientRequest):
body: dict[str, Any] = {
"id": "gen-1",
"object": "chat.completion",
"created": 1,
"model": "anthropic/claude-sonnet-4.5",
"system_fingerprint": None,
"choices": [
{
"index": 0,
"finish_reason": "stop",
"message": {
"role": "assistant",
"content": "A TypeScript library.",
"reasoning_details": [
{"type": "reasoning.text", "text": "Effect is a library."}
],
},
}
],
"usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14},
}
return fx.succeed(response.make(request, json.dumps(body)))
fx.run_main(
program().pipe(fx.provide(Claude)).pipe(fx.provide(http_client.layer_test(chat_completions)))
)language_model.make(generate_text=..., stream_text=...) builds a LanguageModel from two
functions of a request's ProviderOptions (its prompt, tools and response format). The
examples below use it to script the model's answers, as a test does.
Tools and toolkits#
tool.make(name, parameters=..., success=..., failure=...) declares a tool; the model
sees its description and its parameters' JSON Schema. toolkit.make(*tools) groups tools,
and to_layer(*handlers) provides their handlers: pyright checks GetWeather.handler(f)
against the tool's schemas and the services it declares in dependencies.
With toolkit=Tools, generate_text offers the tools, decodes each call's parameters,
runs its handler and adds the results to the response. A handler's failure fails
generate_text, or, for a tool made with failure_mode="return", goes back to the model
as a failure result. tool_choice is "auto", "none", "required", {"tool": name} or
{"one_of": names}, and a tool with needs_approval gives an approval request in place
of running.
to_model_output=f on tool.make maps a successful result to prompt.TextPart and
prompt.FilePart values that the model sees instead of the encoded result, such as a
screenshot. The response keeps the typed result. Anthropic and OpenAI send the parts as
native content; OpenAI-compatible providers and OpenRouter send them as text, with each
file replaced by a placeholder.
from typing import TypedDict
import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import language_model, tool, toolkit
from pyfect.ai.language_model import LanguageModel, ProviderOptions
class City(TypedDict):
city: str
class Weather(TypedDict):
degrees: float
# A tool is a name and schemas; the model sees its JSON Schema.
GetWeather = tool.make(
"GetWeather",
description="The current temperature in a city",
parameters=S.Struct(City),
success=S.Struct(Weather),
failure=S.String,
)
Tools = toolkit.make(GetWeather)
# Handlers are a layer: pyright checks each against its tool's schemas.
TEMPERATURES = {"Lisbon": 21.5}
ToolsLive = Tools.to_layer(
GetWeather.handler(
lambda params, _ctx: fx.succeed(Weather(degrees=TEMPERATURES[params["city"]]))
)
)
@fx.fn
def program():
reply = yield from language_model.generate_text("How warm is Lisbon?", toolkit=Tools)
for call in reply.tool_calls:
print("call", call.name, call.params)
for result in reply.tool_results:
print("result", result.name, result.result)
# A scripted model, as in a test: it asks for the tool once.
def answer(options: ProviderOptions):
print("offered", [t.name for t in options.tools])
call = {"type": "tool-call", "id": "c1", "name": "GetWeather", "params": {"city": "Lisbon"}}
finish = {
"type": "finish",
"reason": "tool-calls",
"usage": {"inputTokens": {"total": 9}, "outputTokens": {"total": 4}},
}
return fx.succeed([call, finish])
Scripted = fx.layer.effect(
LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
fx.run_main(program().pipe(fx.provide(ToolsLive)).pipe(fx.provide(Scripted)))Provider tools#
A provider-defined tool is one the provider knows by name, such as those in
pyfect.ai.anthropic.tool. Anthropic sends and calls them under its own names (bash,
code_execution), while the toolkit and the response parts use the tools' custom names
(AnthropicBash, AnthropicCodeExecution); the provider maps between the two.
A client tool, such as Bash_20250124, is run by your handler: give Bash.handler(f) to the
toolkit's to_layer, as with your own tools. A handler that needs a service declares it with
Bash.add_dependency(Shell), so pyright checks the handler against it and the program must
provide it. A server tool, such as CodeExecution_20250825, is run by Anthropic: to_layer()
takes no handler for it, and its call and result arrive in the response as parts with
provider_executed set. A chat keeps them in the assistant message and sends them back to
Anthropic as they came, so the conversation can go on after the tool ran.
import json
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
import pyfect as fx
from pyfect.ai import chat, toolkit
from pyfect.ai import prompt as P
from pyfect.ai.anthropic import client, tool
from pyfect.ai.anthropic import language_model as anthropic
from pyfect.ai.anthropic.tool import BashParameters
from pyfect.ai.tool import HandlerContext
from pyfect.context import Tag
from pyfect.http import client as http_client
from pyfect.http import response
from pyfect.http.body import Bytes
from pyfect.http.request import HttpClientRequest
from pyfect.redacted import make as redact
@dataclass(frozen=True)
class ShellService:
run: Callable[[str], str]
class Shell(Tag[ShellService], key="app/Shell"):
"""Where the bash tool's commands run."""
# Bash is a client tool: your handler runs it, and it may need services it declares.
Bash = tool.Bash_20250124().add_dependency(Shell)
# Code execution is a server tool: Anthropic runs it, so it takes no handler.
Tools = toolkit.make(Bash, tool.CodeExecution_20250825())
@fx.fn
def run_bash(params: BashParameters, _ctx: HandlerContext[str]):
shell = yield from Shell()
return shell.run(params["command"])
Handlers = Tools.to_layer(Bash.handler(run_bash))
FakeShell = fx.layer.succeed(Shell, ShellService(run=lambda command: f"ran {command}"))
@fx.fn
def program():
session = yield from chat.from_prompt(P.empty)
first = yield from session.generate_text("List the files", toolkit=Tools)
for call in first.tool_calls:
print("call:", call.name, call.params)
for result in first.tool_results:
print("result:", result.name, result.result)
second = yield from session.generate_text([], toolkit=Tools)
for call in second.tool_calls:
print("call:", call.name, call.params, "run by Anthropic:", call.provider_executed)
print("text:", second.text)
yield from session.generate_text("Thanks", toolkit=Tools)
SENT: list[dict[str, Any]] = []
REPLIES: list[list[dict[str, Any]]] = [
[{"type": "tool_use", "id": "toolu_1", "name": "bash", "input": {"command": "ls"}}],
[
{
"type": "server_tool_use",
"id": "srvtoolu_1",
"name": "bash_code_execution",
"input": {"command": "python -c 'print(17 * 23)'"},
},
{
"type": "bash_code_execution_tool_result",
"tool_use_id": "srvtoolu_1",
"content": {
"type": "bash_code_execution_result",
"stdout": "391\n",
"stderr": "",
"return_code": 0,
"content": [],
},
},
{"type": "text", "text": "17 * 23 = 391"},
],
[{"type": "text", "text": "You're welcome."}],
]
def messages_api(request: HttpClientRequest):
assert isinstance(request.body, Bytes)
SENT.append(json.loads(request.body.body))
content = REPLIES[len(SENT) - 1]
message = {
"id": f"msg_{len(SENT)}",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": content,
"stop_reason": "tool_use" if content[0]["type"] == "tool_use" else "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
}
return fx.succeed(response.make(request, json.dumps(message)))
Claude = anthropic.model("claude-sonnet-4-5").layer.pipe(
fx.layer.provide(client.layer(api_key=redact("sk-ant-...")))
)
fx.run_main(
program()
.pipe(fx.provide(Handlers))
.pipe(fx.provide(FakeShell))
.pipe(fx.provide(Claude))
.pipe(fx.provide(http_client.layer_test(messages_api)))
)
# The tools go out under Anthropic's names; the last request replays the history,
# the server tool's call and result inside the assistant message, as Anthropic sent them.
tools = [t["name"] for t in SENT[0]["tools"]]
print("tools:", tools)
replayed = [(m["role"], [block["type"] for block in m["content"]]) for m in SENT[-1]["messages"]]
for role, blocks in replayed:
print(role, blocks)
assert tools == ["bash", "code_execution"]
assert replayed[3] == ("assistant", ["server_tool_use", "bash_code_execution_tool_result", "text"])Structured output#
language_model.generate_object(prompt, schema=...) asks for JSON matching a
schema and decodes it into value, typed by the schema. The provider gets
the schema's JSON Schema, named object_name; output that does not decode fails with an
AiError.
from typing import TypedDict
import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import language_model
from pyfect.ai.language_model import JsonFormat, LanguageModel, ProviderOptions
class Contact(TypedDict):
name: str
email: str
@fx.fn
def program():
reply = yield from language_model.generate_object(
"Ada Lovelace, ada@example.com", schema=S.Struct(Contact), object_name="contact"
)
contact = reply.value # a Contact, decoded by the schema
print(contact["name"], contact["email"])
# A scripted model: it is asked for JSON matching the schema, and answers with it.
def answer(options: ProviderOptions):
if isinstance(options.response_format, JsonFormat):
print("asked for", options.response_format.object_name)
text = '{"name": "Ada Lovelace", "email": "ada@example.com"}'
return fx.succeed([{"type": "text", "text": text}])
Scripted = fx.layer.effect(
LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
fx.run_main(program().pipe(fx.provide(Scripted)))Streaming#
language_model.stream_text(prompt) is a stream of the response's
parts as they arrive: text, reasoning and tool-parameter deltas, tool calls, then their
results, then the finish part. With a toolkit, each handler starts as soon as the stream
moves past its call.
import pyfect as fx
from pyfect import stream
from pyfect.ai import language_model, response
from pyfect.ai.language_model import EncodedPart, LanguageModel, ProviderOptions
# Print the text as it arrives; the stream ends with a finish part.
def show(part: response.StreamPart[str, object]) -> fx.Effect[None]:
if isinstance(part, response.TextDeltaPart):
return fx.sync(lambda: print(part.delta, end="", flush=True))
if isinstance(part, response.FinishPart):
return fx.sync(lambda: print(f"\n[{part.reason}]"))
return fx.void
program = language_model.stream_text("Count to three.").run_for_each(show)
# A scripted model that streams three deltas.
def deltas(_options: ProviderOptions):
words = ["one, ", "two, ", "three"]
usage = {"inputTokens": {"total": 5}, "outputTokens": {"total": 3}}
parts: list[EncodedPart] = [
{"type": "text-start", "id": "t"},
*({"type": "text-delta", "id": "t", "delta": word} for word in words),
{"type": "text-end", "id": "t"},
{"type": "finish", "reason": "stop", "usage": usage},
]
return stream.from_iterable(parts)
Scripted = fx.layer.effect(
LanguageModel,
language_model.make(generate_text=lambda _: fx.succeed([]), stream_text=deltas),
)
fx.run_main(program.pipe(fx.provide(Scripted)))Conversations#
A conversation is a Prompt that grows: concat adds the next user message, and
prompt.from_response_parts(reply.content) turns a reply, with its tool calls and
results, into the messages that continue it. set_system replaces the system message.
import pyfect as fx
from pyfect import stream
from pyfect.ai import language_model, prompt
from pyfect.ai.language_model import LanguageModel, ProviderOptions
# A conversation is a Prompt that grows: each reply is added back to it.
@fx.fn
def program():
history = prompt.make("My name is Ada.").set_system("Answer in one line.")
for question in ["Hello!", "What is my name?"]:
history = history.concat(question)
reply = yield from language_model.generate_text(history)
history = history.concat(prompt.from_response_parts(reply.content))
print(f"> {question}\n< {reply.text}")
print([message.role for message in history.content])
# A scripted model that says how much of the conversation it was sent.
def answer(options: ProviderOptions):
text = f"I was sent {len(options.prompt.content)} messages"
return fx.succeed([{"type": "text", "text": text}])
Scripted = fx.layer.effect(
LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
fx.run_main(program().pipe(fx.provide(Scripted)))pyfect.ai.chat does this for you. A chat (chat.empty, chat.from_prompt(prompt))
has the same generate_text, generate_object and stream_text; each call sends the
history plus the new prompt and adds the reply, and calls on one chat run one at a time.
export_json and chat.from_json save and restore a history. chat.Persistence, from
chat.layer_persisted(store_id=...) over any persistence backing
store, keeps chats by id: get_or_create(id, time_to_live=...) opens one and every call
saves it.
import pyfect as fx
from pyfect import stream
from pyfect.ai import chat, language_model
from pyfect.ai.language_model import LanguageModel, ProviderOptions
from pyfect.persistence import persistence
# A chat keeps the history: each call sends it and adds the reply to it.
@fx.fn
def program():
chats = yield from chat.Persistence()
session = yield from chats.get_or_create("ada")
for question in ["My name is Ada.", "What is my name?"]:
reply = yield from session.generate_text(question)
print(f"> {question}\n< {reply.text}")
# The store has the chat, so it can be opened again later.
reopened = yield from chats.get("ada")
history = yield from reopened.history.get()
print([message.role for message in history.content])
def answer(options: ProviderOptions):
text = f"I was sent {len(options.prompt.content)} messages"
return fx.succeed([{"type": "text", "text": text}])
Scripted = fx.layer.effect(
LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
Chats = chat.layer_persisted(store_id="chats").pipe(
fx.layer.provide(persistence.layer_backing_memory)
)
fx.run_main(program().pipe(fx.provide(Scripted)).pipe(fx.provide(Chats)))A tool loop#
One generate_text call is one model call: it runs the tools the model asks for and adds
their results to the response, but the model has not seen them yet. When
reply.finish_reason == "tool-calls", call the model again with the results. With a chat,
the results are already in its history, so the next call sends no new message:
session.generate_text([], toolkit=Tools). Stop after a few steps, as a model can keep
calling tools.
from typing import Any, TypedDict
import pyfect as fx
from pyfect import schema as S
from pyfect import stream
from pyfect.ai import chat, language_model, tool, toolkit
from pyfect.ai.language_model import LanguageModel, ProviderOptions
class City(TypedDict):
city: str
GetWeather = tool.make("GetWeather", parameters=S.Struct(City), success=S.String)
Tools = toolkit.make(GetWeather)
ToolsLive = Tools.to_layer(GetWeather.handler(lambda params, _: fx.succeed("21.5 °C")))
MAX_STEPS = 5
@fx.fn
def ask(question: str):
session = yield from chat.empty
reply = yield from session.generate_text(question, toolkit=Tools)
steps = 1
# The tools ran and their results are in the chat; the model has not seen them yet.
while reply.finish_reason == "tool-calls" and steps < MAX_STEPS:
print("step", steps, "called", [call.name for call in reply.tool_calls])
reply = yield from session.generate_text([], toolkit=Tools) # no new message
steps += 1
print("step", steps, "answered:", reply.text)
# A scripted model: it calls the tool, then answers from the tool's result.
def answer(options: ProviderOptions):
usage = {"inputTokens": {"total": 9}, "outputTokens": {"total": 4}}
results = [
part
for message in options.prompt.content
if message.role == "tool"
for part in message.content
]
if not results:
call = {"type": "tool-call", "id": "c1", "name": "GetWeather", "params": {"city": "Lisbon"}}
return fx.succeed([call, {"type": "finish", "reason": "tool-calls", "usage": usage}])
result: Any = getattr(results[0], "result", None)
text = f"It is {result} in Lisbon."
return fx.succeed(
[{"type": "text", "text": text}, {"type": "finish", "reason": "stop", "usage": usage}]
)
Scripted = fx.layer.effect(
LanguageModel, language_model.make(generate_text=answer, stream_text=lambda _: stream.empty)
)
fx.run_main(ask("How warm is Lisbon?").pipe(fx.provide(ToolsLive)).pipe(fx.provide(Scripted)))An EmbeddingModel's embed(text) and embed_many(texts) give vectors, and
concurrent embed calls are sent as one batch;
openai.embedding_model.model(name, dimensions=n).layer (or openai_compat's) provides one. Each call
runs in a span, such as LanguageModel.generateText, that
tracing records.
Tokenizer#
A Tokenizer service counts a prompt's tokens for one model, so a program can keep a long
conversation within the model's context window. tokenizer.make(tokenize=f) builds one from
a function that gives a prompt's token ids; tokenize(prompt) gives them, and
truncate(prompt, n) keeps the latest messages whose tokens fit in n. No provider here
supplies a tokenizer, so a program brings its own f, such as a local tokenizer library
for its model.
import pyfect as fx
from pyfect.ai import prompt as P
from pyfect.ai import tokenizer
from pyfect.ai.tokenizer import Tokenizer
# A stand-in tokenizer: one token per word of every text part.
def count_words(prompt: P.Prompt):
words = [
word
for message in prompt.content
for part in (message.content if isinstance(message.content, tuple) else ())
if isinstance(part, P.TextPart)
for word in part.text.split()
]
return fx.succeed(list(range(len(words))))
Words = fx.layer.succeed(Tokenizer, tokenizer.make(tokenize=count_words))
history = [
{"role": "user", "content": "Plan a three day trip to Lisbon in spring"},
{"role": "assistant", "content": "Day one: Alfama and the castle"},
{"role": "user", "content": "Add a day trip"},
]
@fx.fn
def program():
tokens = yield from Tokenizer()
total = yield from tokens.tokenize(history)
print("tokens:", len(total))
# Keep the latest messages that fit in a budget of 12 tokens.
kept = yield from tokens.truncate(history, 12)
print("kept:", [message.role for message in kept.content])
assert len(total) == 19
assert [message.role for message in kept.content] == ["assistant", "user"]
fx.run_main(program().pipe(fx.provide(Words)))ResponseIdTracker is the other side of that problem: a provider that keeps a conversation
on its server is sent the previous response's id and only the new messages. Only OpenAI's
Responses language model reads it, and its WebSocket mode (client.with_web_socket_mode or
client.layer_web_socket_mode()) provides it, so a program doesn't provide one itself.
Anthropic's Messages API keeps no conversation state, so Anthropic ignores it.
Decisions#
A DecisionModel answers judgments instead of writing text. decision.classify picks one
of its labels, decision.rate places the input on ordered levels and decision.probability
gives the chance that a statement is true; decision.make(input=schema, decisions={...})
names them and the schema encodes the input as JSON. decision_model.decide(definition, input=...) asks for every answer in one call. Look an answer up with its decision object,
answers[team], and its label has that decision's literal type. A missing, refused or
invalid answer fails with an AiError; decide_settled instead gives each decision an
Ok answer or an Err(Unanswered). A provider's classify or rate answer must give a
probability for every label (in [0, 1], summing to 1 within its rounding); one that leaves a
label out is invalid, as in Effect.
An image goes in the input as a field of type decision.Image, holding a
decision_model.Image(media_type, data). The provider gets each distinct image once, and
a reference string replaces it in the state. Instructions name an image by its field
path, as in `photo`. decision.locate(image="photo", instructions="the mug") asks
where something is in that image and answers with a BoundingBox (edges from 0 to 1).
Providers without image or locate support fail with InvalidUserInputError. TypeSafe
supports both; OpenAI and Cloudflare take images but not locate.
pyfect.ai.openai.decision_model, pyfect.ai.openrouter.decision_model,
pyfect.ai.cloudflare.decision_model and pyfect.ai.typesafe.decision_model provide one.
decision_model.make(decide=...) builds one from a function of a call's ProviderOptions,
as a test does. Its answers are plain dicts tagged by _tag ("Classify", "Rate",
"Probability", "Locate" or "Refusal"), as in Effect:
from typing import TypedDict
import pyfect as fx
from pyfect import schema as S
from pyfect.ai import decision, decision_model
class Ticket(TypedDict):
message: str
team = decision.classify(
instructions="Which team should handle this?",
criteria={"billing": "Payments and refunds", "technical": "Outages and bugs"},
)
severity = decision.rate(instructions="How severe is it?", criteria=["minor", "major"])
urgent = decision.probability(instructions="Does this need attention now?")
triage = decision.make(
input=S.Struct(Ticket), decisions={"team": team, "severity": severity, "urgent": urgent}
)
@fx.fn
def program():
result = yield from decision_model.decide(triage, input={"message": "Checkout is down"})
answers = result.answers
# Looked up by its decision, an answer's label is one of that decision's labels.
print(answers[team].label, answers[severity].label, answers[urgent].probability)
# A provider such as openai.decision_model.model("gpt-6-luna") answers in production.
# Here a test DecisionModel answers each named decision; its answers are checked
# (distributions must sum to 1) before the program sees them.
def answer(options: decision_model.ProviderOptions):
return fx.succeed(
decision_model.ProviderResponse(
{
"team": {
"_tag": "Classify",
"label": "technical",
"probabilities": {"billing": 0.2, "technical": 0.8},
},
"severity": {
"_tag": "Rate",
"rating": 0.9,
"probabilities": {"minor": 0.1, "major": 0.9},
},
"urgent": {"_tag": "Probability", "probability": 0.97},
}
)
)
TestDecisions = fx.layer.effect(decision_model.DecisionModel, decision_model.make(decide=answer))
fx.run_main(program().pipe(fx.provide(TestDecisions)))Reference: pyfect.ai,
language_model,
tool, toolkit,
tokenizer,
pyfect.ai.anthropic,
pyfect.ai.openai,
pyfect.ai.openrouter,
decision,
decision_model,
pyfect.ai.openai_compat,
pyfect.ai.cloudflare,
pyfect.ai.typesafe.