GitHub

language_model

API reference for language_model.

Generating text, objects and streams with a language model (Effect's LanguageModel).

A provider implements make's two functions, from ProviderOptions to encoded response parts; the service decodes them, runs the tool calls the model made with the toolkit's handlers and returns the parts::

response = yield from language_model.generate_text(
    "What is the weather in Paris?", toolkit=toolkit
)
for part in response.tool_results:
    match part:
        case ToolResultSuccess(name="GetWeather", result=weather): ...

generate_object asks for JSON and decodes it with a schema; stream_text streams the parts as they come, the tool results after the model's parts.

Re-exports

Types

TypeDefinitionDescription
ToolChoicetype ToolChoice = Literal['auto', 'none', 'required'] | ToolChoiceTool | ToolChoiceOneOf

Which tools the model may call: any ("auto"), none, at least one, {"tool": name} or {"one_of": names, "mode": ...}.

ToolCallValidationtype ToolCallValidation = Literal['strict', 'deferred']

How tool calls are checked with disable_tool_call_resolution: "strict" decodes each against its tool; "deferred" returns them as the provider sent them, calls to tools outside the toolkit included (a provider-executed call is still checked).

EncodedParttype EncodedPart = Mapping[str, object]
GenerateFntype GenerateFn = Callable[[ProviderOptions], Effect[Sequence[EncodedPart], AiError, Any]]
StreamFntype StreamFn = Callable[[ProviderOptions], Stream[EncodedPart, AiError, Any]]

Classes

Functions

default_codec_transformer

default_codec_transformer(schema) -> tuple[S.Top, JsonSchema]

schema itself and its JSON Schema, top-level $refs resolved.

ParameterTypeDescription
schemaS.Top

Returns tuple[S.Top, JsonSchema]

def default_codec_transformer(schema: S.Top) -> tuple[S.Top, JsonSchema]:
    """``schema`` itself and its JSON Schema, top-level ``$ref``s
    resolved."""
    document = S.to_json_schema_document(schema)
    root: JsonSchema = dict(resolve_root(document))
    if document.definitions:
        root["$defs"] = dict(document.definitions)
    return schema, root

get_object_name

get_object_name(object_name, schema) -> str

object_name, else the schema's identifier, else "generateObject".

ParameterTypeDescription
object_namestr | None
schemaS.Top

Returns str

def get_object_name(object_name: str | None, schema: S.Top) -> str:
    """``object_name``, else the schema's identifier, else
    ``"generateObject"``."""
    if object_name is not None:
        return object_name
    identifier = (schema.__codec__().ast.annotations or {}).get("identifier")
    return identifier if isinstance(identifier, str) else "generateObject"

make

make(*, generate_text, stream_text, codec_transformer=None, supports_system_messages_in_history=None) -> Effect[Service]

A language model from a provider's two functions. Its requests use the IdGenerator in context here (else the default one).

ParameterTypeDescription
generate_textGenerateFn
stream_textStreamFn
codec_transformerCodecTransformer | None = None
supports_system_messages_in_historyEffect[bool] | None = None

Returns Effect[Service]

def make(
    *,
    generate_text: GenerateFn,
    stream_text: StreamFn,
    codec_transformer: CodecTransformer | None = None,
    supports_system_messages_in_history: Effect[bool] | None = None,
) -> Effect[Service]:
    """A language model from a provider's two functions. Its requests use
    the ``IdGenerator`` in context here (else the default one)."""
    if supports_system_messages_in_history is not None:
        require_effect(supports_system_messages_in_history, "supports_system_messages_in_history")
    return fx.service_option(IdGenerator).flat_map(
        lambda ids: fx.service_option(CurrentSpanTransformer).map(
            lambda transformer: Service(
                generate_text,
                stream_text,
                codec_transformer or default_codec_transformer,
                supports_system_messages_in_history,
                ids.value if ids is not None else default_id_generator,
                transformer.value if transformer is not None else None,
            )
        )
    )

generate_text

generate_text(prompt: P.RawInput, *, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred']) -> Effect[GenerateTextResponse[str, Never], AiError, LanguageModel]
generate_text(prompt: P.RawInput, *, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict') -> Effect[GenerateTextResponse[Never, Never], AiError, LanguageModel]
generate_text[NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred']) -> Effect[GenerateTextResponse[str, RP], AiError | EE, LanguageModel | RR]
generate_text[NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict') -> Effect[GenerateTextResponse[NN, RP], AiError | EE, LanguageModel | RR]

Generate a response to prompt. With a toolkit (its handlers or the toolkit itself), the model may call its tools: their results follow the model's parts, unless disable_tool_call_resolution. concurrency bounds the tool calls running at once.

ParameterTypeDescription
prompt
toolkit= None
tool_choice= 'auto'
concurrency= 'unbounded'
disable_tool_call_resolution= False
tool_call_validation= 'strict'
@overload
def generate_text(
    prompt: P.RawInput,
    *,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
) -> Effect[GenerateTextResponse[str, Never], AiError, LanguageModel]: ...
@overload
def generate_text(
    prompt: P.RawInput,
    *,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
) -> Effect[GenerateTextResponse[Never, Never], AiError, LanguageModel]: ...
@overload
def generate_text[NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
) -> Effect[GenerateTextResponse[str, RP], AiError | EE, LanguageModel | RR]: ...
@overload
def generate_text[NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
) -> Effect[GenerateTextResponse[NN, RP], AiError | EE, LanguageModel | RR]: ...
def generate_text(
    prompt: P.RawInput,
    *,
    toolkit: Any = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: ToolCallValidation = "strict",
) -> Effect[GenerateTextResponse[Any, Any], Any, Any]:
    """Generate a response to ``prompt``. With a ``toolkit`` (its handlers
    or the toolkit itself), the model may call its tools: their results
    follow the model's parts, unless ``disable_tool_call_resolution``.
    ``concurrency`` bounds the tool calls running at once."""
    return LanguageModel().flat_map(
        lambda model: model.generate_text(
            prompt,
            toolkit=toolkit,
            tool_choice=tool_choice,
            concurrency=concurrency,
            disable_tool_call_resolution=disable_tool_call_resolution,
            tool_call_validation=tool_call_validation,
        )
    )

generate_object

generate_object[A](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred'], allow_empty: Literal[False] = False) -> Effect[GenerateObjectResponse[str, Never, A], AiError, LanguageModel]
generate_object[A](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict', allow_empty: Literal[False] = False) -> Effect[GenerateObjectResponse[Never, Never, A], AiError, LanguageModel]
generate_object[A, NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred'], allow_empty: Literal[False] = False) -> Effect[GenerateObjectResponse[str, RP, A], AiError | EE, LanguageModel | RR]
generate_object[A, NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict', allow_empty: Literal[False] = False) -> Effect[GenerateObjectResponse[NN, RP, A], AiError | EE, LanguageModel | RR]
generate_object[A](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred'], allow_empty: Literal[True]) -> Effect[GenerateObjectResponse[str, Never, Option[A]], AiError, LanguageModel]
generate_object[A](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict', allow_empty: Literal[True]) -> Effect[GenerateObjectResponse[Never, Never, Option[A]], AiError, LanguageModel]
generate_object[A, NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred'], allow_empty: Literal[True]) -> Effect[GenerateObjectResponse[str, RP, Option[A]], AiError | EE, LanguageModel | RR]
generate_object[A, NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, schema: S.Codec[A, Any, Any, Any], object_name: str | None = None, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict', allow_empty: Literal[True]) -> Effect[GenerateObjectResponse[NN, RP, Option[A]], AiError | EE, LanguageModel | RR]

Generate a JSON object matching schema (named object_name, else the schema's identifier) and decode it into value. With allow_empty=True the value is an Option: None when the model answers with no text or only whitespace, Some of the decoded object otherwise (invalid text still fails).

ParameterTypeDescription
prompt
schema
object_name= None
toolkit= None
tool_choice= 'auto'
concurrency= 'unbounded'
disable_tool_call_resolution= False
tool_call_validation= 'strict'
allow_empty= False
@overload
def generate_object[A](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
    allow_empty: Literal[False] = False,
) -> Effect[GenerateObjectResponse[str, Never, A], AiError, LanguageModel]: ...
@overload
def generate_object[A](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
    allow_empty: Literal[False] = False,
) -> Effect[GenerateObjectResponse[Never, Never, A], AiError, LanguageModel]: ...
@overload
def generate_object[A, NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
    allow_empty: Literal[False] = False,
) -> Effect[GenerateObjectResponse[str, RP, A], AiError | EE, LanguageModel | RR]: ...
@overload
def generate_object[A, NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
    allow_empty: Literal[False] = False,
) -> Effect[GenerateObjectResponse[NN, RP, A], AiError | EE, LanguageModel | RR]: ...
@overload
def generate_object[A](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
    allow_empty: Literal[True],
) -> Effect[GenerateObjectResponse[str, Never, Option[A]], AiError, LanguageModel]: ...
@overload
def generate_object[A](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
    allow_empty: Literal[True],
) -> Effect[GenerateObjectResponse[Never, Never, Option[A]], AiError, LanguageModel]: ...
@overload
def generate_object[A, NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
    allow_empty: Literal[True],
) -> Effect[GenerateObjectResponse[str, RP, Option[A]], AiError | EE, LanguageModel | RR]: ...
@overload
def generate_object[A, NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    schema: S.Codec[A, Any, Any, Any],
    object_name: str | None = None,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
    allow_empty: Literal[True],
) -> Effect[GenerateObjectResponse[NN, RP, Option[A]], AiError | EE, LanguageModel | RR]: ...
def generate_object(
    prompt: P.RawInput,
    *,
    schema: S.Top,
    object_name: str | None = None,
    toolkit: Any = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: ToolCallValidation = "strict",
    allow_empty: bool = False,
) -> Effect[GenerateObjectResponse[Any, Any, Any], Any, Any]:
    """Generate a JSON object matching ``schema`` (named ``object_name``,
    else the schema's identifier) and decode it into ``value``. With
    ``allow_empty=True`` the value is an ``Option``: ``None`` when the model
    answers with no text or only whitespace, ``Some`` of the decoded object
    otherwise (invalid text still fails)."""
    return LanguageModel().flat_map(
        lambda model: model.generate_object(
            prompt,
            schema=schema,
            object_name=object_name,
            toolkit=toolkit,
            tool_choice=tool_choice,
            concurrency=concurrency,
            disable_tool_call_resolution=disable_tool_call_resolution,
            tool_call_validation=tool_call_validation,
            allow_empty=allow_empty,
        )
    )

stream_text

stream_text(prompt: P.RawInput, *, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred']) -> Stream[R.StreamPart[str, Never], AiError, LanguageModel]
stream_text(prompt: P.RawInput, *, toolkit: None = None, tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict') -> Stream[R.StreamPart[Never, Never], AiError, LanguageModel]
stream_text[NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: Literal[True], tool_call_validation: Literal['deferred']) -> Stream[R.StreamPart[str, RP], AiError | EE, LanguageModel | RR]
stream_text[NN: LiteralString, RP, EE, RR](prompt: P.RawInput, *, toolkit: _ToolkitInput[NN, RP, EE, RR], tool_choice: ToolChoice = 'auto', concurrency: Concurrency = 'unbounded', disable_tool_call_resolution: bool = False, tool_call_validation: Literal['strict'] = 'strict') -> Stream[R.StreamPart[NN, RP], AiError | EE, LanguageModel | RR]

Stream a response to prompt: the model's parts as they come, then its tool calls' results (preliminary ones included), then the finish part.

ParameterTypeDescription
prompt
toolkit= None
tool_choice= 'auto'
concurrency= 'unbounded'
disable_tool_call_resolution= False
tool_call_validation= 'strict'
@overload
def stream_text(
    prompt: P.RawInput,
    *,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
) -> Stream[R.StreamPart[str, Never], AiError, LanguageModel]: ...
@overload
def stream_text(
    prompt: P.RawInput,
    *,
    toolkit: None = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
) -> Stream[R.StreamPart[Never, Never], AiError, LanguageModel]: ...
@overload
def stream_text[NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: Literal[True],
    tool_call_validation: Literal["deferred"],
) -> Stream[R.StreamPart[str, RP], AiError | EE, LanguageModel | RR]: ...
@overload
def stream_text[NN: LiteralString, RP, EE, RR](
    prompt: P.RawInput,
    *,
    toolkit: _ToolkitInput[NN, RP, EE, RR],
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: Literal["strict"] = "strict",
) -> Stream[R.StreamPart[NN, RP], AiError | EE, LanguageModel | RR]: ...
def stream_text(
    prompt: P.RawInput,
    *,
    toolkit: Any = None,
    tool_choice: ToolChoice = "auto",
    concurrency: Concurrency = "unbounded",
    disable_tool_call_resolution: bool = False,
    tool_call_validation: ToolCallValidation = "strict",
) -> Stream[Any, Any, Any]:
    """Stream a response to ``prompt``: the model's parts as they come,
    then its tool calls' results (preliminary ones included), then the
    finish part."""
    return StreamC.unwrap(
        LanguageModel().map(
            lambda model: model.stream_text(
                prompt,
                toolkit=toolkit,
                tool_choice=tool_choice,
                concurrency=concurrency,
                disable_tool_call_resolution=disable_tool_call_resolution,
                tool_call_validation=tool_call_validation,
            )
        )
    )