GitHub

language_model

API reference for language_model.

A LanguageModel over an OpenAI-compatible Chat Completions API (Effect's OpenAiLanguageModel in @effect/ai-openai-compat)::

program.pipe(fx.provide(language_model.model("llama-3.3-70b").layer)).pipe(
    fx.provide(client.layer(api_key=key, api_url="https://api.groq.com/openai/v1"))
)

The configuration is Responses-style (max_output_tokens, top_p, service_tier, ...) and sent as the Chat Completions equivalents; keys the Responses API does not know pass through to the request as provider extras. file_id_prefixes marks file data that is an uploaded file's id, and strict_json_schema (default True) the strictness of tool and response schemas.

Types

TypeDefinitionDescription
ModelConfigtype ModelConfig = ConfigService | Mapping[str, Any]

Options for a model; keys beyond ConfigService are sent as is.

Classes

Functions

make

make(*, model, config=None) -> Effect[lm.Service, Never, OpenAiClient]

A LanguageModel for model over the OpenAiClient; a provided Config overrides config per request.

ParameterTypeDescription
modelstr
configModelConfig | None = None

Returns Effect[lm.Service, Never, OpenAiClient]

def make(
    *, model: str, config: ModelConfig | None = None
) -> Effect[lm.Service, Never, OpenAiClient]:
    """A ``LanguageModel`` for ``model`` over the ``OpenAiClient``; a
    provided ``Config`` overrides ``config`` per request."""

    def current() -> Effect[ConfigService]:
        return P.current_config(Config, {"model": model}, config)

    def build(client: ClientService) -> Effect[lm.Service]:
        def generate(options: lm.ProviderOptions) -> Effect[Sequence[lm.EncodedPart], AiError, Any]:
            names = NameMapper(options.tools)

            def send(cfg: ConfigService) -> Effect[list[Part], AiError, Any]:
                strict = _strict_tools(cfg)

                def respond(sent: tuple[W.ChatCompletion, Any]) -> Effect[list[Part], AiError, Any]:
                    completion, response = sent
                    _annotate_response(options.span, completion)
                    return make_response(options, completion, response, names, strict)

                def post(request: dict[str, Any]) -> Effect[list[Part], AiError, Any]:
                    _annotate_request(options.span, request)
                    return client.create_response(request).flat_map(respond)

                return _make_request(cfg, options, names, strict).flat_map(post)

            return current().flat_map(send)

        def stream(options: lm.ProviderOptions) -> Stream[lm.EncodedPart, AiError, Any]:
            names = NameMapper(options.tools)

            def send(cfg: ConfigService) -> Effect[Stream[Part, AiError, Any], AiError]:
                strict = _strict_tools(cfg)

                def post(request: dict[str, Any]) -> Effect[Stream[Part, AiError, Any], AiError]:
                    _annotate_request(options.span, request)
                    return client.create_response_stream(request).map(
                        lambda sent: make_stream_response(
                            options,
                            sent[1],
                            sent[0],
                            names,
                            client.max_response_bytes,
                            strict=strict,
                        )
                    )

                return _make_request(cfg, options, names, strict).flat_map(post)

            built: Effect[Stream[Part, AiError, Any], AiError] = current().flat_map(send)
            return St.unwrap(built).map(lambda part: _annotate_stream_part(options.span, part))

        return lm.make(
            generate_text=generate, stream_text=stream, codec_transformer=to_codec_openai
        )

    return OpenAiClient().flat_map(build)

layer

layer(*, model, config=None) -> L.Layer[lm.LanguageModel, Never, OpenAiClient]
ParameterTypeDescription
modelstr
configModelConfig | None = None

Returns L.Layer[lm.LanguageModel, Never, OpenAiClient]

def layer(
    *, model: str, config: ModelConfig | None = None
) -> L.Layer[lm.LanguageModel, Never, OpenAiClient]:
    return L.effect(lm.LanguageModel, make(model=model, config=config))

model

model(model, config=None) -> M.Model[lm.LanguageModel, OpenAiClient]

The model as a Model (provider "openai").

ParameterTypeDescription
modelstr
configModelConfig | None = None

Returns M.Model[lm.LanguageModel, OpenAiClient]

def model(model: str, config: ModelConfig | None = None) -> M.Model[lm.LanguageModel, OpenAiClient]:
    """The model as a ``Model`` (provider ``"openai"``)."""
    return M.make("openai", model, layer(model=model, config=config))

with_config_override

with_config_override(overrides) -> P.WithConfigOverride[ConfigService]

Run an effect with overrides merged over the current Config. Use with .pipe.

ParameterTypeDescription
overridesConfigService

Returns P.WithConfigOverride[ConfigService]

def with_config_override(overrides: ConfigService) -> P.WithConfigOverride[ConfigService]:
    """Run an effect with ``overrides`` merged over the current ``Config``.
    Use with ``.pipe``."""
    return P.WithConfigOverride(Config, overrides)