A LanguageModel over an OpenAI-compatible Chat Completions API
(Effect's OpenAiLanguageModel in @effect/ai-openai-compat)::
program.pipe(fx.provide(language_model.model("llama-3.3-70b").layer)).pipe(
fx.provide(client.layer(api_key=key, api_url="https://api.groq.com/openai/v1"))
)The configuration is Responses-style (max_output_tokens, top_p,
service_tier, ...) and sent as the Chat Completions equivalents; keys
the Responses API does not know pass through to the request as provider
extras. file_id_prefixes marks file data that is an uploaded file's
id, and strict_json_schema (default True) the strictness of tool
and response schemas.
Types#
| Type | Definition | Description |
|---|---|---|
ModelConfig | type ModelConfig = ConfigService | Mapping[str, Any] | Options for a model; keys beyond |
Classes#
Functions#
make#
make(*, model, config=None) -> Effect[lm.Service, Never, OpenAiClient]A LanguageModel for model over the OpenAiClient; a
provided Config overrides config per request.
| Parameter | Type | Description |
|---|---|---|
model | str | |
config | ModelConfig | None = None |
Returns Effect[lm.Service, Never, OpenAiClient]
def make(
*, model: str, config: ModelConfig | None = None
) -> Effect[lm.Service, Never, OpenAiClient]:
"""A ``LanguageModel`` for ``model`` over the ``OpenAiClient``; a
provided ``Config`` overrides ``config`` per request."""
def current() -> Effect[ConfigService]:
return P.current_config(Config, {"model": model}, config)
def build(client: ClientService) -> Effect[lm.Service]:
def generate(options: lm.ProviderOptions) -> Effect[Sequence[lm.EncodedPart], AiError, Any]:
names = NameMapper(options.tools)
def send(cfg: ConfigService) -> Effect[list[Part], AiError, Any]:
strict = _strict_tools(cfg)
def respond(sent: tuple[W.ChatCompletion, Any]) -> Effect[list[Part], AiError, Any]:
completion, response = sent
_annotate_response(options.span, completion)
return make_response(options, completion, response, names, strict)
def post(request: dict[str, Any]) -> Effect[list[Part], AiError, Any]:
_annotate_request(options.span, request)
return client.create_response(request).flat_map(respond)
return _make_request(cfg, options, names, strict).flat_map(post)
return current().flat_map(send)
def stream(options: lm.ProviderOptions) -> Stream[lm.EncodedPart, AiError, Any]:
names = NameMapper(options.tools)
def send(cfg: ConfigService) -> Effect[Stream[Part, AiError, Any], AiError]:
strict = _strict_tools(cfg)
def post(request: dict[str, Any]) -> Effect[Stream[Part, AiError, Any], AiError]:
_annotate_request(options.span, request)
return client.create_response_stream(request).map(
lambda sent: make_stream_response(
options,
sent[1],
sent[0],
names,
client.max_response_bytes,
strict=strict,
)
)
return _make_request(cfg, options, names, strict).flat_map(post)
built: Effect[Stream[Part, AiError, Any], AiError] = current().flat_map(send)
return St.unwrap(built).map(lambda part: _annotate_stream_part(options.span, part))
return lm.make(
generate_text=generate, stream_text=stream, codec_transformer=to_codec_openai
)
return OpenAiClient().flat_map(build)layer#
layer(*, model, config=None) -> L.Layer[lm.LanguageModel, Never, OpenAiClient]| Parameter | Type | Description |
|---|---|---|
model | str | |
config | ModelConfig | None = None |
Returns L.Layer[lm.LanguageModel, Never, OpenAiClient]
def layer(
*, model: str, config: ModelConfig | None = None
) -> L.Layer[lm.LanguageModel, Never, OpenAiClient]:
return L.effect(lm.LanguageModel, make(model=model, config=config))model#
model(model, config=None) -> M.Model[lm.LanguageModel, OpenAiClient]The model as a Model (provider "openai").
| Parameter | Type | Description |
|---|---|---|
model | str | |
config | ModelConfig | None = None |
Returns M.Model[lm.LanguageModel, OpenAiClient]
def model(model: str, config: ModelConfig | None = None) -> M.Model[lm.LanguageModel, OpenAiClient]:
"""The model as a ``Model`` (provider ``"openai"``)."""
return M.make("openai", model, layer(model=model, config=config))with_config_override#
with_config_override(overrides) -> P.WithConfigOverride[ConfigService]Run an effect with overrides merged over the current Config.
Use with .pipe.
| Parameter | Type | Description |
|---|---|---|
overrides | ConfigService |
Returns P.WithConfigOverride[ConfigService]
def with_config_override(overrides: ConfigService) -> P.WithConfigOverride[ConfigService]:
"""Run an effect with ``overrides`` merged over the current ``Config``.
Use with ``.pipe``."""
return P.WithConfigOverride(Config, overrides)