Functions#
__init__#
__init__(_tokenize) -> None| Parameter | Type | Description |
|---|---|---|
_tokenize | Tokenize |
tokenize#
tokenize(input) -> Effect[Sequence[int], AiError]The token ids of input (a string, messages or a Prompt,
made into a Prompt first); fails with the tokenizer's
AiError.
| Parameter | Type | Description |
|---|---|---|
input | P.RawInput |
Returns Effect[Sequence[int], AiError]
def tokenize(self, input: P.RawInput) -> Effect[Sequence[int], AiError]:
"""The token ids of ``input`` (a string, messages or a ``Prompt``,
made into a ``Prompt`` first); fails with the tokenizer's
``AiError``."""
return fx.suspend(lambda: self._tokenize(P.make(input)))truncate#
truncate(input, tokens) -> Effect[P.Prompt, AiError]The latest messages of input that fit in tokens.
| Parameter | Type | Description |
|---|---|---|
input | P.RawInput | |
tokens | int |
Returns Effect[P.Prompt, AiError]
def truncate(self, input: P.RawInput, tokens: int) -> Effect[P.Prompt, AiError]:
"""The latest messages of ``input`` that fit in ``tokens``."""
return fx.suspend(lambda: _truncate(P.make(input).content, (), self._tokenize, tokens))