The batch exporter every OTLP signal posts through (Effect's
OtlpExporter): it buffers encoded telemetry, posts batches on an
interval or when a batch is full, retries transient failures, stops
exporting for 60 seconds after one it cannot handle, and flushes when its
scope closes.
Flusher drains every exporter sharing it on demand; the signal layers
share layer_flusher, so one flush drains traces, logs and metrics.
Values#
| Attribute | Type | Description |
|---|---|---|
layer_flusher | Final[Layer[Flusher]] | A |
Types#
| Type | Definition | Description |
|---|---|---|
Compression | type Compression = Literal['gzip', 'none'] | Request compression; |
Classes#
Functions#
make#
make(*, url, headers, label, export_interval, max_batch_size, body, shutdown_timeout, compression='none', export_empty=False) -> Effect[Exporter, Never, Flusher | HttpClient | Scope]A scoped batch exporter posting body(items) to url.
It exports export_interval after the first item is buffered (each
export finishing before the next interval starts) and as soon as
max_batch_size items are buffered; with max_batch_size="disabled"
only on the interval or a flush. An export that empties the buffer stops
the interval until the next push, so an idle exporter keeps no timer;
with export_empty and batching disabled it exports every interval.
Transient failures are retried three times. Any other failure drops the
buffer and disables exporting for 60 seconds; the first push or export
after that ends the window. When the scope closes it exports
what is left and waits up to shutdown_timeout for its exports,
then interrupts them. "gzip" compression happens once, before the
retries; body's effect runs after a successful post.
| Parameter | Type | Description |
|---|---|---|
url | str | |
headers | _headers.Input | None | |
label | str | |
export_interval | DurationInput | |
max_batch_size | int | Literal['disabled'] | |
body | Callable[[list[Any]], tuple[HttpBody, Effect[None]]] | |
shutdown_timeout | DurationInput | |
compression | Compression = 'none' | |
export_empty | bool = False |
Returns Effect[Exporter, Never, Flusher | HttpClient | Scope]
def make(
*,
url: str,
headers: _headers.Input | None,
label: str,
export_interval: DurationInput,
max_batch_size: int | Literal["disabled"],
body: Callable[[list[Any]], tuple[HttpBody, Effect[None]]],
shutdown_timeout: DurationInput,
compression: Compression = "none",
export_empty: bool = False,
) -> Effect[Exporter, Never, Flusher | HttpClient | Scope]:
"""A scoped batch exporter posting ``body(items)`` to ``url``.
It exports ``export_interval`` after the first item is buffered (each
export finishing before the next interval starts) and as soon as
``max_batch_size`` items are buffered; with ``max_batch_size="disabled"``
only on the interval or a flush. An export that empties the buffer stops
the interval until the next push, so an idle exporter keeps no timer;
with ``export_empty`` and batching disabled it exports every interval.
Transient failures are retried three times. Any other failure drops the
buffer and disables exporting for 60 seconds; the first push or export
after that ends the window. When the scope closes it exports
what is left and waits up to ``shutdown_timeout`` for its exports,
then interrupts them. ``"gzip"`` compression happens once, before the
retries; ``body``'s effect runs after a successful post."""
interval = from_input_unsafe(export_interval).max(zero)
timeout = from_input_unsafe(shutdown_timeout)
@gen
def build():
clock = yield from Clock()
client = _client((yield from HttpClient()))
request = _post(url, headers, label)
exports_empty = max_batch_size == "disabled" and export_empty
disabled_until: int | None = None
buffer: list[Any] = []
def is_disabled() -> bool:
nonlocal disabled_until
if disabled_until is None:
return False
if now_millis_unsafe(clock) < disabled_until:
return True
disabled_until = None
return False
def export() -> Effect[None, Any]:
nonlocal buffer
if is_disabled() or (not buffer and not exports_empty):
return succeed(None)
items, buffer = buffer, []
payload, on_success = body(items)
batch = request.set_body(payload)
response = (
batch.compress_body("gzip").flat_map(client.execute)
if compression == "gzip"
else client.execute(batch)
)
return response.and_then(on_success).as_void().with_tracer_enabled(False)
def disable(cause: Cause[object]) -> Effect[None]:
nonlocal disabled_until, buffer
if disabled_until is not None:
return succeed(None)
disabled_until = now_millis_unsafe(clock) + _DISABLED_MILLIS
buffer = []
return log_debug("Disabling exporter for 60 seconds", cause)
run_export: Effect[None] = (
suspend(export)
.catch_cause(disable)
.annotate_logs({"package": "pyfect", "module": label})
)
export_fibers = yield from FiberSet[None, Never].make()
fork_export = yield from export_fibers.runtime()
def shut_down() -> Effect[None]:
if disabled_until is not None:
return succeed(None)
fork_export(run_export)
return export_fibers.await_empty()
yield from add_finalizer(
lambda _exit: suspend(shut_down).ignore().interruptible().timeout_option(timeout)
)
timer = _Timer(armed=Latch(exports_empty), keep_armed=exports_empty)
# Forked after the shutdown finalizer, so the loop stops before the
# shutdown flush.
scope = yield from Scope()
yield from timer.loop(
interval, export_fibers.run(run_export), lambda: bool(buffer)
).fork_in(scope)
export_now = run_export.and_then(sync(lambda: timer.disarm_if_empty(bool(buffer))))
flusher = yield from Flusher()
yield from flusher.register(export_now)
def push(item: Any) -> None:
# With no idle timer, the next push ends the disabled window.
if is_disabled():
return
buffer.append(item)
if max_batch_size != "disabled" and len(buffer) >= max_batch_size:
fork_export(export_now)
else:
timer.armed.open_unsafe()
return Exporter(push)
return build