class DurableQueue[P, A = None, E = Never]
A queue of P items whose workers succeed with A or fail with
E. Make one with make.
Attributes#
| Attribute | Type | Description |
|---|---|---|
name | str | The queue's name. |
payload_schema | Codec[P, Any, Never, Never] | The schema of the queued items. |
idempotency_key | Callable[[P], str] | The key that identifies an item, so offering it twice queues it once. |
deferred | DurableDeferred[A, E] | The deferred a worker answers an item through. |
Functions#
__init__#
__init__(*, name, payload_schema, idempotency_key, deferred) -> None| Parameter | Type | Description |
|---|---|---|
name | str | |
payload_schema | Codec[P, Any, Never, Never] | |
idempotency_key | Callable[[P], str] | |
deferred | DurableDeferred[A, E] |
def __init__(
self,
*,
name: str,
payload_schema: Codec[P, Any, Never, Never],
idempotency_key: Callable[[P], str],
deferred: DurableDeferred[A, E],
) -> None:
self._name = name
self._payload_schema = payload_schema
self._idempotency_key = idempotency_key
self._deferred = deferredprocess#
process(payload, *, retry_schedule=None) -> Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]Offer payload and wait (suspended) for a worker to process
it. A failed offer is retried on retry_schedule (logged), then
is a defect.
| Parameter | Type | Description |
|---|---|---|
payload | P | |
retry_schedule | Schedule[Any, Any, Any, Any] | None = None |
Returns Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]
def process(
self, payload: P, *, retry_schedule: Schedule[Any, Any, Any, Any] | None = None
) -> Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]:
"""Offer ``payload`` and wait (suspended) for a worker to process
it. A failed offer is retried on ``retry_schedule`` (logged), then
is a defect."""
def body() -> Generator[Effect[Any, Any, Any], Any, A]:
made = self.payload_schema.__codec__().make(payload)
queue_name = f"DurableQueue/{self.name}"
queue = yield from persisted_queue.make(
name=queue_name, schema=_queue_schema(self.payload_schema)
)
item_id = yield from activity_idempotency_key(
f"{queue_name}/{self.idempotency_key(made)}"
)
deferred = self._item_deferred(f"{self.deferred.name}/{item_id}")
token = yield from deferred.token()
def offer(span: Span) -> Effect[None]:
item: _QueueItem = {
"token": token,
"payload": made,
"traceId": span.trace_id,
"spanId": span.span_id,
"sampled": span.sampled,
}
def schema_defect(
error: PersistedQueueError | SchemaError,
) -> Effect[Never, PersistedQueueError]:
return die(error) if isinstance(error, SchemaError) else fail(error)
return (
queue.offer(item, id=item_id)
.tap_cause(log_warning)
.catch_all(schema_defect)
.retry(DEFAULT_RETRY_SCHEDULE if retry_schedule is None else retry_schedule)
.or_die()
.as_void()
.annotate_logs(
{
"package": "effect",
"module": "DurableQueue",
"fiber": "process",
"queueName": self.name,
}
)
)
yield from use_span(
f"DurableQueue/{self.name}/process", offer, attributes={"id": item_id}
)
return (yield from deferred.wait())
return gen(body)make_worker#
make_worker[A2, E2, R = Never](f, *, concurrency=1) -> Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]Take items forever, concurrency at a time, recording each
handler's exit for the workflow that offered it. A failure is
logged and the worker goes on.
| Parameter | Type | Description |
|---|---|---|
f | Callable[[P], Effect[A2, E2, R]] | |
concurrency | int = 1 |
Returns Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]
def make_worker[A2, E2, R = Never](
self: DurableQueue[P, A2, E2],
f: Callable[[P], Effect[A2, E2, R]],
*,
concurrency: int = 1,
) -> Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]:
"""Take items forever, ``concurrency`` at a time, recording each
handler's exit for the workflow that offered it. A failure is
logged and the worker goes on."""
def body() -> Generator[Effect[Any, Any, Any], Any, Any]:
queue = yield from persisted_queue.make(
name=f"DurableQueue/{self.name}", schema=_queue_schema(self.payload_schema)
)
def handle(
item: _QueueItem, _meta: persisted_queue.Metadata
) -> Effect[None, Never, Any]:
payload = cast("P", item["payload"])
return (
f(payload)
.exit()
.flat_map(lambda ex: self._item_done(item["token"], ex))
.with_span(
f"DurableQueue/{self.name}/worker",
capture_stack_trace=False,
parent=ExternalSpan(
trace_id=item["traceId"],
span_id=item["spanId"],
sampled=item["sampled"],
),
)
)
worker = (
queue.take(handle)
.catch_cause(log_warning)
.forever()
.annotate_logs(
{
"package": "effect",
"module": "DurableQueue",
"fiber": "worker",
"queueName": self.name,
}
)
)
yield from worker.replicate_effect(concurrency, concurrency=concurrency, discard=True)
return (yield from never)
return gen(body)worker#
worker[A2, E2, R = Never](f, *, concurrency=1) -> Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]A layer running make_worker in the background.
| Parameter | Type | Description |
|---|---|---|
f | Callable[[P], Effect[A2, E2, R]] | |
concurrency | int = 1 |
Returns Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]
def worker[A2, E2, R = Never](
self: DurableQueue[P, A2, E2], f: Callable[[P], Effect[A2, E2, R]], *, concurrency: int = 1
) -> Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]:
"""A layer running ``make_worker`` in the background."""
started: Effect[object, Never, Any] = self.make_worker(
f, concurrency=concurrency
).fork_scoped()
return cast(
"Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]",
effect_discard(cast("Effect[object, Never, Scope]", started)),
)