GitHub

DurableQueue

API reference for DurableQueue.

class DurableQueue[P, A = None, E = Never]

A queue of P items whose workers succeed with A or fail with E. Make one with make.

Attributes

AttributeTypeDescription
namestr

The queue's name.

payload_schemaCodec[P, Any, Never, Never]

The schema of the queued items.

idempotency_keyCallable[[P], str]

The key that identifies an item, so offering it twice queues it once.

deferredDurableDeferred[A, E]

The deferred a worker answers an item through.

Functions

__init__

__init__(*, name, payload_schema, idempotency_key, deferred) -> None
ParameterTypeDescription
namestr
payload_schemaCodec[P, Any, Never, Never]
idempotency_keyCallable[[P], str]
deferredDurableDeferred[A, E]
def __init__(
    self,
    *,
    name: str,
    payload_schema: Codec[P, Any, Never, Never],
    idempotency_key: Callable[[P], str],
    deferred: DurableDeferred[A, E],
) -> None:
    self._name = name
    self._payload_schema = payload_schema
    self._idempotency_key = idempotency_key
    self._deferred = deferred

process

process(payload, *, retry_schedule=None) -> Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]

Offer payload and wait (suspended) for a worker to process it. A failed offer is retried on retry_schedule (logged), then is a defect.

ParameterTypeDescription
payloadP
retry_scheduleSchedule[Any, Any, Any, Any] | None = None

Returns Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]

def process(
    self, payload: P, *, retry_schedule: Schedule[Any, Any, Any, Any] | None = None
) -> Effect[A, E, WorkflowEngine | WorkflowInstance | PersistedQueueFactory]:
    """Offer ``payload`` and wait (suspended) for a worker to process
    it. A failed offer is retried on ``retry_schedule`` (logged), then
    is a defect."""

    def body() -> Generator[Effect[Any, Any, Any], Any, A]:
        made = self.payload_schema.__codec__().make(payload)
        queue_name = f"DurableQueue/{self.name}"
        queue = yield from persisted_queue.make(
            name=queue_name, schema=_queue_schema(self.payload_schema)
        )
        item_id = yield from activity_idempotency_key(
            f"{queue_name}/{self.idempotency_key(made)}"
        )
        deferred = self._item_deferred(f"{self.deferred.name}/{item_id}")
        token = yield from deferred.token()

        def offer(span: Span) -> Effect[None]:
            item: _QueueItem = {
                "token": token,
                "payload": made,
                "traceId": span.trace_id,
                "spanId": span.span_id,
                "sampled": span.sampled,
            }

            def schema_defect(
                error: PersistedQueueError | SchemaError,
            ) -> Effect[Never, PersistedQueueError]:
                return die(error) if isinstance(error, SchemaError) else fail(error)

            return (
                queue.offer(item, id=item_id)
                .tap_cause(log_warning)
                .catch_all(schema_defect)
                .retry(DEFAULT_RETRY_SCHEDULE if retry_schedule is None else retry_schedule)
                .or_die()
                .as_void()
                .annotate_logs(
                    {
                        "package": "effect",
                        "module": "DurableQueue",
                        "fiber": "process",
                        "queueName": self.name,
                    }
                )
            )

        yield from use_span(
            f"DurableQueue/{self.name}/process", offer, attributes={"id": item_id}
        )
        return (yield from deferred.wait())

    return gen(body)

make_worker

make_worker[A2, E2, R = Never](f, *, concurrency=1) -> Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]

Take items forever, concurrency at a time, recording each handler's exit for the workflow that offered it. A failure is logged and the worker goes on.

ParameterTypeDescription
fCallable[[P], Effect[A2, E2, R]]
concurrencyint = 1

Returns Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]

def make_worker[A2, E2, R = Never](
    self: DurableQueue[P, A2, E2],
    f: Callable[[P], Effect[A2, E2, R]],
    *,
    concurrency: int = 1,
) -> Effect[Never, Never, WorkflowEngine | PersistedQueueFactory | R]:
    """Take items forever, ``concurrency`` at a time, recording each
    handler's exit for the workflow that offered it. A failure is
    logged and the worker goes on."""

    def body() -> Generator[Effect[Any, Any, Any], Any, Any]:
        queue = yield from persisted_queue.make(
            name=f"DurableQueue/{self.name}", schema=_queue_schema(self.payload_schema)
        )

        def handle(
            item: _QueueItem, _meta: persisted_queue.Metadata
        ) -> Effect[None, Never, Any]:
            payload = cast("P", item["payload"])
            return (
                f(payload)
                .exit()
                .flat_map(lambda ex: self._item_done(item["token"], ex))
                .with_span(
                    f"DurableQueue/{self.name}/worker",
                    capture_stack_trace=False,
                    parent=ExternalSpan(
                        trace_id=item["traceId"],
                        span_id=item["spanId"],
                        sampled=item["sampled"],
                    ),
                )
            )

        worker = (
            queue.take(handle)
            .catch_cause(log_warning)
            .forever()
            .annotate_logs(
                {
                    "package": "effect",
                    "module": "DurableQueue",
                    "fiber": "worker",
                    "queueName": self.name,
                }
            )
        )
        yield from worker.replicate_effect(concurrency, concurrency=concurrency, discard=True)
        return (yield from never)

    return gen(body)

worker

worker[A2, E2, R = Never](f, *, concurrency=1) -> Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]

A layer running make_worker in the background.

ParameterTypeDescription
fCallable[[P], Effect[A2, E2, R]]
concurrencyint = 1

Returns Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]

def worker[A2, E2, R = Never](
    self: DurableQueue[P, A2, E2], f: Callable[[P], Effect[A2, E2, R]], *, concurrency: int = 1
) -> Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]:
    """A layer running ``make_worker`` in the background."""
    started: Effect[object, Never, Any] = self.make_worker(
        f, concurrency=concurrency
    ).fork_scoped()
    return cast(
        "Layer[Never, Never, WorkflowEngine | PersistedQueueFactory | R]",
        effect_discard(cast("Effect[object, Never, Scope]", started)),
    )