GitHub

Persistence

Key-value stores, caches whose results outlive the process, and durable queues, in memory, files, SQL or Redis.

pyfect.persistence keeps data outside a fiber's memory behind services, so a program picks where it lives by the layer it provides: memory in tests, a directory or a SQL table on one machine, Redis when several processes share it. The code that reads and writes stays the same.

Key-value stores

KeyValueStore holds text and bytes by string key. get gives None for a missing key, keys lists the keys and clear_prefix(p) removes those starting with p. prefix(p) gives a view whose keys all start with p: its keys, size and clear see only those. to_schema_store(schema) holds one schema's values as JSON text, so get gives an Option of the decoded value. key_value_store.layer_memory keeps a dict, layer_file_system(dir) a file per key, layer_sql(table=...) a table of the SqlClient, and layer_storage(f) any MutableMapping[str, str].

key_value.py
import tempfile

import pyfect as fx
from pyfect import layer, platform
from pyfect import schema as S
from pyfect.persistence import KeyValueStore, key_value_store


class Settings(S.Class, identifier="Settings"):
    theme: str
    font_size: int


@fx.fn
def program():
    kv = yield from KeyValueStore()
    yield from kv.set("greeting", "hello")
    print((yield from kv.get("greeting")))
    print((yield from kv.get("missing")))

    # A prefix keeps one part of a program's keys apart from the rest.
    users = kv.prefix("user:")
    yield from users.set("1", "Ada")
    print((yield from kv.has("user:1")))
    # A view lists and clears only its own keys.
    print((yield from users.keys))
    yield from users.clear
    print((yield from kv.keys))

    # A schema store holds typed values, written as JSON text.
    settings = kv.to_schema_store(Settings)
    yield from settings.set("ada", Settings(theme="dark", font_size=14))
    yield from settings.modify("ada", lambda s: Settings(theme=s.theme, font_size=16))
    print((yield from settings.get("ada")))


# A file per key, in a directory; key_value_store.layer_memory is the same in a dict.
with tempfile.TemporaryDirectory() as directory:
    Store = key_value_store.layer_file_system(directory).pipe(layer.provide(platform.layer))
    fx.run_main(program().pipe(fx.provide(Store)))

Cached results that survive a restart

A persistable request is a frozen dataclass on persistable.Class(success=..., error=...) with a primary_key(): the schemas say how to write its outcome, and the key says where. persisted_cache.make(lookup, store_id=..., time_to_live=...) is a cache in front of the Persistence service. get answers from memory, then from the store, and only then runs lookup and stores its Exit, so failures are cached too, for as long as time_to_live gives for that outcome. lookup must succeed and fail with the request's schemas' types: a lookup giving an int for success=S.String makes get a pyright error.

nonoyesyesusers.get(GetUser(1))in memory?in the Persistence store?lookup, then store its Exitthe outcome: User orUserNotFound
nonoyesyesusers.get(GetUser(1))in memory?in the Persistence store?lookup, then store its Exitthe outcome: User orUserNotFound
cached_users.py
from dataclasses import dataclass

import pyfect as fx
from pyfect import layer
from pyfect import schema as S
from pyfect.persistence import persistable, persisted_cache, persistence
from pyfect.sql import sqlite


class User(S.Class, identifier="User"):
    id: int
    name: str


class UserNotFound(S.TaggedError):
    id: int


# A request names its stored key and carries the schemas of its outcome.
@dataclass(frozen=True)
class GetUser(persistable.Class(success=User, error=UserNotFound)):
    id: int

    def primary_key(self) -> str:
        return f"GetUser:{self.id}"


lookups: list[int] = []


def fetch_user(request: GetUser) -> fx.Effect[User, UserNotFound]:
    lookups.append(request.id)
    if request.id == 1:
        return fx.succeed(User(id=1, name="Ada"))
    return fx.fail(UserNotFound(id=request.id))


def users_cache():
    return persisted_cache.make(
        fetch_user, store_id="users", time_to_live=lambda _exit, _request: "1 hour"
    )


@fx.fn
def program():
    users = yield from users_cache()
    print((yield from users.get(GetUser(1))))
    print(repr((yield from users.get(GetUser(2)).flip())))

    # A new cache, as after a restart: both outcomes come from the store.
    users = yield from users_cache()
    print((yield from users.get(GetUser(1))))
    print(repr((yield from users.get(GetUser(2)).flip())))
    print("lookups:", lookups)


# Outcomes in a SQL table; persistence.layer_memory, layer_kvs and layer_redis also work.
Persistence = persistence.layer_sql.pipe(
    layer.provide(sqlite.layer(sqlite.SqliteClientConfig(filename=":memory:")))
)
fx.run_main(fx.scoped(program()).pipe(fx.provide(Persistence)))

persistence.layer_memory, layer_kvs (over a KeyValueStore), layer_sql, layer_sql_multi_table and layer_redis provide Persistence. Each store_id is a namespace of its own: the key-value and Redis backends percent-encode % and : in it, so the stores "cache" and "cache:users" never share an entry, and clearing one leaves the other. A stored outcome the request's schemas no longer decode is looked up again and replaced, so changing a schema doesn't break old entries. RequestResolver.persisted stores a batching resolver's outcomes the same way.

Durable queues

persisted_queue.make(name=..., schema=...) is a queue of schema-typed values. offer stores a value under an id (a new UUID, or one you give, in which case offering it again adds nothing). take(handler) runs handler on the next value with its Metadata: if it succeeds the value is done; if it fails, take fails and the value goes back on the queue after the retry schedule's delay, until max_attempts runs out and it is marked failed. Delivery is at least once, so handlers must be safe to run twice.

emails.py
import pyfect as fx
from pyfect import layer, schedule
from pyfect import schema as S
from pyfect.persistence import persisted_queue
from pyfect.persistence.persisted_queue import Metadata


class Email(S.Class, identifier="Email"):
    to: str
    subject: str


def send(email: Email, meta: Metadata) -> fx.Effect[str, str]:
    print(f"attempt {meta.attempts}: {email.subject} to {email.to}")
    if meta.attempts == 1:
        return fx.fail("SMTP timeout")
    return fx.succeed(meta.id)


@fx.fn
def program():
    queue = yield from persisted_queue.make(
        name="emails", schema=Email, max_attempts=3, retry_schedule=schedule.spaced("50 millis")
    )
    welcome = Email(to="ada@example.com", subject="Welcome")
    yield from queue.offer(welcome, id="welcome-ada")
    yield from queue.offer(welcome, id="welcome-ada")  # the same id is stored once
    # A take whose handler fails fails too, and the email goes back on the queue.
    print("failed:", (yield from queue.take(send).flip()))
    # The next take gets it again once the retry delay has passed.
    print("sent:", (yield from queue.take(send)))


# Queues in memory; persisted_queue.layer_store_sql and layer_store_redis keep them
# across restarts and share them between processes.
Queues = persisted_queue.layer.pipe(layer.provide(persisted_queue.layer_store_memory))
fx.run_main(program().pipe(fx.provide(Queues)))

A consumer is usually queue.take(handler).forever() on a fiber of its own. layer_store_sql (any SqlClient) and layer_store_redis keep queues across restarts and let several processes take from one queue: a value taken by a process that dies is released for another once its lock expires. Each queue keeps its own custom ids, and a handler whose expired claim was handed to another worker can no longer settle it. persisted_queue.layer_cleanup removes old completed values.

That wait is lock_expiration, 120 seconds on layer_store_sql and 90 on layer_store_redis, as in Effect, so a job whose worker was killed sits for that long while others finish. A worker that is alive renews its lock every lock_refresh_interval (30 seconds on both), so lowering the expiration only costs anything if it gets close to that.

lock_expiration.py
import os
import subprocess
import sys
import tempfile
from collections.abc import Callable
from pathlib import Path

import pyfect as fx
from pyfect import layer, reactivity
from pyfect import schema as S
from pyfect.persistence import persisted_queue
from pyfect.persistence.persisted_queue import Metadata
from pyfect.sql import sqlite
from pyfect.sql.client import SqlClient


def queues(database: str):
    sql = layer.effect(SqlClient, sqlite.make(sqlite.SqliteClientConfig(filename=database))).pipe(
        layer.provide(reactivity.layer)
    )
    # 120 s by default (90 s on Redis). A live worker renews its lock every
    # lock_refresh_interval (30 s by default), so keep the expiration above that.
    store = persisted_queue.layer_store_sql(
        poll_interval="50 millis", lock_refresh_interval="500 millis", lock_expiration="2 seconds"
    )
    return persisted_queue.layer.pipe(layer.provide(store)).pipe(layer.provide(sql))


def die(_job: str, _meta: Metadata) -> fx.Effect[None]:
    return fx.sync(lambda: os._exit(9))  # the worker is gone, its lock is still held


def finish(job: str, meta: Metadata) -> fx.Effect[None]:
    return fx.sync(lambda: print(f"{job} taken again, attempt {meta.attempts}"))


@fx.fn
def worker(handler: Callable[[str, Metadata], fx.Effect[None]]):
    queue = yield from persisted_queue.make(name="jobs", schema=S.String)
    return (yield from queue.take(handler))


@fx.fn
def offer():
    queue = yield from persisted_queue.make(name="jobs", schema=S.String)
    yield from queue.offer("pack order 7", id="7")


if len(sys.argv) > 1:
    fx.run_main(worker(die).pipe(fx.provide(queues(sys.argv[1]))))
else:
    with tempfile.TemporaryDirectory() as directory:
        database = str(Path(directory) / "queues.db")
        fx.run_main(offer().pipe(fx.provide(queues(database))))
        killed = subprocess.run([sys.executable, __file__, database], check=False)
        print("worker exited with", killed.returncode)
        fx.run_main(worker(finish).pipe(fx.provide(queues(database))))

Reference: pyfect.persistence, key_value_store, persisted_cache, persisted_queue.