fx.tx(eff) runs eff as a transaction. It reads and writes TxRefs through a journal,
then commits every write at once. If another fiber changed a ref the transaction read before
it could commit, the body runs again. No lock is held while a body runs (checking and
committing take a process-wide lock for an instant), so transactions don't deadlock and
compose freely: a tx inside another tx joins the outer one.
TxRef#
A TxRef[A] is a mutable cell. get, set, update and modify join the running
transaction, or run as a transaction of their own when there is none.
import pyfect as fx
seats = fx.TxRef(2)
booked = fx.TxRef[tuple[str, ...]](())
@fx.fn
def book(name: str):
left = yield from seats.get()
if left == 0:
return False
yield from fx.yield_now() # let the other bookings run in between
yield from seats.set(left - 1)
yield from booked.update(lambda names: (*names, name))
return True
@fx.fn
def program():
# Each booking reads and writes two refs as one atomic step: no other fiber sees
# a seat taken but not booked, and concurrent bookings never oversell.
results = yield from fx.for_each("abcd", lambda name: fx.tx(book(name)), concurrency=4)
print(results)
print((yield from seats.get()), (yield from booked.get()))
fx.run_blocking(program())The body can run more than once, so keep side effects other than Tx* operations, such as
logging or requests, outside tx.
Waiting with tx_retry#
fx.tx_retry abandons the current attempt and waits until another transaction commits to one
of the refs this one read, then runs the body again. This is how you wait for a
condition, with no polling and no condition variables.
import pyfect as fx
stock = fx.TxRef(0)
@fx.fn
def take(n: int):
available = yield from stock.get()
if available < n:
# Roll back and wait: the transaction runs again once a ref it read changes.
return (yield from fx.tx_retry)
yield from stock.set(available - n)
return n
@fx.fn
def program():
buyer = yield from fx.tx(take(3)).fork()
for _ in range(3):
yield from fx.sleep("10 millis")
yield from stock.update(lambda n: n + 1)
print("restocked to", (yield from stock.get()))
print("bought", (yield from buyer.join()), "left", (yield from stock.get()))
fx.run_blocking(program())A bank transfer#
A transfer deposits into one account and withdraws from another. A typed failure inside tx
discards every write the transaction made, so a failed withdrawal undoes the deposit before
it. Two hundred transfers run concurrently, and the total balance stays the same.
from dataclasses import dataclass
import pyfect as fx
@dataclass
class InsufficientFunds(fx.TaggedError):
account: str
needed: int
@dataclass(frozen=True)
class Account:
name: str
balance: fx.TxRef[int]
def open_account(name: str, balance: int) -> Account:
return Account(name, fx.TxRef(balance))
@fx.fn
def withdraw(account: Account, amount: int):
balance = yield from account.balance.get()
if balance < amount:
return (yield from InsufficientFunds(account.name, amount - balance))
yield from account.balance.set(balance - amount)
def deposit(account: Account, amount: int) -> fx.Effect[None]:
return account.balance.update(lambda balance: balance + amount)
def transfer(source: Account, target: Account, amount: int) -> fx.Effect[None, InsufficientFunds]:
# Both writes commit together or not at all: if the withdrawal fails, the
# deposit made just before it is discarded.
return fx.tx(deposit(target, amount).and_then(withdraw(source, amount)))
def total(*accounts: Account) -> fx.Effect[int]:
return fx.tx(fx.for_each(accounts, lambda account: account.balance.get())).map(sum)
@fx.fn
def program():
alice, bob = open_account("alice", 100), open_account("bob", 50)
# A hundred transfers each way at once: whatever the interleaving, no money is
# created or lost.
moves = [transfer(alice, bob, 1) for _ in range(100)] + [
transfer(bob, alice, 1) for _ in range(100)
]
yield from fx.for_each(moves, lambda move: move.ignore(), concurrency="unbounded")
assert (yield from total(alice, bob)) == 150
# A failed transfer leaves both balances as they were.
before = yield from alice.balance.get()
failed = yield from transfer(bob, alice, 500).result()
assert isinstance(failed, fx.Err) and failed.error.account == "bob"
assert (yield from alice.balance.get()) == before
assert (yield from total(alice, bob)) == 150
fx.run_blocking(program())Sharing refs between runs#
TxRefs, and every transactional collection below, can live at module level and be used by
runs on several threads at once, such as fx.run_blocking calls from the threads of a web
server (see Threads). Commits from every run go
one at a time, so a transaction never commits over a write it did not see, and a fiber
waiting in tx_retry in one run wakes when a run on another thread commits to a ref it read.
Catching a failure inside tx#
A transaction has no savepoints. The writes are undone only when the whole tx fails. A
failure caught inside tx leaves the transaction running, and every write made before it,
including those of the effect that failed, commits with the rest. Effect behaves the same.
To undo a step's writes, catch its failure outside tx, or check before writing.
from dataclasses import dataclass
import pyfect as fx
@dataclass
class OutOfStock(fx.TaggedError):
short: int
@fx.fn
def take(stock: fx.TxRef[int], amount: int):
# Writes first, then checks: the write is in the journal when the failure happens.
yield from stock.update(lambda left: left - amount)
left = yield from stock.get()
if left < 0:
return (yield from OutOfStock(-left))
def ignore_out_of_stock(eff: fx.Effect[None, OutOfStock]) -> fx.Effect[None]:
return eff.pipe(fx.catch_tag(OutOfStock, lambda _: fx.void))
@fx.fn
def program():
# Caught outside tx: the transaction failed, so nothing it wrote commits.
outside = fx.TxRef(1)
yield from ignore_out_of_stock(fx.tx(take(outside, 3)))
print("caught outside:", (yield from outside.get()))
# Caught inside tx: the transaction succeeds and commits the write made before
# the failure.
inside = fx.TxRef(1)
yield from fx.tx(ignore_out_of_stock(take(inside, 3)))
print("caught inside:", (yield from inside.get()))
fx.run_blocking(program())Transactional collections#
TxQueue, TxHashMap, TxHashSet, TxChunk, TxPriorityQueue, TxPubSub,
TxSemaphore, TxReentrantLock, TxDeferred and TxSubscriptionRef are built on TxRef,
so they can all take part in the same transaction. A take from an empty queue, or an
offer to a full bounded one, waits through tx_retry.
import pyfect as fx
@fx.fn
def program():
# A queue that can be ended says so in its error type: takers fail with Done.
jobs = yield from fx.TxQueue[str, fx.Done].bounded(2)
claimed = yield from fx.TxHashMap[str, str].empty()
@fx.fn
def claim(worker: str):
# Taking a job and recording who has it is one transaction, so a job is
# never out of the queue without an owner.
job = yield from jobs.take()
yield from claimed.set(job, worker)
return job
def work(worker: str) -> fx.Effect[None]:
return (
fx.tx(claim(worker))
.flat_map(lambda _: fx.sleep("10 millis"))
.forever()
.pipe(fx.catch_tag(fx.Done, lambda _: fx.void))
)
workers = yield from fx.for_each(["w1", "w2"], lambda w: work(w).fork())
for job in ["a", "b", "c", "d", "e"]:
yield from jobs.offer(job) # waits while two jobs are queued
yield from jobs.end()
yield from fx.Fiber.join_all(workers)
owners = yield from claimed.snapshot()
print(dict(sorted(owners)))
fx.run_blocking(program())A TxPriorityQueue keeps its items sorted by key=, smallest first, as sorted(key=...)
would. Items with equal keys come out in the order they went in, and items without a
key= must be comparable themselves.
from dataclasses import dataclass
import pyfect as fx
@dataclass(frozen=True)
class Run:
job: str
priority: int # 0 runs first
@fx.fn
def program():
# `key=` orders the queue, smallest first, as `sorted(key=...)` would.
# Runs with the same key come out in the order they went in.
queue = yield from fx.TxPriorityQueue[Run].empty(key=lambda run: run.priority)
@fx.fn
def enqueue_due():
# Offered in one transaction: a taker sees all of them or none.
yield from queue.offer(Run("report", 2))
yield from queue.offer(Run("backup", 0))
yield from queue.offer(Run("mirror", 1))
yield from queue.offer(Run("cleanup", 2))
yield from fx.tx(enqueue_due())
taken = yield from fx.for_each(range(4), lambda _: queue.take())
print([run.job for run in taken])
fx.run_blocking(program())A TxHashSet is a set and a TxChunk a sequence (its value is a tuple), and a check
followed by a write on either is safe only inside one tx, like any other read and write.
import pyfect as fx
@fx.fn
def program():
taken = yield from fx.TxHashSet[str].empty()
bookings = yield from fx.TxChunk[str].empty()
@fx.fn
def book(seat: str, who: str):
# Checking the seat, taking it and logging the booking are one transaction,
# so two fibers can't both get it.
if (yield from taken.has(seat)):
return False
yield from taken.add(seat)
yield from bookings.append(f"{seat}: {who}")
return True
results = yield from fx.for_each(
[("1A", "ada"), ("1A", "grace"), ("1B", "edsger")],
lambda booking: fx.tx(book(*booking)),
concurrency="unbounded",
)
print(results, (yield from taken.size()), (yield from bookings.get()))
fx.run_blocking(program())Coordinating fibers#
The transactional versions of Deferred, Semaphore and a read/write lock wait through
tx_retry, so they compose with other refs in one transaction. TxDeferred.wait() waits
until the deferred is completed with succeed, fail or done(result).
TxSemaphore.make(n) hands out n permits: with_permit(eff) holds one while eff runs,
and acquire, release and try_acquire work inside a larger tx. A TxReentrantLock
lets any number of fibers hold a read lock (with_read_lock) while no fiber holds the write
lock (with_write_lock); a fiber can take a lock it already holds.
from typing import Never
import pyfect as fx
@fx.fn
def program():
ready = yield from fx.TxDeferred[str, Never].make()
slots = yield from fx.TxSemaphore.make(2)
lock = yield from fx.TxReentrantLock.make()
config = fx.TxRef("eu")
running, peak = fx.TxRef(0), fx.TxRef(0)
log: list[str] = []
@fx.fn
def started():
now = yield from running.modify(lambda n: (n + 1, n + 1))
yield from peak.update(lambda p: max(p, now))
@fx.fn
def fetch(n: int, host: str):
region = yield from lock.with_read_lock(config.get())
log.append(f"{n} from {host}/{region}")
yield from fx.tx(started())
yield from fx.sleep("10 millis")
yield from running.update(lambda n: n - 1)
@fx.fn
def download(n: int):
# wait() retries until the deferred is completed, as tx_retry does.
host = yield from ready.wait()
# At most two fetches at once, however many downloads are started.
yield from slots.with_permit(fetch(n, host))
workers = yield from fx.for_each(range(4), lambda n: download(n).fork())
yield from fx.sleep("10 millis")
assert log == [] # all waiting for `ready`
yield from ready.succeed("cdn")
yield from fx.Fiber.join_all(workers)
# The write lock is taken once no fiber holds a read lock.
yield from lock.with_write_lock(config.set("us"))
print("\n".join(log))
print("peak:", (yield from peak.get()), "region:", (yield from config.get()))
fx.run_blocking(program())A TxPubSub gives each subscriber its own TxQueue for as long as its scope lasts
(subscribe()), and publish offers a value to all of them in the running transaction, so
a transaction that rolls back publishes nothing. bounded, dropping, sliding and
unbounded choose what happens when a subscriber's queue is full, as for PubSub. A
TxSubscriptionRef is a TxRef that publishes every value it stores: changes() is a
queue of the current value and then each committed one, and changes_stream() the same as a
Stream.
import pyfect as fx
@fx.fn
def program():
stock = yield from fx.TxSubscriptionRef.make(10)
alerts = yield from fx.TxPubSub[str].unbounded()
@fx.fn
def sell(n: int):
left = yield from stock.update_and_get(lambda s: s - n)
if left < 5:
# Published in the same transaction: a rolled-back sale alerts nobody.
yield from alerts.publish(f"low stock: {left}")
@fx.fn
def watch():
# Subscriptions last as long as the scope; changes() starts with the
# current value.
changes = yield from stock.changes()
low = yield from alerts.subscribe()
yield from fx.tx(sell(3))
yield from fx.tx(sell(4))
print((yield from changes.take_all()), (yield from low.take_all()))
yield from fx.scoped(watch())
fx.run_blocking(program())Reference: pyfect.transaction,
pyfect.tx_ref, pyfect.tx_queue,
pyfect.tx_priority_queue,
pyfect.tx_hash_set, pyfect.tx_chunk,
pyfect.tx_deferred,
pyfect.tx_semaphore,
pyfect.tx_reentrant_lock,
pyfect.tx_pubsub,
pyfect.tx_subscription_ref.