Runner storage in a SQL database (Effect's SqlRunnerStorage).
Runners are rows of <prefix>_runners. Shard locks are PostgreSQL or
MySQL advisory locks held by one reserved connection (rebuilt when a
statement on it fails), or, on SQLite and SQL Server (or with
shard_lock_disable_advisory), leases in <prefix>_locks that
expire after shard_lock_expiration. Every lock operation has a
deadline of the effective lock refresh interval.
On MySQL and SQL Server address and shard_id compare exactly, as
on SQLite and PostgreSQL (migration 0002, pyfect's; Effect leaves them
in the server's collation, which usually ignores case). Both end in
digits, so a BIN2 collation is exact on SQL Server without comparing
lengths. MySQL's named locks ignore case, so shards whose Effect lock
names differ only in case get names of their own (_mysql_lock_names).
Values#
| Attribute | Type | Description |
|---|---|---|
layer | layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig] |
|
Functions#
postgres_lock_namespace#
postgres_lock_namespace(prefix) -> intThe advisory-lock namespace of prefix: FNV-1a of
effect-cluster:<prefix> as a signed 32-bit integer. A persistent
wire format shared with Effect: it must never change.
| Parameter | Type | Description |
|---|---|---|
prefix | str |
Returns int
def postgres_lock_namespace(prefix: str) -> int:
"""The advisory-lock namespace of ``prefix``: FNV-1a of
``effect-cluster:<prefix>`` as a signed 32-bit integer. A persistent
wire format shared with Effect: it must never change."""
h = 0x811C9DC5
for byte in text_encode(f"effect-cluster:{prefix}"):
h = ((h ^ byte) * 0x01000193) & 0xFFFFFFFF
return h - 0x1_0000_0000 if h >= 0x8000_0000 else hmake_storage#
make_storage(*, prefix='cluster') -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]Runner storage on the tables of prefix (already migrated).
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]
def make_storage(
*, prefix: str = "cluster"
) -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]:
"""Runner storage on the tables of ``prefix`` (already migrated)."""
def body() -> Generator[Effect[Any, Any, Any], Any, RunnerStorageService]:
config = yield from ShardingConfig()
sql = (yield from SqlClient()).without_transforms()
layer_scope = yield from Scope()
# PGlite runs every statement on one session: it gets table leases
# on the shared client.
single_session = sql.driver == "pglite"
disable_advisory = config.shard_lock_disable_advisory or single_session
acquire = None if single_session else _acquire_lock_conn(sql, disable_advisory, config)
lock_conn = None
if acquire is not None:
lock_conn = yield from ResourceRef.make(layer_scope, acquire)
storage = _Storage(
sql=sql,
config=config,
prefix=prefix,
layer_scope=layer_scope,
lock_conn=lock_conn,
disable_advisory=disable_advisory,
)
return make_encoded(storage)
return gen(body).with_tracer_enabled(False)migrations#
migrations(*, prefix='cluster') -> migrator.LoaderThe runner tables' migrations (existing deployments may have the tables without a migration history: they are created if absent).
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns migrator.Loader
def migrations(*, prefix: str = "cluster") -> migrator.Loader:
"""The runner tables' migrations (existing deployments may have the
tables without a migration history: they are created if absent)."""
runners_table = f"{prefix}_runners"
locks_table = f"{prefix}_locks"
def create() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
runners = sql.identifier(runners_table)
locks = sql.identifier(locks_table)
yield from sql.on_dialect_or_else(
mssql=lambda: sql(
"IF OBJECT_ID(N'{}', N'U') IS NULL CREATE TABLE {} ("
"machine_id INT IDENTITY PRIMARY KEY, address VARCHAR(255) NOT NULL, "
"runner TEXT NOT NULL, healthy BIT NOT NULL DEFAULT 1, "
"last_heartbeat DATETIME NOT NULL DEFAULT GETDATE(), UNIQUE(address))",
runners,
runners,
),
mysql=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} ("
"machine_id INT AUTO_INCREMENT PRIMARY KEY, address VARCHAR(255) NOT NULL, "
"runner TEXT NOT NULL, healthy BOOLEAN NOT NULL DEFAULT TRUE, "
"last_heartbeat DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(address))",
runners,
),
pg=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} ("
"machine_id SERIAL PRIMARY KEY, address VARCHAR(255) NOT NULL, "
"runner TEXT NOT NULL, healthy BOOLEAN NOT NULL DEFAULT TRUE, "
"last_heartbeat TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE(address))",
runners,
),
or_else=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} ("
"machine_id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT NOT NULL, "
"runner TEXT NOT NULL, healthy INTEGER NOT NULL DEFAULT 1, "
"last_heartbeat DATETIME NOT NULL DEFAULT (CURRENT_TIMESTAMP), UNIQUE(address))",
runners,
),
)
yield from sql.on_dialect_or_else(
mssql=lambda: sql(
"IF OBJECT_ID(N'{}', N'U') IS NULL CREATE TABLE {} ("
"shard_id VARCHAR(50) PRIMARY KEY, address VARCHAR(255) NOT NULL, "
"acquired_at DATETIME NOT NULL)",
locks,
locks,
),
mysql=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} (shard_id VARCHAR(50) PRIMARY KEY, "
"address VARCHAR(255) NOT NULL, acquired_at DATETIME NOT NULL)",
locks,
),
pg=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} (shard_id VARCHAR(50) PRIMARY KEY, "
"address VARCHAR(255) NOT NULL, acquired_at TIMESTAMP NOT NULL)",
locks,
),
or_else=lambda: sql(
"CREATE TABLE IF NOT EXISTS {} (shard_id TEXT PRIMARY KEY, "
"address TEXT NOT NULL, acquired_at DATETIME NOT NULL)",
locks,
),
)
def exact_keys() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
yield from sql.on_dialect_or_else(
mysql=lambda: _exact_keys_mysql(sql, runners_table, locks_table),
mssql=lambda: _exact_keys_mssql(sql, runners_table, locks_table),
or_else=lambda: void,
)
return migrator.from_record(
{"0001_create_tables": gen(create), "0002_exact_keys": gen(exact_keys)}
)run_migrations#
run_migrations(*, prefix='cluster') -> Effect[None, Never, SqlClient]Run the runner migrations; a failure is a defect. Message and runner migration ids overlap, so they keep separate histories.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[None, Never, SqlClient]
def run_migrations(*, prefix: str = "cluster") -> Effect[None, Never, SqlClient]:
"""Run the runner migrations; a failure is a defect. Message and runner
migration ids overlap, so they keep separate histories."""
return (
migrator.make()(loader=migrations(prefix=prefix), table=f"{prefix}_runner_migrations")
.as_void()
.or_die()
)make#
make(*, prefix='cluster') -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]Run the migrations, then make_storage.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]
def make(
*, prefix: str = "cluster"
) -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]:
"""Run the migrations, then ``make_storage``."""
return run_migrations(prefix=prefix).flat_map(lambda _: make_storage(prefix=prefix))layer_migrations#
layer_migrations(*, prefix='cluster') -> layer_.Layer[Never, Never, SqlClient]| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[Never, Never, SqlClient]
def layer_migrations(*, prefix: str = "cluster") -> layer_.Layer[Never, Never, SqlClient]:
return layer_.effect_discard(run_migrations(prefix=prefix))layer_storage#
layer_storage(*, prefix='cluster') -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]
def layer_storage(
*, prefix: str = "cluster"
) -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]:
return layer_.effect(RunnerStorage, make_storage(prefix=prefix))layer_with#
layer_with(*, prefix='cluster') -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]
def layer_with(
*, prefix: str = "cluster"
) -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]:
return layer_.effect(RunnerStorage, make(prefix=prefix))effective_interval#
effective_interval(config) -> DurationThe refresh interval, at most a third of the lock expiration.
| Parameter | Type | Description |
|---|---|---|
config | ShardingConfigService |
Returns Duration
def effective_interval(config: ShardingConfigService) -> Duration:
"""The refresh interval, at most a third of the lock expiration."""
return min(config.shard_lock_refresh_interval, config.shard_lock_expiration.divide_unsafe(3))