GitHub

sql_runner_storage

API reference for sql_runner_storage.

Runner storage in a SQL database (Effect's SqlRunnerStorage).

Runners are rows of <prefix>_runners. Shard locks are PostgreSQL or MySQL advisory locks held by one reserved connection (rebuilt when a statement on it fails), or, on SQLite and SQL Server (or with shard_lock_disable_advisory), leases in <prefix>_locks that expire after shard_lock_expiration. Every lock operation has a deadline of the effective lock refresh interval.

On MySQL and SQL Server address and shard_id compare exactly, as on SQLite and PostgreSQL (migration 0002, pyfect's; Effect leaves them in the server's collation, which usually ignores case). Both end in digits, so a BIN2 collation is exact on SQL Server without comparing lengths. MySQL's named locks ignore case, so shards whose Effect lock names differ only in case get names of their own (_mysql_lock_names).

Values

AttributeTypeDescription
layerlayer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]

RunnerStorage on the SqlClient in tables prefixed cluster (layer_with()).

Functions

postgres_lock_namespace

postgres_lock_namespace(prefix) -> int

The advisory-lock namespace of prefix: FNV-1a of effect-cluster:<prefix> as a signed 32-bit integer. A persistent wire format shared with Effect: it must never change.

ParameterTypeDescription
prefixstr

Returns int

def postgres_lock_namespace(prefix: str) -> int:
    """The advisory-lock namespace of ``prefix``: FNV-1a of
    ``effect-cluster:<prefix>`` as a signed 32-bit integer. A persistent
    wire format shared with Effect: it must never change."""
    h = 0x811C9DC5
    for byte in text_encode(f"effect-cluster:{prefix}"):
        h = ((h ^ byte) * 0x01000193) & 0xFFFFFFFF
    return h - 0x1_0000_0000 if h >= 0x8000_0000 else h

make_storage

make_storage(*, prefix='cluster') -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]

Runner storage on the tables of prefix (already migrated).

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]

def make_storage(
    *, prefix: str = "cluster"
) -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]:
    """Runner storage on the tables of ``prefix`` (already migrated)."""

    def body() -> Generator[Effect[Any, Any, Any], Any, RunnerStorageService]:
        config = yield from ShardingConfig()
        sql = (yield from SqlClient()).without_transforms()
        layer_scope = yield from Scope()
        # PGlite runs every statement on one session: it gets table leases
        # on the shared client.
        single_session = sql.driver == "pglite"
        disable_advisory = config.shard_lock_disable_advisory or single_session
        acquire = None if single_session else _acquire_lock_conn(sql, disable_advisory, config)
        lock_conn = None
        if acquire is not None:
            lock_conn = yield from ResourceRef.make(layer_scope, acquire)
        storage = _Storage(
            sql=sql,
            config=config,
            prefix=prefix,
            layer_scope=layer_scope,
            lock_conn=lock_conn,
            disable_advisory=disable_advisory,
        )
        return make_encoded(storage)

    return gen(body).with_tracer_enabled(False)

migrations

migrations(*, prefix='cluster') -> migrator.Loader

The runner tables' migrations (existing deployments may have the tables without a migration history: they are created if absent).

ParameterTypeDescription
prefixstr = 'cluster'

Returns migrator.Loader

def migrations(*, prefix: str = "cluster") -> migrator.Loader:
    """The runner tables' migrations (existing deployments may have the
    tables without a migration history: they are created if absent)."""
    runners_table = f"{prefix}_runners"
    locks_table = f"{prefix}_locks"

    def create() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        runners = sql.identifier(runners_table)
        locks = sql.identifier(locks_table)
        yield from sql.on_dialect_or_else(
            mssql=lambda: sql(
                "IF OBJECT_ID(N'{}', N'U') IS NULL CREATE TABLE {} ("
                "machine_id INT IDENTITY PRIMARY KEY, address VARCHAR(255) NOT NULL, "
                "runner TEXT NOT NULL, healthy BIT NOT NULL DEFAULT 1, "
                "last_heartbeat DATETIME NOT NULL DEFAULT GETDATE(), UNIQUE(address))",
                runners,
                runners,
            ),
            mysql=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} ("
                "machine_id INT AUTO_INCREMENT PRIMARY KEY, address VARCHAR(255) NOT NULL, "
                "runner TEXT NOT NULL, healthy BOOLEAN NOT NULL DEFAULT TRUE, "
                "last_heartbeat DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(address))",
                runners,
            ),
            pg=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} ("
                "machine_id SERIAL PRIMARY KEY, address VARCHAR(255) NOT NULL, "
                "runner TEXT NOT NULL, healthy BOOLEAN NOT NULL DEFAULT TRUE, "
                "last_heartbeat TIMESTAMP NOT NULL DEFAULT NOW(), UNIQUE(address))",
                runners,
            ),
            or_else=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} ("
                "machine_id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT NOT NULL, "
                "runner TEXT NOT NULL, healthy INTEGER NOT NULL DEFAULT 1, "
                "last_heartbeat DATETIME NOT NULL DEFAULT (CURRENT_TIMESTAMP), UNIQUE(address))",
                runners,
            ),
        )
        yield from sql.on_dialect_or_else(
            mssql=lambda: sql(
                "IF OBJECT_ID(N'{}', N'U') IS NULL CREATE TABLE {} ("
                "shard_id VARCHAR(50) PRIMARY KEY, address VARCHAR(255) NOT NULL, "
                "acquired_at DATETIME NOT NULL)",
                locks,
                locks,
            ),
            mysql=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} (shard_id VARCHAR(50) PRIMARY KEY, "
                "address VARCHAR(255) NOT NULL, acquired_at DATETIME NOT NULL)",
                locks,
            ),
            pg=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} (shard_id VARCHAR(50) PRIMARY KEY, "
                "address VARCHAR(255) NOT NULL, acquired_at TIMESTAMP NOT NULL)",
                locks,
            ),
            or_else=lambda: sql(
                "CREATE TABLE IF NOT EXISTS {} (shard_id TEXT PRIMARY KEY, "
                "address TEXT NOT NULL, acquired_at DATETIME NOT NULL)",
                locks,
            ),
        )

    def exact_keys() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        yield from sql.on_dialect_or_else(
            mysql=lambda: _exact_keys_mysql(sql, runners_table, locks_table),
            mssql=lambda: _exact_keys_mssql(sql, runners_table, locks_table),
            or_else=lambda: void,
        )

    return migrator.from_record(
        {"0001_create_tables": gen(create), "0002_exact_keys": gen(exact_keys)}
    )

run_migrations

run_migrations(*, prefix='cluster') -> Effect[None, Never, SqlClient]

Run the runner migrations; a failure is a defect. Message and runner migration ids overlap, so they keep separate histories.

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[None, Never, SqlClient]

def run_migrations(*, prefix: str = "cluster") -> Effect[None, Never, SqlClient]:
    """Run the runner migrations; a failure is a defect. Message and runner
    migration ids overlap, so they keep separate histories."""
    return (
        migrator.make()(loader=migrations(prefix=prefix), table=f"{prefix}_runner_migrations")
        .as_void()
        .or_die()
    )

make

make(*, prefix='cluster') -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]

Run the migrations, then make_storage.

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]

def make(
    *, prefix: str = "cluster"
) -> Effect[RunnerStorageService, Never, SqlClient | ShardingConfig | Scope]:
    """Run the migrations, then ``make_storage``."""
    return run_migrations(prefix=prefix).flat_map(lambda _: make_storage(prefix=prefix))

layer_migrations

layer_migrations(*, prefix='cluster') -> layer_.Layer[Never, Never, SqlClient]
ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[Never, Never, SqlClient]

def layer_migrations(*, prefix: str = "cluster") -> layer_.Layer[Never, Never, SqlClient]:
    return layer_.effect_discard(run_migrations(prefix=prefix))

layer_storage

layer_storage(*, prefix='cluster') -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]
ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]

def layer_storage(
    *, prefix: str = "cluster"
) -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]:
    return layer_.effect(RunnerStorage, make_storage(prefix=prefix))

layer_with

layer_with(*, prefix='cluster') -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]
ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]

def layer_with(
    *, prefix: str = "cluster"
) -> layer_.Layer[RunnerStorage, Never, SqlClient | ShardingConfig]:
    return layer_.effect(RunnerStorage, make(prefix=prefix))

effective_interval

effective_interval(config) -> Duration

The refresh interval, at most a third of the lock expiration.

ParameterTypeDescription
configShardingConfigService

Returns Duration

def effective_interval(config: ShardingConfigService) -> Duration:
    """The refresh interval, at most a third of the lock expiration."""
    return min(config.shard_lock_refresh_interval, config.shard_lock_expiration.divide_unsafe(3))