GitHub

sql_message_storage

API reference for sql_message_storage.

Message storage in a SQL database (Effect's SqlMessageStorage).

Envelopes are rows of <prefix>_messages and replies rows of <prefix>_replies, both keyed by their snowflakes. A request with a primary key stores it in message_id (a SHA-256 digest when longer than the 255-character column), which detects duplicates. unprocessed_messages claims the rows it returns by setting last_read; a claim lapses after ten minutes.

On MySQL and SQL Server the key columns (message_id, shard_id, entity_type, entity_id) compare exactly, as on SQLite and PostgreSQL (migration 0004, pyfect's): MySQL's in utf8mb4_0900_bin, SQL Server's in a BIN2 collation, also comparing lengths where a trailing space can occur (persistence._sql_keys); Effect leaves them in the server's collation, which usually ignores case. Python's integers are exact, so ids need no SafeIntegers; SHA-256 comes from hashlib (Effect reads a Digest service).

Values

AttributeTypeDescription
layerlayer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]

MessageStorage on the SqlClient in tables prefixed cluster, after their migrations (layer_with()).

Functions

message_id_for_primary_key

message_id_for_primary_key(primary_key) -> str

The stored key: the primary key when it fits the column (in UTF-16 code units, as Effect measures it), else its SHA-256 hex digest (a digest has no /, a composed key always does, so the two never collide).

ParameterTypeDescription
primary_keystr

Returns str

def message_id_for_primary_key(primary_key: str) -> str:
    """The stored key: the primary key when it fits the column (in UTF-16
    code units, as Effect measures it), else its
    SHA-256 hex digest (a digest has no ``/``, a composed key always
    does, so the two never collide)."""
    if js_length(primary_key) <= _MESSAGE_ID_WIDTH:
        return primary_key
    return hashlib.sha256(text_encode(primary_key)).hexdigest()

make_encoded_storage

make_encoded_storage(*, prefix='cluster') -> Effect[EncodedStorage, Never, SqlClient]

The driver on the tables of prefix (already migrated).

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[EncodedStorage, Never, SqlClient]

def make_encoded_storage(*, prefix: str = "cluster") -> Effect[EncodedStorage, Never, SqlClient]:
    """The driver on the tables of ``prefix`` (already migrated)."""
    return SqlClient().map(lambda sql: _Storage(sql.without_transforms(), prefix))

make

make(*, prefix='cluster') -> Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]

Run the migrations, then a message storage on the tables.

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]

def make(
    *, prefix: str = "cluster"
) -> Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]:
    """Run the migrations, then a message storage on the tables."""
    return (
        run_migrations(prefix=prefix)
        .flat_map(lambda _: make_encoded_storage(prefix=prefix))
        .flat_map(make_encoded)
    )

migrations

migrations(*, prefix='cluster') -> migrator.Loader

The message tables' migrations: 0001 to 0003 are Effect's, 0004 is pyfect's. PostgreSQL DDL is serialized by the migrator; errors propagate (a retry in an aborted transaction cannot succeed). Deployed databases have run the earlier migrations: a change to the schema is a new one, never an edit of an old one.

ParameterTypeDescription
prefixstr = 'cluster'

Returns migrator.Loader

def migrations(*, prefix: str = "cluster") -> migrator.Loader:
    """The message tables' migrations: ``0001`` to ``0003`` are Effect's,
    ``0004`` is pyfect's. PostgreSQL DDL is serialized by the migrator;
    errors propagate (a retry in an aborted transaction cannot succeed).
    Deployed databases have run the earlier migrations: a change to the
    schema is a new one, never an edit of an old one."""
    messages_table = f"{prefix}_messages"
    replies_table = f"{prefix}_replies"

    def create_tables() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        yield from _create_messages(sql, messages_table)
        yield from _index(
            sql,
            f"{messages_table}_shard_idx",
            messages_table,
            "shard_id, processed, last_read, deliver_at",
        )
        yield from _index(sql, f"{messages_table}_request_id_idx", messages_table, "request_id")
        yield from _create_replies(sql, replies_table)
        yield from _index(
            sql, f"{replies_table}_request_lookup_idx", replies_table, "request_id, kind, acked"
        )

    def entity_type_size() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        messages = sql.identifier(messages_table)
        yield from sql.on_dialect_or_else(
            mssql=lambda: sql(
                "ALTER TABLE {} ALTER COLUMN entity_type VARCHAR(150) NOT NULL;", messages
            ),
            mysql=lambda: sql(
                "ALTER TABLE {} MODIFY entity_type VARCHAR(150) NOT NULL;", messages
            ).unprepared.ignore(log=False),
            pg=lambda: sql("ALTER TABLE {} ALTER COLUMN entity_type TYPE VARCHAR(150);", messages),
            or_else=lambda: void,
        )

    def pg_rowid_index() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        yield from sql.on_dialect_or_else(
            pg=lambda: _index(sql, f"{messages_table}_rowid_idx", messages_table, "rowid"),
            or_else=lambda: void,
        )

    def exact_keys() -> Generator[Effect[Any, Any, Any], Any]:
        sql = (yield from SqlClient()).without_transforms()
        yield from sql.on_dialect_or_else(
            mysql=lambda: _exact_keys_mysql(sql, messages_table),
            mssql=lambda: _exact_keys_mssql(sql, messages_table),
            or_else=lambda: void,
        )

    return migrator.from_record(
        {
            "0001_create_tables": gen(create_tables),
            "0002_entity_type_size": gen(entity_type_size),
            "0003_pg_messages_rowid_index": gen(pg_rowid_index),
            "0004_exact_keys": gen(exact_keys),
        }
    )

run_migrations

run_migrations(*, prefix='cluster') -> Effect[None, Never, SqlClient]

Run the message migrations; a failure is a defect.

ParameterTypeDescription
prefixstr = 'cluster'

Returns Effect[None, Never, SqlClient]

def run_migrations(*, prefix: str = "cluster") -> Effect[None, Never, SqlClient]:
    """Run the message migrations; a failure is a defect."""
    return (
        migrator.make()(loader=migrations(prefix=prefix), table=f"{prefix}_migrations")
        .as_void()
        .or_die()
    )

layer_migrations

layer_migrations(*, prefix='cluster') -> layer_.Layer[Never, Never, SqlClient]
ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[Never, Never, SqlClient]

def layer_migrations(*, prefix: str = "cluster") -> layer_.Layer[Never, Never, SqlClient]:
    return layer_.effect_discard(run_migrations(prefix=prefix))

layer_storage

layer_storage(*, prefix='cluster') -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]

Message storage on migrated tables (with its own snowflake generator).

ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]

def layer_storage(
    *, prefix: str = "cluster"
) -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]:
    """Message storage on migrated tables (with its own snowflake
    generator)."""
    return layer_.effect(
        MessageStorage, make_encoded_storage(prefix=prefix).flat_map(make_encoded)
    ).pipe(layer_.provide(snowflake_.layer_generator))

layer_with

layer_with(*, prefix='cluster') -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]

layer_storage after the migrations.

ParameterTypeDescription
prefixstr = 'cluster'

Returns layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]

def layer_with(
    *, prefix: str = "cluster"
) -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]:
    """``layer_storage`` after the migrations."""
    return layer_storage(prefix=prefix).pipe(layer_.provide(layer_migrations(prefix=prefix)))