Message storage in a SQL database (Effect's SqlMessageStorage).
Envelopes are rows of <prefix>_messages and replies rows of
<prefix>_replies, both keyed by their snowflakes. A request with a
primary key stores it in message_id (a SHA-256 digest when longer than
the 255-character column), which detects duplicates. unprocessed_messages
claims the rows it returns by setting last_read; a claim lapses after
ten minutes.
On MySQL and SQL Server the key columns (message_id, shard_id,
entity_type, entity_id) compare exactly, as on SQLite and
PostgreSQL (migration 0004, pyfect's): MySQL's in utf8mb4_0900_bin,
SQL Server's in a BIN2 collation, also comparing lengths where a
trailing space can occur (persistence._sql_keys); Effect leaves them in
the server's collation, which usually ignores case.
Python's integers are exact, so ids need no SafeIntegers; SHA-256 comes
from hashlib (Effect reads a Digest service).
Values#
| Attribute | Type | Description |
|---|---|---|
layer | layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig] |
|
Functions#
message_id_for_primary_key#
message_id_for_primary_key(primary_key) -> strThe stored key: the primary key when it fits the column (in UTF-16
code units, as Effect measures it), else its
SHA-256 hex digest (a digest has no /, a composed key always
does, so the two never collide).
| Parameter | Type | Description |
|---|---|---|
primary_key | str |
Returns str
def message_id_for_primary_key(primary_key: str) -> str:
"""The stored key: the primary key when it fits the column (in UTF-16
code units, as Effect measures it), else its
SHA-256 hex digest (a digest has no ``/``, a composed key always
does, so the two never collide)."""
if js_length(primary_key) <= _MESSAGE_ID_WIDTH:
return primary_key
return hashlib.sha256(text_encode(primary_key)).hexdigest()make_encoded_storage#
make_encoded_storage(*, prefix='cluster') -> Effect[EncodedStorage, Never, SqlClient]The driver on the tables of prefix (already migrated).
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[EncodedStorage, Never, SqlClient]
def make_encoded_storage(*, prefix: str = "cluster") -> Effect[EncodedStorage, Never, SqlClient]:
"""The driver on the tables of ``prefix`` (already migrated)."""
return SqlClient().map(lambda sql: _Storage(sql.without_transforms(), prefix))make#
make(*, prefix='cluster') -> Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]Run the migrations, then a message storage on the tables.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]
def make(
*, prefix: str = "cluster"
) -> Effect[MessageStorageService, Never, SqlClient | snowflake_.Generator]:
"""Run the migrations, then a message storage on the tables."""
return (
run_migrations(prefix=prefix)
.flat_map(lambda _: make_encoded_storage(prefix=prefix))
.flat_map(make_encoded)
)migrations#
migrations(*, prefix='cluster') -> migrator.LoaderThe message tables' migrations: 0001 to 0003 are Effect's,
0004 is pyfect's. PostgreSQL DDL is serialized by the migrator;
errors propagate (a retry in an aborted transaction cannot succeed).
Deployed databases have run the earlier migrations: a change to the
schema is a new one, never an edit of an old one.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns migrator.Loader
def migrations(*, prefix: str = "cluster") -> migrator.Loader:
"""The message tables' migrations: ``0001`` to ``0003`` are Effect's,
``0004`` is pyfect's. PostgreSQL DDL is serialized by the migrator;
errors propagate (a retry in an aborted transaction cannot succeed).
Deployed databases have run the earlier migrations: a change to the
schema is a new one, never an edit of an old one."""
messages_table = f"{prefix}_messages"
replies_table = f"{prefix}_replies"
def create_tables() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
yield from _create_messages(sql, messages_table)
yield from _index(
sql,
f"{messages_table}_shard_idx",
messages_table,
"shard_id, processed, last_read, deliver_at",
)
yield from _index(sql, f"{messages_table}_request_id_idx", messages_table, "request_id")
yield from _create_replies(sql, replies_table)
yield from _index(
sql, f"{replies_table}_request_lookup_idx", replies_table, "request_id, kind, acked"
)
def entity_type_size() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
messages = sql.identifier(messages_table)
yield from sql.on_dialect_or_else(
mssql=lambda: sql(
"ALTER TABLE {} ALTER COLUMN entity_type VARCHAR(150) NOT NULL;", messages
),
mysql=lambda: sql(
"ALTER TABLE {} MODIFY entity_type VARCHAR(150) NOT NULL;", messages
).unprepared.ignore(log=False),
pg=lambda: sql("ALTER TABLE {} ALTER COLUMN entity_type TYPE VARCHAR(150);", messages),
or_else=lambda: void,
)
def pg_rowid_index() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
yield from sql.on_dialect_or_else(
pg=lambda: _index(sql, f"{messages_table}_rowid_idx", messages_table, "rowid"),
or_else=lambda: void,
)
def exact_keys() -> Generator[Effect[Any, Any, Any], Any]:
sql = (yield from SqlClient()).without_transforms()
yield from sql.on_dialect_or_else(
mysql=lambda: _exact_keys_mysql(sql, messages_table),
mssql=lambda: _exact_keys_mssql(sql, messages_table),
or_else=lambda: void,
)
return migrator.from_record(
{
"0001_create_tables": gen(create_tables),
"0002_entity_type_size": gen(entity_type_size),
"0003_pg_messages_rowid_index": gen(pg_rowid_index),
"0004_exact_keys": gen(exact_keys),
}
)run_migrations#
run_migrations(*, prefix='cluster') -> Effect[None, Never, SqlClient]Run the message migrations; a failure is a defect.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns Effect[None, Never, SqlClient]
def run_migrations(*, prefix: str = "cluster") -> Effect[None, Never, SqlClient]:
"""Run the message migrations; a failure is a defect."""
return (
migrator.make()(loader=migrations(prefix=prefix), table=f"{prefix}_migrations")
.as_void()
.or_die()
)layer_migrations#
layer_migrations(*, prefix='cluster') -> layer_.Layer[Never, Never, SqlClient]| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[Never, Never, SqlClient]
def layer_migrations(*, prefix: str = "cluster") -> layer_.Layer[Never, Never, SqlClient]:
return layer_.effect_discard(run_migrations(prefix=prefix))layer_storage#
layer_storage(*, prefix='cluster') -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]Message storage on migrated tables (with its own snowflake generator).
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]
def layer_storage(
*, prefix: str = "cluster"
) -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]:
"""Message storage on migrated tables (with its own snowflake
generator)."""
return layer_.effect(
MessageStorage, make_encoded_storage(prefix=prefix).flat_map(make_encoded)
).pipe(layer_.provide(snowflake_.layer_generator))layer_with#
layer_with(*, prefix='cluster') -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]layer_storage after the migrations.
| Parameter | Type | Description |
|---|---|---|
prefix | str = 'cluster' |
Returns layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]
def layer_with(
*, prefix: str = "cluster"
) -> layer_.Layer[MessageStorage, Never, SqlClient | ShardingConfig]:
"""``layer_storage`` after the migrations."""
return layer_storage(prefix=prefix).pipe(layer_.provide(layer_migrations(prefix=prefix)))