A string-keyed store of text and bytes (Effect's
effect/persistence/KeyValueStore)::
kv = yield from KeyValueStore()
yield from kv.set("greeting", "hello")
text = yield from kv.get("greeting") # "hello"; None when missinglayer_memory: a dict;layer_file_system(directory): a file per key;layer_sql(table=...): a SQL table;layer_storage(evaluate): anyMutableMapping[str, str].make/make_string_onlybuild a store from a backend's operations;store.prefix(p)putspbefore every key.store.to_schema_store(schema)holds one schema's values as JSON text.
Bytes read as text come back as Base64, text read as bytes as UTF-8.
Values#
| Attribute | Type | Description |
|---|---|---|
layer_memory | Layer[KeyValueStore] | A fresh in-memory store each time the layer is built. |
Types#
| Type | Definition | Description |
|---|---|---|
Value | type Value = str | bytes | What a store holds: text, or bytes (Effect's |
Classes#
Functions#
layer_file_system#
layer_file_system(directory) -> Layer[KeyValueStore, PlatformError, FileSystem | Path]A store keeping each value in a file of directory (made if
missing) named by its percent-encoded key, upper-case letters and a
trailing . escaped too (files 0.6.5 named are renamed on open). The keys "", "."
and ".." are rejected without touching the file system. keys
decodes the file names (failing on one that is not percent-encoded
UTF-8), and clear_prefix skips such files.
| Parameter | Type | Description |
|---|---|---|
directory | str |
Returns Layer[KeyValueStore, PlatformError, FileSystem | Path]
def layer_file_system(
directory: str,
) -> Layer[KeyValueStore, PlatformError, FileSystem | Path]:
"""A store keeping each value in a file of ``directory`` (made if
missing) named by its percent-encoded key, upper-case letters and a
trailing ``.`` escaped too (files 0.6.5 named are renamed on open). The keys ``""``, ``"."``
and ``".."`` are rejected without touching the file system. ``keys``
decodes the file names (failing on one that is not percent-encoded
UTF-8), and ``clear_prefix`` skips such files."""
def build() -> Generator[Effect[Any, PlatformError, FileSystem | Path], Any, Any]:
fs = yield from FileSystem()
path = yield from Path()
if not (yield from fs.exists(directory)):
yield from fs.make_directory(directory, recursive=True)
yield from _rename_legacy_files(fs, path, directory)
def with_key_path[A](
method: str, key: str, f: Callable[[str], Effect[A, KeyValueStoreError]]
) -> Effect[A, KeyValueStoreError]:
name = _file_name(key)
if name is None:
return fail(KeyValueStoreError(f"Invalid key {key}", method, key))
return f(path.join(directory, name))
def failed(
method: str, key: str, message: str
) -> Callable[[PlatformError], KeyValueStoreError]:
return lambda cause: KeyValueStoreError(message, method, key, cause)
def missing_or_fail(
method: str, key: str
) -> Callable[[PlatformError], Effect[None, KeyValueStoreError]]:
def handle(cause: PlatformError) -> Effect[None, KeyValueStoreError]:
if isinstance(cause.reason, NotFound):
return succeed(None)
return fail(
KeyValueStoreError(f"Unable to get item with key {key}", method, key, cause)
)
return handle
def get(key: str) -> Effect[str | None, KeyValueStoreError]:
return with_key_path(
"get",
key,
lambda p: fs.read_file_string(p).catch_all(missing_or_fail("get", key)),
)
def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
return with_key_path(
"get_bytes",
key,
lambda p: fs.read_file(p).catch_all(missing_or_fail("get_bytes", key)),
)
def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
def write(p: str) -> Effect[None, KeyValueStoreError]:
written = (
fs.write_file_string(p, value)
if isinstance(value, str)
else fs.write_file(p, value)
)
return written.map_error(failed("set", key, f"Unable to set item with key {key}"))
return with_key_path("set", key, write)
def remove(key: str) -> Effect[None, KeyValueStoreError]:
return with_key_path(
"remove",
key,
lambda p: fs.remove(p, force=True).map_error(
failed("remove", key, f"Unable to remove item with key {key}")
),
)
def has(key: str) -> Effect[bool, KeyValueStoreError]:
return with_key_path(
"has",
key,
lambda p: fs.exists(p).map_error(
failed("has", key, f"Unable to check existence of item with key {key}")
),
)
clear = (
fs.remove(directory, recursive=True)
.and_then(fs.make_directory(directory, recursive=True))
.map_error(
lambda cause: KeyValueStoreError("Unable to clear storage", "clear", None, cause)
)
)
size = (
fs.read_directory(directory)
.map(len)
.map_error(lambda cause: KeyValueStoreError("Unable to get size", "size", None, cause))
)
def remove_file(name: str) -> Effect[None, PlatformError]:
return fs.remove(path.join(directory, name)).catch_all(
lambda cause: succeed(None) if isinstance(cause.reason, NotFound) else fail(cause)
)
def clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
def matching(files: list[str]) -> list[str]:
return [
name
for name in files
if (key := _decode_file_key(name)) is not None and key.startswith(prefix)
]
return (
fs.read_directory(directory)
.flat_map(lambda files: for_each(matching(files), remove_file))
.as_void()
.map_error(
lambda cause: KeyValueStoreError(
f"Unable to clear items with prefix {prefix}", "clear_prefix", prefix, cause
)
)
)
def decode_all(files: list[str]) -> Effect[list[str], Exception]:
return try_(lambda: [_strict_decode(name) for name in files], lambda cause: cause)
keys: Effect[list[str], KeyValueStoreError] = (
fs.read_directory(directory)
.flat_map(decode_all)
.map_error(lambda cause: KeyValueStoreError("Unable to get keys", "keys", None, cause))
)
return make(
get=get,
get_bytes=get_bytes,
set=set_value,
remove=remove,
has=has,
clear=clear,
clear_prefix=clear_prefix,
size=size,
keys=keys,
)
return layer_effect(KeyValueStore, gen(build))layer_storage#
layer_storage(evaluate) -> Layer[KeyValueStore]A store over the text mapping evaluate() returns when the layer
is built (Effect's layerStorage over Web Storage; a shelve
database, say). What the mapping raises becomes a
KeyValueStoreError.
| Parameter | Type | Description |
|---|---|---|
evaluate | Callable[[], MutableMapping[str, str]] |
Returns Layer[KeyValueStore]
def layer_storage(evaluate: Callable[[], MutableMapping[str, str]]) -> Layer[KeyValueStore]:
"""A store over the text mapping ``evaluate()`` returns when the layer
is built (Effect's ``layerStorage`` over Web ``Storage``; a ``shelve``
database, say). What the mapping raises becomes a
``KeyValueStoreError``."""
def build() -> KeyValueStoreService:
storage = evaluate()
def attempt[A](
method: str, key: str | None, message: str, thunk: Callable[[], A]
) -> Effect[A, KeyValueStoreError]:
return try_(thunk, lambda cause: KeyValueStoreError(message, method, key, cause))
return make_string_only(
get=lambda key: attempt(
"get", key, f"Unable to get item with key {key}", lambda: storage.get(key)
),
set=lambda key, value: attempt(
"set",
key,
f"Unable to set item with key {key}",
lambda: storage.__setitem__(key, value),
),
remove=lambda key: attempt(
"remove",
key,
f"Unable to remove item with key {key}",
lambda: storage.pop(key, None),
).as_void(),
clear=attempt("clear", None, "Unable to clear storage", storage.clear),
clear_prefix=lambda prefix: attempt(
"clear_prefix",
prefix,
f"Unable to clear items with prefix {prefix}",
lambda: _remove_prefixed(storage, prefix),
),
size=attempt("size", None, "Unable to get size", lambda: len(storage)),
keys=attempt("keys", None, "Unable to get keys", lambda: list(storage)),
)
return layer_sync(KeyValueStore, build)make_memory#
make_memory() -> KeyValueStoreServiceA store in a dict of its own.
Returns KeyValueStoreService
def make_memory() -> KeyValueStoreService:
"""A store in a dict of its own."""
store: dict[str, Value] = {}
def get(key: str) -> Effect[str | None, KeyValueStoreError]:
def read() -> str | None:
value = store.get(key)
return value if value is None or isinstance(value, str) else base64_encode(value)
return sync(read)
def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
def read() -> bytes | None:
value = store.get(key)
return text_encode(value) if isinstance(value, str) else value
return sync(read)
def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
return sync(lambda: store.__setitem__(key, value))
def clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
def clear() -> None:
for key in [key for key in store if key.startswith(prefix)]:
del store[key]
return sync(clear)
return make(
get=get,
get_bytes=get_bytes,
set=set_value,
remove=lambda key: sync(lambda: store.pop(key, None)).as_void(),
clear=sync(store.clear),
clear_prefix=clear_prefix,
size=sync(lambda: len(store)),
keys=sync(lambda: list(store)),
)layer_sql#
layer_sql(*, table='effect_key_value_store') -> Layer[KeyValueStore, Never, SqlClient]A store in table (made if missing) of the SqlClient: one row
per key, the value as bytes and whether it was text.
| Parameter | Type | Description |
|---|---|---|
table | str = 'effect_key_value_store' |
Returns Layer[KeyValueStore, Never, SqlClient]
def layer_sql(*, table: str = "effect_key_value_store") -> Layer[KeyValueStore, Never, SqlClient]:
"""A store in ``table`` (made if missing) of the ``SqlClient``: one row
per key, the value as bytes and whether it was text."""
return layer_effect(KeyValueStore, gen(lambda: _build(table)))make#
make(*, get, get_bytes, set, remove, clear, size, keys, clear_prefix=None, has=None, is_empty=None, modify=None, modify_bytes=None) -> KeyValueStoreServiceA store from its backend's operations; has, is_empty,
clear_prefix, modify and modify_bytes are derived unless
given. The derived clear_prefix is clear for "", else it
removes each of keys starting with the prefix.
| Parameter | Type | Description |
|---|---|---|
get | Callable[[str], Effect[str | None, KeyValueStoreError]] | |
get_bytes | Callable[[str], Effect[bytes | None, KeyValueStoreError]] | |
set | Callable[[str, Value], Effect[None, KeyValueStoreError]] | |
remove | Callable[[str], Effect[None, KeyValueStoreError]] | |
clear | Effect[None, KeyValueStoreError] | |
size | Effect[int, KeyValueStoreError] | |
keys | Effect[list[str], KeyValueStoreError] | |
clear_prefix | Callable[[str], Effect[None, KeyValueStoreError]] | None = None | |
has | Callable[[str], Effect[bool, KeyValueStoreError]] | None = None | |
is_empty | Effect[bool, KeyValueStoreError] | None = None | |
modify | Callable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]] | None = None | |
modify_bytes | Callable[[str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]] | None = None |
Returns KeyValueStoreService
def make(
*,
get: Callable[[str], Effect[str | None, KeyValueStoreError]],
get_bytes: Callable[[str], Effect[bytes | None, KeyValueStoreError]],
set: Callable[[str, Value], Effect[None, KeyValueStoreError]],
remove: Callable[[str], Effect[None, KeyValueStoreError]],
clear: Effect[None, KeyValueStoreError],
size: Effect[int, KeyValueStoreError],
keys: Effect[list[str], KeyValueStoreError],
clear_prefix: Callable[[str], Effect[None, KeyValueStoreError]] | None = None,
has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
is_empty: Effect[bool, KeyValueStoreError] | None = None,
modify: Callable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]]
| None = None,
modify_bytes: Callable[
[str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]
]
| None = None,
) -> KeyValueStoreService:
"""A store from its backend's operations; ``has``, ``is_empty``,
``clear_prefix``, ``modify`` and ``modify_bytes`` are derived unless
given. The derived ``clear_prefix`` is ``clear`` for ``""``, else it
removes each of ``keys`` starting with the prefix."""
require_effect(clear, "clear")
require_effect(size, "size")
require_effect(keys, "keys")
if is_empty is not None:
require_effect(is_empty, "is_empty")
def derived_clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
if not prefix:
return clear
return keys.flat_map(
lambda found: for_each([key for key in found if key.startswith(prefix)], remove)
).as_void()
def derived_has(key: str) -> Effect[bool, KeyValueStoreError]:
return get(key).map(lambda value: value is not None)
def derived_modify(key: str, f: Callable[[str], str]) -> Effect[str | None, KeyValueStoreError]:
def update(value: str | None) -> Effect[str | None, KeyValueStoreError]:
if value is None:
return succeed(None)
updated = f(value)
return set(key, updated).as_(updated)
return get(key).flat_map(update)
def derived_modify_bytes(
key: str, f: Callable[[bytes], bytes]
) -> Effect[bytes | None, KeyValueStoreError]:
def update(value: bytes | None) -> Effect[bytes | None, KeyValueStoreError]:
if value is None:
return succeed(None)
updated = f(value)
return set(key, updated).as_(updated)
return get_bytes(key).flat_map(update)
return KeyValueStoreService(
get=get,
get_bytes=get_bytes,
set=set,
remove=remove,
clear=clear,
clear_prefix=clear_prefix or derived_clear_prefix,
size=size,
keys=keys,
has=has or derived_has,
is_empty=is_empty or size.map(lambda n: n == 0),
modify=modify or derived_modify,
modify_bytes=modify_bytes or derived_modify_bytes,
)make_string_only#
make_string_only(*, get, set, remove, clear, size, keys, clear_prefix=None, has=None, is_empty=None) -> KeyValueStoreServiceA store over a backend that holds only text: bytes are written as
Base64, and get_bytes reads a value that is valid Base64 as the
bytes it encodes, any other as its UTF-8.
| Parameter | Type | Description |
|---|---|---|
get | Callable[[str], Effect[str | None, KeyValueStoreError]] | |
set | Callable[[str, str], Effect[None, KeyValueStoreError]] | |
remove | Callable[[str], Effect[None, KeyValueStoreError]] | |
clear | Effect[None, KeyValueStoreError] | |
size | Effect[int, KeyValueStoreError] | |
keys | Effect[list[str], KeyValueStoreError] | |
clear_prefix | Callable[[str], Effect[None, KeyValueStoreError]] | None = None | |
has | Callable[[str], Effect[bool, KeyValueStoreError]] | None = None | |
is_empty | Effect[bool, KeyValueStoreError] | None = None |
Returns KeyValueStoreService
def make_string_only(
*,
get: Callable[[str], Effect[str | None, KeyValueStoreError]],
set: Callable[[str, str], Effect[None, KeyValueStoreError]],
remove: Callable[[str], Effect[None, KeyValueStoreError]],
clear: Effect[None, KeyValueStoreError],
size: Effect[int, KeyValueStoreError],
keys: Effect[list[str], KeyValueStoreError],
clear_prefix: Callable[[str], Effect[None, KeyValueStoreError]] | None = None,
has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
is_empty: Effect[bool, KeyValueStoreError] | None = None,
) -> KeyValueStoreService:
"""A store over a backend that holds only text: bytes are written as
Base64, and ``get_bytes`` reads a value that is valid Base64 as the
bytes it encodes, any other as its UTF-8."""
def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
return get(key).map(lambda value: None if value is None else _text_to_bytes(value))
def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
if isinstance(value, str):
return set(key, value)
return suspend(lambda: set(key, base64_encode(value)))
return make(
get=get,
get_bytes=get_bytes,
set=set_value,
remove=remove,
clear=clear,
size=size,
keys=keys,
clear_prefix=clear_prefix,
has=has,
is_empty=is_empty,
)