GitHub

key_value_store

API reference for key_value_store.

A string-keyed store of text and bytes (Effect's effect/persistence/KeyValueStore)::

kv = yield from KeyValueStore()
yield from kv.set("greeting", "hello")
text = yield from kv.get("greeting")       # "hello"; None when missing
  • layer_memory: a dict; layer_file_system(directory): a file per key; layer_sql(table=...): a SQL table; layer_storage(evaluate): any MutableMapping[str, str].
  • make / make_string_only build a store from a backend's operations; store.prefix(p) puts p before every key.
  • store.to_schema_store(schema) holds one schema's values as JSON text.

Bytes read as text come back as Base64, text read as bytes as UTF-8.

Values

AttributeTypeDescription
layer_memoryLayer[KeyValueStore]

A fresh in-memory store each time the layer is built.

Types

TypeDefinitionDescription
Valuetype Value = str | bytes

What a store holds: text, or bytes (Effect's Uint8Array).

Classes

Functions

layer_file_system

layer_file_system(directory) -> Layer[KeyValueStore, PlatformError, FileSystem | Path]

A store keeping each value in a file of directory (made if missing) named by its percent-encoded key, upper-case letters and a trailing . escaped too (files 0.6.5 named are renamed on open). The keys "", "." and ".." are rejected without touching the file system. keys decodes the file names (failing on one that is not percent-encoded UTF-8), and clear_prefix skips such files.

ParameterTypeDescription
directorystr

Returns Layer[KeyValueStore, PlatformError, FileSystem | Path]

def layer_file_system(
    directory: str,
) -> Layer[KeyValueStore, PlatformError, FileSystem | Path]:
    """A store keeping each value in a file of ``directory`` (made if
    missing) named by its percent-encoded key, upper-case letters and a
    trailing ``.`` escaped too (files 0.6.5 named are renamed on open). The keys ``""``, ``"."``
    and ``".."`` are rejected without touching the file system. ``keys``
    decodes the file names (failing on one that is not percent-encoded
    UTF-8), and ``clear_prefix`` skips such files."""

    def build() -> Generator[Effect[Any, PlatformError, FileSystem | Path], Any, Any]:
        fs = yield from FileSystem()
        path = yield from Path()
        if not (yield from fs.exists(directory)):
            yield from fs.make_directory(directory, recursive=True)
        yield from _rename_legacy_files(fs, path, directory)

        def with_key_path[A](
            method: str, key: str, f: Callable[[str], Effect[A, KeyValueStoreError]]
        ) -> Effect[A, KeyValueStoreError]:
            name = _file_name(key)
            if name is None:
                return fail(KeyValueStoreError(f"Invalid key {key}", method, key))
            return f(path.join(directory, name))

        def failed(
            method: str, key: str, message: str
        ) -> Callable[[PlatformError], KeyValueStoreError]:
            return lambda cause: KeyValueStoreError(message, method, key, cause)

        def missing_or_fail(
            method: str, key: str
        ) -> Callable[[PlatformError], Effect[None, KeyValueStoreError]]:
            def handle(cause: PlatformError) -> Effect[None, KeyValueStoreError]:
                if isinstance(cause.reason, NotFound):
                    return succeed(None)
                return fail(
                    KeyValueStoreError(f"Unable to get item with key {key}", method, key, cause)
                )

            return handle

        def get(key: str) -> Effect[str | None, KeyValueStoreError]:
            return with_key_path(
                "get",
                key,
                lambda p: fs.read_file_string(p).catch_all(missing_or_fail("get", key)),
            )

        def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
            return with_key_path(
                "get_bytes",
                key,
                lambda p: fs.read_file(p).catch_all(missing_or_fail("get_bytes", key)),
            )

        def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
            def write(p: str) -> Effect[None, KeyValueStoreError]:
                written = (
                    fs.write_file_string(p, value)
                    if isinstance(value, str)
                    else fs.write_file(p, value)
                )
                return written.map_error(failed("set", key, f"Unable to set item with key {key}"))

            return with_key_path("set", key, write)

        def remove(key: str) -> Effect[None, KeyValueStoreError]:
            return with_key_path(
                "remove",
                key,
                lambda p: fs.remove(p, force=True).map_error(
                    failed("remove", key, f"Unable to remove item with key {key}")
                ),
            )

        def has(key: str) -> Effect[bool, KeyValueStoreError]:
            return with_key_path(
                "has",
                key,
                lambda p: fs.exists(p).map_error(
                    failed("has", key, f"Unable to check existence of item with key {key}")
                ),
            )

        clear = (
            fs.remove(directory, recursive=True)
            .and_then(fs.make_directory(directory, recursive=True))
            .map_error(
                lambda cause: KeyValueStoreError("Unable to clear storage", "clear", None, cause)
            )
        )
        size = (
            fs.read_directory(directory)
            .map(len)
            .map_error(lambda cause: KeyValueStoreError("Unable to get size", "size", None, cause))
        )

        def remove_file(name: str) -> Effect[None, PlatformError]:
            return fs.remove(path.join(directory, name)).catch_all(
                lambda cause: succeed(None) if isinstance(cause.reason, NotFound) else fail(cause)
            )

        def clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
            def matching(files: list[str]) -> list[str]:
                return [
                    name
                    for name in files
                    if (key := _decode_file_key(name)) is not None and key.startswith(prefix)
                ]

            return (
                fs.read_directory(directory)
                .flat_map(lambda files: for_each(matching(files), remove_file))
                .as_void()
                .map_error(
                    lambda cause: KeyValueStoreError(
                        f"Unable to clear items with prefix {prefix}", "clear_prefix", prefix, cause
                    )
                )
            )

        def decode_all(files: list[str]) -> Effect[list[str], Exception]:
            return try_(lambda: [_strict_decode(name) for name in files], lambda cause: cause)

        keys: Effect[list[str], KeyValueStoreError] = (
            fs.read_directory(directory)
            .flat_map(decode_all)
            .map_error(lambda cause: KeyValueStoreError("Unable to get keys", "keys", None, cause))
        )
        return make(
            get=get,
            get_bytes=get_bytes,
            set=set_value,
            remove=remove,
            has=has,
            clear=clear,
            clear_prefix=clear_prefix,
            size=size,
            keys=keys,
        )

    return layer_effect(KeyValueStore, gen(build))

layer_storage

layer_storage(evaluate) -> Layer[KeyValueStore]

A store over the text mapping evaluate() returns when the layer is built (Effect's layerStorage over Web Storage; a shelve database, say). What the mapping raises becomes a KeyValueStoreError.

ParameterTypeDescription
evaluateCallable[[], MutableMapping[str, str]]

Returns Layer[KeyValueStore]

def layer_storage(evaluate: Callable[[], MutableMapping[str, str]]) -> Layer[KeyValueStore]:
    """A store over the text mapping ``evaluate()`` returns when the layer
    is built (Effect's ``layerStorage`` over Web ``Storage``; a ``shelve``
    database, say). What the mapping raises becomes a
    ``KeyValueStoreError``."""

    def build() -> KeyValueStoreService:
        storage = evaluate()

        def attempt[A](
            method: str, key: str | None, message: str, thunk: Callable[[], A]
        ) -> Effect[A, KeyValueStoreError]:
            return try_(thunk, lambda cause: KeyValueStoreError(message, method, key, cause))

        return make_string_only(
            get=lambda key: attempt(
                "get", key, f"Unable to get item with key {key}", lambda: storage.get(key)
            ),
            set=lambda key, value: attempt(
                "set",
                key,
                f"Unable to set item with key {key}",
                lambda: storage.__setitem__(key, value),
            ),
            remove=lambda key: attempt(
                "remove",
                key,
                f"Unable to remove item with key {key}",
                lambda: storage.pop(key, None),
            ).as_void(),
            clear=attempt("clear", None, "Unable to clear storage", storage.clear),
            clear_prefix=lambda prefix: attempt(
                "clear_prefix",
                prefix,
                f"Unable to clear items with prefix {prefix}",
                lambda: _remove_prefixed(storage, prefix),
            ),
            size=attempt("size", None, "Unable to get size", lambda: len(storage)),
            keys=attempt("keys", None, "Unable to get keys", lambda: list(storage)),
        )

    return layer_sync(KeyValueStore, build)

make_memory

make_memory() -> KeyValueStoreService

A store in a dict of its own.

Returns KeyValueStoreService

def make_memory() -> KeyValueStoreService:
    """A store in a dict of its own."""
    store: dict[str, Value] = {}

    def get(key: str) -> Effect[str | None, KeyValueStoreError]:
        def read() -> str | None:
            value = store.get(key)
            return value if value is None or isinstance(value, str) else base64_encode(value)

        return sync(read)

    def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
        def read() -> bytes | None:
            value = store.get(key)
            return text_encode(value) if isinstance(value, str) else value

        return sync(read)

    def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
        return sync(lambda: store.__setitem__(key, value))

    def clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
        def clear() -> None:
            for key in [key for key in store if key.startswith(prefix)]:
                del store[key]

        return sync(clear)

    return make(
        get=get,
        get_bytes=get_bytes,
        set=set_value,
        remove=lambda key: sync(lambda: store.pop(key, None)).as_void(),
        clear=sync(store.clear),
        clear_prefix=clear_prefix,
        size=sync(lambda: len(store)),
        keys=sync(lambda: list(store)),
    )

layer_sql

layer_sql(*, table='effect_key_value_store') -> Layer[KeyValueStore, Never, SqlClient]

A store in table (made if missing) of the SqlClient: one row per key, the value as bytes and whether it was text.

ParameterTypeDescription
tablestr = 'effect_key_value_store'

Returns Layer[KeyValueStore, Never, SqlClient]

def layer_sql(*, table: str = "effect_key_value_store") -> Layer[KeyValueStore, Never, SqlClient]:
    """A store in ``table`` (made if missing) of the ``SqlClient``: one row
    per key, the value as bytes and whether it was text."""
    return layer_effect(KeyValueStore, gen(lambda: _build(table)))

make

make(*, get, get_bytes, set, remove, clear, size, keys, clear_prefix=None, has=None, is_empty=None, modify=None, modify_bytes=None) -> KeyValueStoreService

A store from its backend's operations; has, is_empty, clear_prefix, modify and modify_bytes are derived unless given. The derived clear_prefix is clear for "", else it removes each of keys starting with the prefix.

ParameterTypeDescription
getCallable[[str], Effect[str | None, KeyValueStoreError]]
get_bytesCallable[[str], Effect[bytes | None, KeyValueStoreError]]
setCallable[[str, Value], Effect[None, KeyValueStoreError]]
removeCallable[[str], Effect[None, KeyValueStoreError]]
clearEffect[None, KeyValueStoreError]
sizeEffect[int, KeyValueStoreError]
keysEffect[list[str], KeyValueStoreError]
clear_prefixCallable[[str], Effect[None, KeyValueStoreError]] | None = None
hasCallable[[str], Effect[bool, KeyValueStoreError]] | None = None
is_emptyEffect[bool, KeyValueStoreError] | None = None
modifyCallable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]] | None = None
modify_bytesCallable[[str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]] | None = None

Returns KeyValueStoreService

def make(
    *,
    get: Callable[[str], Effect[str | None, KeyValueStoreError]],
    get_bytes: Callable[[str], Effect[bytes | None, KeyValueStoreError]],
    set: Callable[[str, Value], Effect[None, KeyValueStoreError]],
    remove: Callable[[str], Effect[None, KeyValueStoreError]],
    clear: Effect[None, KeyValueStoreError],
    size: Effect[int, KeyValueStoreError],
    keys: Effect[list[str], KeyValueStoreError],
    clear_prefix: Callable[[str], Effect[None, KeyValueStoreError]] | None = None,
    has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
    is_empty: Effect[bool, KeyValueStoreError] | None = None,
    modify: Callable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]]
    | None = None,
    modify_bytes: Callable[
        [str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]
    ]
    | None = None,
) -> KeyValueStoreService:
    """A store from its backend's operations; ``has``, ``is_empty``,
    ``clear_prefix``, ``modify`` and ``modify_bytes`` are derived unless
    given. The derived ``clear_prefix`` is ``clear`` for ``""``, else it
    removes each of ``keys`` starting with the prefix."""
    require_effect(clear, "clear")
    require_effect(size, "size")
    require_effect(keys, "keys")
    if is_empty is not None:
        require_effect(is_empty, "is_empty")

    def derived_clear_prefix(prefix: str) -> Effect[None, KeyValueStoreError]:
        if not prefix:
            return clear
        return keys.flat_map(
            lambda found: for_each([key for key in found if key.startswith(prefix)], remove)
        ).as_void()

    def derived_has(key: str) -> Effect[bool, KeyValueStoreError]:
        return get(key).map(lambda value: value is not None)

    def derived_modify(key: str, f: Callable[[str], str]) -> Effect[str | None, KeyValueStoreError]:
        def update(value: str | None) -> Effect[str | None, KeyValueStoreError]:
            if value is None:
                return succeed(None)
            updated = f(value)
            return set(key, updated).as_(updated)

        return get(key).flat_map(update)

    def derived_modify_bytes(
        key: str, f: Callable[[bytes], bytes]
    ) -> Effect[bytes | None, KeyValueStoreError]:
        def update(value: bytes | None) -> Effect[bytes | None, KeyValueStoreError]:
            if value is None:
                return succeed(None)
            updated = f(value)
            return set(key, updated).as_(updated)

        return get_bytes(key).flat_map(update)

    return KeyValueStoreService(
        get=get,
        get_bytes=get_bytes,
        set=set,
        remove=remove,
        clear=clear,
        clear_prefix=clear_prefix or derived_clear_prefix,
        size=size,
        keys=keys,
        has=has or derived_has,
        is_empty=is_empty or size.map(lambda n: n == 0),
        modify=modify or derived_modify,
        modify_bytes=modify_bytes or derived_modify_bytes,
    )

make_string_only

make_string_only(*, get, set, remove, clear, size, keys, clear_prefix=None, has=None, is_empty=None) -> KeyValueStoreService

A store over a backend that holds only text: bytes are written as Base64, and get_bytes reads a value that is valid Base64 as the bytes it encodes, any other as its UTF-8.

ParameterTypeDescription
getCallable[[str], Effect[str | None, KeyValueStoreError]]
setCallable[[str, str], Effect[None, KeyValueStoreError]]
removeCallable[[str], Effect[None, KeyValueStoreError]]
clearEffect[None, KeyValueStoreError]
sizeEffect[int, KeyValueStoreError]
keysEffect[list[str], KeyValueStoreError]
clear_prefixCallable[[str], Effect[None, KeyValueStoreError]] | None = None
hasCallable[[str], Effect[bool, KeyValueStoreError]] | None = None
is_emptyEffect[bool, KeyValueStoreError] | None = None

Returns KeyValueStoreService

def make_string_only(
    *,
    get: Callable[[str], Effect[str | None, KeyValueStoreError]],
    set: Callable[[str, str], Effect[None, KeyValueStoreError]],
    remove: Callable[[str], Effect[None, KeyValueStoreError]],
    clear: Effect[None, KeyValueStoreError],
    size: Effect[int, KeyValueStoreError],
    keys: Effect[list[str], KeyValueStoreError],
    clear_prefix: Callable[[str], Effect[None, KeyValueStoreError]] | None = None,
    has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
    is_empty: Effect[bool, KeyValueStoreError] | None = None,
) -> KeyValueStoreService:
    """A store over a backend that holds only text: bytes are written as
    Base64, and ``get_bytes`` reads a value that is valid Base64 as the
    bytes it encodes, any other as its UTF-8."""

    def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
        return get(key).map(lambda value: None if value is None else _text_to_bytes(value))

    def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
        if isinstance(value, str):
            return set(key, value)
        return suspend(lambda: set(key, base64_encode(value)))

    return make(
        get=get,
        get_bytes=get_bytes,
        set=set_value,
        remove=remove,
        clear=clear,
        size=size,
        keys=keys,
        clear_prefix=clear_prefix,
        has=has,
        is_empty=is_empty,
    )