CSV channels, with optional schemas (Effect's encoding/Csv).
RFC 4180 with a configurable delimiter. Cells are strings, kept as they are
(whitespace and empty cells included); quoted fields may hold delimiters,
line breaks and doubled quotes. Decoders accept LF, CRLF and CR and skip a
leading byte-order mark. Use the channels with Stream.pipe_through_channel::
from pyfect.encoding import csv
records = text_stream.pipe_through_channel(csv.decode_string(headers=True))decode_schema_string / encode_schema_string (and their byte forms)
decode records to typed values, or encode values to records, with a schema.
Values#
| Attribute | Type | Description |
|---|---|---|
DEFAULT_MAX_RECORD_SIZE | Literal[1048576] | 1 MiB, in UTF-16 code units. |
Types#
| Type | Definition | Description |
|---|---|---|
HeaderRecord | type HeaderRecord = dict[str, str] | A record keyed by the header row's names. |
Newline | type Newline = Literal['\n', '\r\n'] | |
CsvErrorKind | type CsvErrorKind = Literal['Syntax', 'RecordSize', 'Header'] | |
Row | type Row = tuple[str, ...] | A record as its cells. |
Classes#
Functions#
decode#
decode[E = Never, D = None](*, headers: Literal[True], delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]
decode[E = Never, D = None](*, headers: Literal[False] = False, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[bytes, ...], E, D]
decode[E = Never, D = None](*, headers: bool, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]decode_string of UTF-8 bytes.
| Parameter | Type | Description |
|---|---|---|
headers | = False | |
delimiter | = ',' | |
max_record_size | = DEFAULT_MAX_RECORD_SIZE | |
ignore_empty_lines | = False |
@overload
def decode[E = Never, D = None](
*,
headers: Literal[True],
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
@overload
def decode[E = Never, D = None](
*,
headers: Literal[False] = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
@overload
def decode[E = Never, D = None](
*,
headers: bool,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
def decode(
*,
headers: bool = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[Any, Any, Any, Any, Any, Any]:
"""``decode_string`` of UTF-8 bytes."""
text: Channel[Any, Any, Any, Any, Any, Any] = decode_text("utf-8", ignore_bom=True)
return text.pipe_to(
decode_string(
headers=headers,
delimiter=delimiter,
max_record_size=max_record_size,
ignore_empty_lines=ignore_empty_lines,
)
)decode_string#
decode_string[E = Never, D = None](*, headers: Literal[True], delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]
decode_string[E = Never, D = None](*, headers: Literal[False] = False, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[str, ...], E, D]
decode_string[E = Never, D = None](*, headers: bool, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]Parse chunks of text into records: rows of cells, or with headers
dicts keyed by the first record's names (which must be unique, and every
record as wide). max_record_size bounds a record's length in UTF-16
code units, quotes and delimiters included (math.inf for none);
ignore_empty_lines skips records with no characters, which are
otherwise one empty cell. A leading byte-order mark is skipped, and LF,
CRLF and CR end a record. Invalid options raise ValueError here; the
parser's state is made per run.
| Parameter | Type | Description |
|---|---|---|
headers | = False | |
delimiter | = ',' | |
max_record_size | = DEFAULT_MAX_RECORD_SIZE | |
ignore_empty_lines | = False |
@overload
def decode_string[E = Never, D = None](
*,
headers: Literal[True],
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
@overload
def decode_string[E = Never, D = None](
*,
headers: Literal[False] = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
@overload
def decode_string[E = Never, D = None](
*,
headers: bool,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
def decode_string(
*,
headers: bool = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
) -> Channel[Any, Any, Any, Any, Any, Any]:
"""Parse chunks of text into records: rows of cells, or with ``headers``
dicts keyed by the first record's names (which must be unique, and every
record as wide). ``max_record_size`` bounds a record's length in UTF-16
code units, quotes and delimiters included (``math.inf`` for none);
``ignore_empty_lines`` skips records with no characters, which are
otherwise one empty cell. A leading byte-order mark is skipped, and LF,
CRLF and CR end a record. Invalid options raise ``ValueError`` here; the
parser's state is made per run."""
return decode_records(delimiter, max_record_size, ignore_empty_lines, headers).map(_values)encode#
encode[E = Never, D = None](*, columns: Sequence[str], headers: bool = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[bytes, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]
encode[E = Never, D = None](*, columns: None = None, headers: Literal[False] = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[bytes, ...], E, D, tuple[Sequence[str | None], ...], E, D]encode_string as UTF-8 bytes, one bytes per input chunk.
| Parameter | Type | Description |
|---|---|---|
columns | = None | |
headers | = False | |
delimiter | = ',' | |
newline | = '\r\n' |
@overload
def encode[E = Never, D = None](
*,
columns: Sequence[str],
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[tuple[bytes, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]: ...
@overload
def encode[E = Never, D = None](
*,
columns: None = None,
headers: Literal[False] = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[tuple[bytes, ...], E, D, tuple[Sequence[str | None], ...], E, D]: ...
def encode(
*,
columns: Sequence[str] | None = None,
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[Any, Any, Any, Any, Any, Any]:
"""``encode_string`` as UTF-8 bytes, one ``bytes`` per input chunk."""
return encode_records(delimiter, newline, columns, headers).map(_joined_bytes)encode_string#
encode_string[E = Never, D = None](*, columns: Sequence[str], headers: bool = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[str, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]
encode_string[E = Never, D = None](*, columns: None = None, headers: Literal[False] = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[str, ...], E, D, tuple[Sequence[str | None], ...], E, D]Write records as text, one string per record, each ending with
newline. Without columns records are rows of cells; with them,
mappings written by those keys (a missing or None value is an empty
cell), and headers writes the columns first, even for an empty
stream. A cell that is not a string is a defect (TypeError); invalid
options raise ValueError here.
| Parameter | Type | Description |
|---|---|---|
columns | = None | |
headers | = False | |
delimiter | = ',' | |
newline | = '\r\n' |
@overload
def encode_string[E = Never, D = None](
*,
columns: Sequence[str],
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[tuple[str, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]: ...
@overload
def encode_string[E = Never, D = None](
*,
columns: None = None,
headers: Literal[False] = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[tuple[str, ...], E, D, tuple[Sequence[str | None], ...], E, D]: ...
def encode_string(
*,
columns: Sequence[str] | None = None,
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
) -> Channel[Any, Any, Any, Any, Any, Any]:
"""Write records as text, one string per record, each ending with
``newline``. Without ``columns`` records are rows of cells; with them,
mappings written by those keys (a missing or ``None`` value is an empty
cell), and ``headers`` writes the columns first, even for an empty
stream. A cell that is not a string is a defect (``TypeError``); invalid
options raise ``ValueError`` here."""
return encode_records(delimiter, newline, columns, headers)is_csv_error#
is_csv_error(value) -> TypeIs[CsvError]| Parameter | Type | Description |
|---|---|---|
value | object |
Returns TypeIs[CsvError]
def is_csv_error(value: object) -> TypeIs[CsvError]:
return isinstance(value, CsvError)decode_schema#
decode_schema[T, RD, E = Never, D = None](schema, /, *, headers=False, delimiter=',', max_record_size=DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines=False, disable_codecs=False, **options) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]decode_schema_string of UTF-8 bytes.
| Parameter | Type | Description |
|---|---|---|
schema | Codec[T, Any, RD, object] | |
headers | bool = False | |
delimiter | str = ',' | |
max_record_size | float = DEFAULT_MAX_RECORD_SIZE | |
ignore_empty_lines | bool = False | |
disable_codecs | bool = False | |
options | Unpack[ParseOptions] = {} |
Returns Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]
def decode_schema[T, RD, E = Never, D = None](
schema: Codec[T, Any, RD, object],
/,
*,
headers: bool = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
disable_codecs: bool = False,
**options: Unpack[ParseOptions],
) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]:
"""``decode_schema_string`` of UTF-8 bytes."""
text: _AnyChannel = decode_text("utf-8", ignore_bom=True)
records: _AnyChannel = decode_schema_string(
schema,
headers=headers,
delimiter=delimiter,
max_record_size=max_record_size,
ignore_empty_lines=ignore_empty_lines,
disable_codecs=disable_codecs,
**options,
)
return text.pipe_to(records)decode_schema_string#
decode_schema_string[T, RD, E = Never, D = None](schema, /, *, headers=False, delimiter=',', max_record_size=DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines=False, disable_codecs=False, **options) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]Parse chunks of text as decode_string does and decode each record
with schema: a dict keyed by header name with headers, else a row.
By default the schema is wrapped in to_codec_string_tree, so columns
may be numbers, booleans or dates, and with headers an empty cell of a
key that is optional on the encoded side is a missing key;
disable_codecs decodes with schema as given. A column that does
not encode to a string (a list, a nested struct) raises ValueError
here. Each chunk is decoded at once; a failure is a SchemaError whose
paths start with the record number.
| Parameter | Type | Description |
|---|---|---|
schema | Codec[T, Any, RD, object] | |
headers | bool = False | |
delimiter | str = ',' | |
max_record_size | float = DEFAULT_MAX_RECORD_SIZE | |
ignore_empty_lines | bool = False | |
disable_codecs | bool = False | |
options | Unpack[ParseOptions] = {} |
Returns Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]
def decode_schema_string[T, RD, E = Never, D = None](
schema: Codec[T, Any, RD, object],
/,
*,
headers: bool = False,
delimiter: str = ",",
max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
ignore_empty_lines: bool = False,
disable_codecs: bool = False,
**options: Unpack[ParseOptions],
) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]:
"""Parse chunks of text as ``decode_string`` does and decode each record
with ``schema``: a dict keyed by header name with ``headers``, else a row.
By default the schema is wrapped in ``to_codec_string_tree``, so columns
may be numbers, booleans or dates, and with ``headers`` an empty cell of a
key that is optional on the encoded side is a missing key;
``disable_codecs`` decodes with ``schema`` as given. A column that does
not encode to a string (a list, a nested struct) raises ``ValueError``
here. Each chunk is decoded at once; a failure is a ``SchemaError`` whose
paths start with the record number."""
codec = _to_codec(schema, disable_codecs)
encoded = _resolve_suspend(ast.to_encoded(codec.__codec__().ast))
_check_cells(encoded, None)
# CSV cannot tell an empty cell from a missing one, so empty cells of
# optional keys are decoded as missing keys.
optional = (
[
property.name
for property in encoded.properties
if isinstance(property.name, str)
and (property.optional or ast.is_optional(property.type))
]
if not disable_codecs and headers and isinstance(encoded, ast.Objects)
else []
)
def value_of(record: Decoded) -> object:
value = record.value
if not optional or not isinstance(value, dict):
return value
# Header records are made per row, so they can be changed in place.
for key in optional:
if value.get(key) == "":
del value[key]
return value
decode = decode_with(NonEmptyArray(codec), **options)
def decoded(records: tuple[Decoded, ...]) -> Any:
return decode(tuple(map(value_of, records))).map_error(
lambda error: _rebase(error, lambda index: records[index].record)
)
channel: _AnyChannel = decode_records(delimiter, max_record_size, ignore_empty_lines, headers)
return channel.map_effect(decoded)encode_schema#
encode_schema[T, RE, E = Never, D = None](schema, /, *, columns=None, headers=False, delimiter=',', newline='\r\n', disable_codecs=False) -> Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]encode_schema_string as UTF-8 bytes, one bytes per input
chunk. When the bytes stream an HTTP response body, an encoding failure
ends the body; it cannot change a status already sent.
| Parameter | Type | Description |
|---|---|---|
schema | Codec[T, Any, object, RE] | |
columns | Sequence[str] | None = None | |
headers | bool = False | |
delimiter | str = ',' | |
newline | Newline = '\r\n' | |
disable_codecs | bool = False |
Returns Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]
def encode_schema[T, RE, E = Never, D = None](
schema: Codec[T, Any, object, RE],
/,
*,
columns: Sequence[str] | None = None,
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
disable_codecs: bool = False,
) -> Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]:
"""``encode_schema_string`` as UTF-8 bytes, one ``bytes`` per input
chunk. When the bytes stream an HTTP response body, an encoding failure
ends the body; it cannot change a status already sent."""
from pyfect.encoding.csv._channels import _joined_bytes
records = _encode_schema_records(
schema,
delimiter=delimiter,
newline=newline,
columns=columns,
headers=headers,
disable_codecs=disable_codecs,
)
return records.map(_joined_bytes)encode_schema_string#
encode_schema_string[T, RE, E = Never, D = None](schema, /, *, columns=None, headers=False, delimiter=',', newline='\r\n', disable_codecs=False) -> Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]Encode each value with schema and write it as CSV text, one string
per record. By default the schema is wrapped in to_codec_string_tree,
so every column encodes to a string (disable_codecs keeps it as
given). columns defaults to a struct's encoded keys (renamed ones
too) and is needed for other mappings; tuples and arrays write cells by
position and use columns only as header names. A missing or None
value is an empty cell. A column that does not encode to a string raises
ValueError here. A failure is a SchemaError whose paths start
with the output record number, the header row counted.
| Parameter | Type | Description |
|---|---|---|
schema | Codec[T, Any, object, RE] | |
columns | Sequence[str] | None = None | |
headers | bool = False | |
delimiter | str = ',' | |
newline | Newline = '\r\n' | |
disable_codecs | bool = False |
Returns Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]
def encode_schema_string[T, RE, E = Never, D = None](
schema: Codec[T, Any, object, RE],
/,
*,
columns: Sequence[str] | None = None,
headers: bool = False,
delimiter: str = ",",
newline: Newline = "\r\n",
disable_codecs: bool = False,
) -> Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]:
"""Encode each value with ``schema`` and write it as CSV text, one string
per record. By default the schema is wrapped in ``to_codec_string_tree``,
so every column encodes to a string (``disable_codecs`` keeps it as
given). ``columns`` defaults to a struct's encoded keys (renamed ones
too) and is needed for other mappings; tuples and arrays write cells by
position and use ``columns`` only as header names. A missing or ``None``
value is an empty cell. A column that does not encode to a string raises
``ValueError`` here. A failure is a ``SchemaError`` whose paths start
with the output record number, the header row counted."""
return _encode_schema_records(
schema,
delimiter=delimiter,
newline=newline,
columns=columns,
headers=headers,
disable_codecs=disable_codecs,
)