GitHub

API reference for csv.

CSV channels, with optional schemas (Effect's encoding/Csv).

RFC 4180 with a configurable delimiter. Cells are strings, kept as they are (whitespace and empty cells included); quoted fields may hold delimiters, line breaks and doubled quotes. Decoders accept LF, CRLF and CR and skip a leading byte-order mark. Use the channels with Stream.pipe_through_channel::

from pyfect.encoding import csv

records = text_stream.pipe_through_channel(csv.decode_string(headers=True))

decode_schema_string / encode_schema_string (and their byte forms) decode records to typed values, or encode values to records, with a schema.

Values

AttributeTypeDescription
DEFAULT_MAX_RECORD_SIZELiteral[1048576]

1 MiB, in UTF-16 code units.

Types

TypeDefinitionDescription
HeaderRecordtype HeaderRecord = dict[str, str]

A record keyed by the header row's names.

Newlinetype Newline = Literal['\n', '\r\n']
CsvErrorKindtype CsvErrorKind = Literal['Syntax', 'RecordSize', 'Header']
Rowtype Row = tuple[str, ...]

A record as its cells.

Classes

Functions

decode

decode[E = Never, D = None](*, headers: Literal[True], delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]
decode[E = Never, D = None](*, headers: Literal[False] = False, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[bytes, ...], E, D]
decode[E = Never, D = None](*, headers: bool, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]

decode_string of UTF-8 bytes.

ParameterTypeDescription
headers= False
delimiter= ','
max_record_size= DEFAULT_MAX_RECORD_SIZE
ignore_empty_lines= False
@overload
def decode[E = Never, D = None](
    *,
    headers: Literal[True],
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
@overload
def decode[E = Never, D = None](
    *,
    headers: Literal[False] = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
@overload
def decode[E = Never, D = None](
    *,
    headers: bool,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[bytes, ...], E, D]: ...
def decode(
    *,
    headers: bool = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[Any, Any, Any, Any, Any, Any]:
    """``decode_string`` of UTF-8 bytes."""
    text: Channel[Any, Any, Any, Any, Any, Any] = decode_text("utf-8", ignore_bom=True)
    return text.pipe_to(
        decode_string(
            headers=headers,
            delimiter=delimiter,
            max_record_size=max_record_size,
            ignore_empty_lines=ignore_empty_lines,
        )
    )

decode_string

decode_string[E = Never, D = None](*, headers: Literal[True], delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]
decode_string[E = Never, D = None](*, headers: Literal[False] = False, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[str, ...], E, D]
decode_string[E = Never, D = None](*, headers: bool, delimiter: str = ',', max_record_size: float = DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines: bool = False) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]

Parse chunks of text into records: rows of cells, or with headers dicts keyed by the first record's names (which must be unique, and every record as wide). max_record_size bounds a record's length in UTF-16 code units, quotes and delimiters included (math.inf for none); ignore_empty_lines skips records with no characters, which are otherwise one empty cell. A leading byte-order mark is skipped, and LF, CRLF and CR end a record. Invalid options raise ValueError here; the parser's state is made per run.

ParameterTypeDescription
headers= False
delimiter= ','
max_record_size= DEFAULT_MAX_RECORD_SIZE
ignore_empty_lines= False
@overload
def decode_string[E = Never, D = None](
    *,
    headers: Literal[True],
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
@overload
def decode_string[E = Never, D = None](
    *,
    headers: Literal[False] = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[Row, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
@overload
def decode_string[E = Never, D = None](
    *,
    headers: bool,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[tuple[Row | HeaderRecord, ...], E | CsvError, D, tuple[str, ...], E, D]: ...
def decode_string(
    *,
    headers: bool = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
) -> Channel[Any, Any, Any, Any, Any, Any]:
    """Parse chunks of text into records: rows of cells, or with ``headers``
    dicts keyed by the first record's names (which must be unique, and every
    record as wide). ``max_record_size`` bounds a record's length in UTF-16
    code units, quotes and delimiters included (``math.inf`` for none);
    ``ignore_empty_lines`` skips records with no characters, which are
    otherwise one empty cell. A leading byte-order mark is skipped, and LF,
    CRLF and CR end a record. Invalid options raise ``ValueError`` here; the
    parser's state is made per run."""
    return decode_records(delimiter, max_record_size, ignore_empty_lines, headers).map(_values)

encode

encode[E = Never, D = None](*, columns: Sequence[str], headers: bool = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[bytes, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]
encode[E = Never, D = None](*, columns: None = None, headers: Literal[False] = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[bytes, ...], E, D, tuple[Sequence[str | None], ...], E, D]

encode_string as UTF-8 bytes, one bytes per input chunk.

ParameterTypeDescription
columns= None
headers= False
delimiter= ','
newline= '\r\n'
@overload
def encode[E = Never, D = None](
    *,
    columns: Sequence[str],
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[tuple[bytes, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]: ...
@overload
def encode[E = Never, D = None](
    *,
    columns: None = None,
    headers: Literal[False] = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[tuple[bytes, ...], E, D, tuple[Sequence[str | None], ...], E, D]: ...
def encode(
    *,
    columns: Sequence[str] | None = None,
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[Any, Any, Any, Any, Any, Any]:
    """``encode_string`` as UTF-8 bytes, one ``bytes`` per input chunk."""
    return encode_records(delimiter, newline, columns, headers).map(_joined_bytes)

encode_string

encode_string[E = Never, D = None](*, columns: Sequence[str], headers: bool = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[str, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]
encode_string[E = Never, D = None](*, columns: None = None, headers: Literal[False] = False, delimiter: str = ',', newline: Newline = '\r\n') -> Channel[tuple[str, ...], E, D, tuple[Sequence[str | None], ...], E, D]

Write records as text, one string per record, each ending with newline. Without columns records are rows of cells; with them, mappings written by those keys (a missing or None value is an empty cell), and headers writes the columns first, even for an empty stream. A cell that is not a string is a defect (TypeError); invalid options raise ValueError here.

ParameterTypeDescription
columns= None
headers= False
delimiter= ','
newline= '\r\n'
@overload
def encode_string[E = Never, D = None](
    *,
    columns: Sequence[str],
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[tuple[str, ...], E, D, tuple[Mapping[str, str | None], ...], E, D]: ...
@overload
def encode_string[E = Never, D = None](
    *,
    columns: None = None,
    headers: Literal[False] = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[tuple[str, ...], E, D, tuple[Sequence[str | None], ...], E, D]: ...
def encode_string(
    *,
    columns: Sequence[str] | None = None,
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
) -> Channel[Any, Any, Any, Any, Any, Any]:
    """Write records as text, one string per record, each ending with
    ``newline``. Without ``columns`` records are rows of cells; with them,
    mappings written by those keys (a missing or ``None`` value is an empty
    cell), and ``headers`` writes the columns first, even for an empty
    stream. A cell that is not a string is a defect (``TypeError``); invalid
    options raise ``ValueError`` here."""
    return encode_records(delimiter, newline, columns, headers)

is_csv_error

is_csv_error(value) -> TypeIs[CsvError]
ParameterTypeDescription
valueobject

Returns TypeIs[CsvError]

def is_csv_error(value: object) -> TypeIs[CsvError]:
    return isinstance(value, CsvError)

decode_schema

decode_schema[T, RD, E = Never, D = None](schema, /, *, headers=False, delimiter=',', max_record_size=DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines=False, disable_codecs=False, **options) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]

decode_schema_string of UTF-8 bytes.

ParameterTypeDescription
schemaCodec[T, Any, RD, object]
headersbool = False
delimiterstr = ','
max_record_sizefloat = DEFAULT_MAX_RECORD_SIZE
ignore_empty_linesbool = False
disable_codecsbool = False
optionsUnpack[ParseOptions] = {}

Returns Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]

def decode_schema[T, RD, E = Never, D = None](
    schema: Codec[T, Any, RD, object],
    /,
    *,
    headers: bool = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
    disable_codecs: bool = False,
    **options: Unpack[ParseOptions],
) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[bytes, ...], E, D, RD]:
    """``decode_schema_string`` of UTF-8 bytes."""
    text: _AnyChannel = decode_text("utf-8", ignore_bom=True)
    records: _AnyChannel = decode_schema_string(
        schema,
        headers=headers,
        delimiter=delimiter,
        max_record_size=max_record_size,
        ignore_empty_lines=ignore_empty_lines,
        disable_codecs=disable_codecs,
        **options,
    )
    return text.pipe_to(records)

decode_schema_string

decode_schema_string[T, RD, E = Never, D = None](schema, /, *, headers=False, delimiter=',', max_record_size=DEFAULT_MAX_RECORD_SIZE, ignore_empty_lines=False, disable_codecs=False, **options) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]

Parse chunks of text as decode_string does and decode each record with schema: a dict keyed by header name with headers, else a row. By default the schema is wrapped in to_codec_string_tree, so columns may be numbers, booleans or dates, and with headers an empty cell of a key that is optional on the encoded side is a missing key; disable_codecs decodes with schema as given. A column that does not encode to a string (a list, a nested struct) raises ValueError here. Each chunk is decoded at once; a failure is a SchemaError whose paths start with the record number.

ParameterTypeDescription
schemaCodec[T, Any, RD, object]
headersbool = False
delimiterstr = ','
max_record_sizefloat = DEFAULT_MAX_RECORD_SIZE
ignore_empty_linesbool = False
disable_codecsbool = False
optionsUnpack[ParseOptions] = {}

Returns Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]

def decode_schema_string[T, RD, E = Never, D = None](
    schema: Codec[T, Any, RD, object],
    /,
    *,
    headers: bool = False,
    delimiter: str = ",",
    max_record_size: float = DEFAULT_MAX_RECORD_SIZE,
    ignore_empty_lines: bool = False,
    disable_codecs: bool = False,
    **options: Unpack[ParseOptions],
) -> Channel[tuple[T, ...], E | CsvError | SchemaError, D, tuple[str, ...], E, D, RD]:
    """Parse chunks of text as ``decode_string`` does and decode each record
    with ``schema``: a dict keyed by header name with ``headers``, else a row.
    By default the schema is wrapped in ``to_codec_string_tree``, so columns
    may be numbers, booleans or dates, and with ``headers`` an empty cell of a
    key that is optional on the encoded side is a missing key;
    ``disable_codecs`` decodes with ``schema`` as given. A column that does
    not encode to a string (a list, a nested struct) raises ``ValueError``
    here. Each chunk is decoded at once; a failure is a ``SchemaError`` whose
    paths start with the record number."""
    codec = _to_codec(schema, disable_codecs)
    encoded = _resolve_suspend(ast.to_encoded(codec.__codec__().ast))
    _check_cells(encoded, None)
    # CSV cannot tell an empty cell from a missing one, so empty cells of
    # optional keys are decoded as missing keys.
    optional = (
        [
            property.name
            for property in encoded.properties
            if isinstance(property.name, str)
            and (property.optional or ast.is_optional(property.type))
        ]
        if not disable_codecs and headers and isinstance(encoded, ast.Objects)
        else []
    )

    def value_of(record: Decoded) -> object:
        value = record.value
        if not optional or not isinstance(value, dict):
            return value
        # Header records are made per row, so they can be changed in place.
        for key in optional:
            if value.get(key) == "":
                del value[key]
        return value

    decode = decode_with(NonEmptyArray(codec), **options)

    def decoded(records: tuple[Decoded, ...]) -> Any:
        return decode(tuple(map(value_of, records))).map_error(
            lambda error: _rebase(error, lambda index: records[index].record)
        )

    channel: _AnyChannel = decode_records(delimiter, max_record_size, ignore_empty_lines, headers)
    return channel.map_effect(decoded)

encode_schema

encode_schema[T, RE, E = Never, D = None](schema, /, *, columns=None, headers=False, delimiter=',', newline='\r\n', disable_codecs=False) -> Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]

encode_schema_string as UTF-8 bytes, one bytes per input chunk. When the bytes stream an HTTP response body, an encoding failure ends the body; it cannot change a status already sent.

ParameterTypeDescription
schemaCodec[T, Any, object, RE]
columnsSequence[str] | None = None
headersbool = False
delimiterstr = ','
newlineNewline = '\r\n'
disable_codecsbool = False

Returns Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]

def encode_schema[T, RE, E = Never, D = None](
    schema: Codec[T, Any, object, RE],
    /,
    *,
    columns: Sequence[str] | None = None,
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
    disable_codecs: bool = False,
) -> Channel[tuple[bytes, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]:
    """``encode_schema_string`` as UTF-8 bytes, one ``bytes`` per input
    chunk. When the bytes stream an HTTP response body, an encoding failure
    ends the body; it cannot change a status already sent."""
    from pyfect.encoding.csv._channels import _joined_bytes

    records = _encode_schema_records(
        schema,
        delimiter=delimiter,
        newline=newline,
        columns=columns,
        headers=headers,
        disable_codecs=disable_codecs,
    )
    return records.map(_joined_bytes)

encode_schema_string

encode_schema_string[T, RE, E = Never, D = None](schema, /, *, columns=None, headers=False, delimiter=',', newline='\r\n', disable_codecs=False) -> Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]

Encode each value with schema and write it as CSV text, one string per record. By default the schema is wrapped in to_codec_string_tree, so every column encodes to a string (disable_codecs keeps it as given). columns defaults to a struct's encoded keys (renamed ones too) and is needed for other mappings; tuples and arrays write cells by position and use columns only as header names. A missing or None value is an empty cell. A column that does not encode to a string raises ValueError here. A failure is a SchemaError whose paths start with the output record number, the header row counted.

ParameterTypeDescription
schemaCodec[T, Any, object, RE]
columnsSequence[str] | None = None
headersbool = False
delimiterstr = ','
newlineNewline = '\r\n'
disable_codecsbool = False

Returns Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]

def encode_schema_string[T, RE, E = Never, D = None](
    schema: Codec[T, Any, object, RE],
    /,
    *,
    columns: Sequence[str] | None = None,
    headers: bool = False,
    delimiter: str = ",",
    newline: Newline = "\r\n",
    disable_codecs: bool = False,
) -> Channel[tuple[str, ...], E | SchemaError, D, tuple[T, ...], E, D, RE]:
    """Encode each value with ``schema`` and write it as CSV text, one string
    per record. By default the schema is wrapped in ``to_codec_string_tree``,
    so every column encodes to a string (``disable_codecs`` keeps it as
    given). ``columns`` defaults to a struct's encoded keys (renamed ones
    too) and is needed for other mappings; tuples and arrays write cells by
    position and use ``columns`` only as header names. A missing or ``None``
    value is an empty cell. A column that does not encode to a string raises
    ``ValueError`` here. A failure is a ``SchemaError`` whose paths start
    with the output record number, the header row counted."""
    return _encode_schema_records(
        schema,
        delimiter=delimiter,
        newline=newline,
        columns=columns,
        headers=headers,
        disable_codecs=disable_codecs,
    )