diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 6f837d21861d..e765b732b8b6 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -48,6 +48,78 @@ _ensure_filesystem) from pyarrow.util import guid, _is_path_like, _stringify_path, _deprecate_api +_read_docstring_common = """\ +read_dictionary : list, default None + List of names or column paths (for nested types) to read directly + as DictionaryArray. Only supported for BYTE_ARRAY storage. To read + a flat column as dictionary-encoded pass the column name. For + nested types, you must pass the full column "path", which could be + something like level1.level2.list.item. Refer to the Parquet + file's schema to obtain the paths. +binary_type : pyarrow.DataType, default None + If given, Parquet binary columns will be read as this datatype. + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +list_type : subclass of pyarrow.DataType, default None + If given, non-MAP repeated columns will be read as an instance of + this datatype (either pyarrow.ListType or pyarrow.LargeListType). + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +memory_map : bool, default False + If the source is a file path, use a memory map to read file, which can + improve performance in some environments. +buffer_size : int, default 0 + If positive, perform read buffering when deserializing individual + column chunks. Otherwise IO calls are unbuffered. +pre_buffer : bool, default True + Coalesce and issue file reads in parallel to improve performance on + high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a + background I/O thread pool. If using a filesystem layer that itself + performs readahead (e.g. fsspec's S3FS), disable readahead for best + results. Set to False if you want to prioritize minimal memory usage + over maximum speed. +coerce_int96_timestamp_unit : str, default None + Cast timestamps that are stored in INT96 format to a particular resolution + (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 + timestamps will be inferred as timestamps in nanoseconds. +decryption_properties : FileDecryptionProperties or None + File-level decryption properties. + The decryption properties can be created using + ``CryptoFactory.file_decryption_properties()``. +thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +schema_depth_limit : int, default None + If not None, override the maximum nesting depth of the Parquet file schema. + This guards against recursion overflow on invalid schemas. + The default limit should be sufficient for most Parquet files. +filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem. +page_checksum_verification : bool, default False + If True, verify the page checksum for each page read from the file. +arrow_extensions_enabled : bool, default True + If True, read Parquet logical types as Arrow extension types where possible, + (e.g., read JSON as the canonical `arrow.json` extension type or UUID as + the canonical `arrow.uuid` extension type).""" + + +_read_docstring_dataset = """\ +partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ +default "hive" + The partitioning scheme for a partitioned dataset. The default of "hive" + assumes directory names with key=value pairs like "/year=2009/month=11". + In addition, a scheme like "/2009/11" is also supported, in which case + you need to specify the field names or a full schema. If no partitioning + is used, pass ``None``. + See the ``pyarrow.dataset.partitioning()`` function for more details.""" + def _check_contains_null(val): if isinstance(val, bytes): @@ -206,7 +278,7 @@ def convert_single_predicate(col, op, val): class ParquetFile: - """ + __doc__ = f""" Reader interface for a single Parquet file. Parameters @@ -219,59 +291,7 @@ class ParquetFile: common_metadata : FileMetaData, default None Will be used in reads for pandas schema metadata if not found in the main file's metadata, no other uses at the moment. - read_dictionary : list - List of column names to read directly as DictionaryArray. - binary_type : pyarrow.DataType, default None - If given, Parquet binary columns will be read as this datatype. - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. - list_type : subclass of pyarrow.DataType, default None - If given, non-MAP repeated columns will be read as an instance of - this datatype (either pyarrow.ListType or pyarrow.LargeListType). - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. - memory_map : bool, default False - If the source is a file path, use a memory map to read file, which can - improve performance in some environments. - buffer_size : int, default 0 - If positive, perform read buffering when deserializing individual - column chunks. Otherwise IO calls are unbuffered. - pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. Set to False if you want to prioritize minimal memory usage - over maximum speed. - coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular - resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' - and therefore INT96 timestamps will be inferred as timestamps - in nanoseconds. - decryption_properties : FileDecryptionProperties, default None - File decryption properties for Parquet Modular Encryption. - thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. - filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. - page_checksum_verification : bool, default False - If True, verify the checksum for each page read from the file. - arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where - possible (e.g., read JSON as the canonical `arrow.json` extension type - or UUID as the canonical `arrow.uuid` extension type). + {_read_docstring_common.replace(chr(10), chr(10) + " ")} Examples -------- @@ -279,9 +299,9 @@ class ParquetFile: Generate an example PyArrow Table and write it to Parquet file: >>> import pyarrow as pa - >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + >>> table = pa.table({{'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", - ... "Brittle stars", "Centipede"]}) + ... "Brittle stars", "Centipede"]}}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') @@ -1256,39 +1276,6 @@ def _get_pandas_index_columns(keyvalues): EXCLUDED_PARQUET_PATHS = {'_SUCCESS'} -_read_docstring_common = """\ -read_dictionary : list, default None - List of names or column paths (for nested types) to read directly - as DictionaryArray. Only supported for BYTE_ARRAY storage. To read - a flat column as dictionary-encoded pass the column name. For - nested types, you must pass the full column "path", which could be - something like level1.level2.list.item. Refer to the Parquet - file's schema to obtain the paths. -binary_type : pyarrow.DataType, default None - If given, Parquet binary columns will be read as this datatype. - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -list_type : subclass of pyarrow.DataType, default None - If given, non-MAP repeated columns will be read as an instance of - this datatype (either pyarrow.ListType or pyarrow.LargeListType). - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -memory_map : bool, default False - If the source is a file path, use a memory map to read file, which can - improve performance in some environments. -buffer_size : int, default 0 - If positive, perform read buffering when deserializing individual - column chunks. Otherwise IO calls are unbuffered. -partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ -default "hive" - The partitioning scheme for a partitioned dataset. The default of "hive" - assumes directory names with key=value pairs like "/year=2009/month=11". - In addition, a scheme like "/2009/11" is also supported, in which case - you need to specify the field names or a full schema. If no partitioning - is used, pass ``None``. - See the ``pyarrow.dataset.partitioning()`` function for more details.""" - - _parquet_dataset_example = """\ Generate an example PyArrow Table and write it to a partitioned dataset: @@ -1336,10 +1323,6 @@ class ParquetDataset: ---------- path_or_paths : str or List[str] A directory name, single file name, or list of file names. -filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. schema : pyarrow.parquet.Schema Optionally provide the Schema for the Dataset, in which case it will not be inferred from the source. @@ -1350,46 +1333,14 @@ class ParquetDataset: Within-file level filtering and different partitioning schemes are supported. {_DNF_filter_doc} -{_read_docstring_common} +{_read_docstring_dataset} ignore_prefixes : list, optional Files matching any of these prefixes will be ignored by the discovery process. This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. -pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. Set to False if you want to prioritize minimal memory usage - over maximum speed. -coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular resolution - (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 - timestamps will be inferred as timestamps in nanoseconds. -decryption_properties : FileDecryptionProperties or None - File-level decryption properties. - The decryption properties can be created using - ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. -page_checksum_verification : bool, default False - If True, verify the page checksum for each page read from the file. -arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where possible, - (e.g., read JSON as the canonical `arrow.json` extension type or UUID as - the canonical `arrow.uuid` extension type). +{_read_docstring_common} Examples -------- @@ -1739,8 +1690,8 @@ def partitioning(self): return self._dataset.partitioning -_read_table_docstring = """ -{0} +_read_table_docstring = f""" +{{0}} Parameters ---------- @@ -1760,63 +1711,27 @@ def partitioning(self): schema : Schema, optional Optionally provide the Schema for the parquet dataset, in which case it will not be inferred from the source. -{1} -filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. +{{1}} filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None Rows which do not match the filter predicate will be removed from scanned data. Partition keys embedded in a nested directory structure will be exploited to avoid loading files at all if they contain no matching rows. Within-file level filtering and different partitioning schemes are supported. - {3} + {{3}} ignore_prefixes : list, optional Files matching any of these prefixes will be ignored by the discovery process. This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. -pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. -coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular - resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' - and therefore INT96 timestamps will be inferred as timestamps - in nanoseconds. -decryption_properties : FileDecryptionProperties or None - File-level decryption properties. - The decryption properties can be created using - ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. -page_checksum_verification : bool, default False - If True, verify the checksum for each page read from the file. -arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where possible, - (e.g., read JSON as the canonical `arrow.json` extension type or UUID as - the canonical `arrow.uuid` extension type). +{_read_docstring_common} Returns ------- -{2} +{{2}} -{4} +{{4}} """ _read_table_example = """\ @@ -1988,7 +1903,7 @@ def read_table(source, *, columns=None, use_threads=True, """Read a Table from Parquet format""", "\n".join(("""use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that - index columns are also loaded.""", _read_docstring_common)), + index columns are also loaded.""", _read_docstring_dataset)), """pyarrow.Table Content of the file as a table (of columns)""", _DNF_filter_doc, _read_table_example) @@ -2003,7 +1918,7 @@ def read_pandas(source, columns=None, **kwargs): read_pandas.__doc__ = _read_table_docstring.format( 'Read a Table from Parquet format, also reading DataFrame\n' 'index values if known in the file metadata', - "\n".join((_read_docstring_common, + "\n".join((_read_docstring_dataset, """**kwargs additional options for :func:`read_table`""")), """pyarrow.Table